bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From evan...@apache.org
Subject [bigtop] branch master updated: BIGTOP-3189. Allow to specify more properties/options at command line for docker provisioner (#491)
Date Thu, 14 Mar 2019 17:13:10 GMT
This is an automated email from the ASF dual-hosted git repository.

evansye pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/master by this push:
     new 21297aa  BIGTOP-3189. Allow to specify more properties/options at command line for
docker provisioner (#491)
21297aa is described below

commit 21297aa7b5afffc8b0d2c44e0fb7e44b2a32ff69
Author: Evans Ye <evansye@apache.org>
AuthorDate: Fri Mar 15 01:13:06 2019 +0800

    BIGTOP-3189. Allow to specify more properties/options at command line for docker provisioner
(#491)
---
 build.gradle                        | 64 ++++++++++++++++++++++++++++++-----
 packages.gradle                     |  2 +-
 provisioner/docker/docker-hadoop.sh | 67 +++++++++++++++++++++++++------------
 3 files changed, 101 insertions(+), 32 deletions(-)

diff --git a/build.gradle b/build.gradle
index d7e1045..dafa5c5 100644
--- a/build.gradle
+++ b/build.gradle
@@ -257,10 +257,36 @@ task "toolchain-devtools"(type:Exec,
 }
 
 task "docker-provisioner"(type:Exec,
-    description: 'Provision a Bigtop cluster on Docker container(s). Default to CentOS and
1 node.\nCreate a 3 node cluster:\n  $ ./gradlew -Pnum_instances=3 docker-provisioner\nRun
smoke tests on 3 node cluster:\n  $ ./gradlew -Pnum_instances=3 -Prun_smoke_tests=true docker-provisioner\nSpecify
configuration file (located in provisioner/docker/):\n  $ ./gradlew -Pconfig=config_debian8.yaml
docker-provisioner',
+    description: '''
+Provision a Bigtop stack cluster on Docker container(s). Default to CentOS and 1 node.
+Properties:
+  -Pconfig=[CONFIG_FILE] (located under provisioner/docker/)
+  -Penable_local_repo
+  -Pimage=[DOCKER_IMAGE] (overwrites -POS and -Pprefix)
+  -Pnum_instances=[NUM_INSTANCES]
+  -Pnexus=[NEXUS_URL] (NEXUS_URL is optional)
+  -POS=[centos-7|fedora-26|debian-9|ubuntu-16.04|opensuse-42.3]
+  -Pprefix=[trunk|1.2.1|1.2.0|1.1.0|...]
+  -Prun_smoke_tests (run test components defined in config file)
+  -Psmoke_tests=[COMPONENTS]
+  -Pstack=[COMPONENTS]
+
+For detailed explaination, please refer to the helper message:
+  $ provisioner/docker/docker-hadoop.sh --help
+
+Examples:
+Create a 3 node cluster:
+  $ ./gradlew -Pnum_instances=3 docker-provisioner
+Run smoke tests on 3 node cluster:
+  $ ./gradlew -Pnum_instances=3 -Prun_smoke_tests docker-provisioner
+Specify custom configuration file:
+  $ ./gradlew -Pconfig=config_debian-9.yaml docker-provisioner
+''',
     group: DEPLOY_GROUP) {
   def _config = project.hasProperty("config") ? config : "config.yaml"
   def _num_instances = project.hasProperty("num_instances") ? num_instances : 1
+  def _OS = project.hasProperty("OS") ? OS : 'centos-7'
+  def _prefix = project.hasProperty("prefix") ? prefix : 'trunk'
   def command = [
       './docker-hadoop.sh',
       '-C', _config,
@@ -279,6 +305,18 @@ task "docker-provisioner"(type:Exec,
   }
   if ( project.hasProperty("nexus") ) {
     command.add('--nexus')
+    command.add("${nexus}")
+  }
+  if ( project.hasProperty("enable_local_repo") ) {
+    command.add('--enable-local-repo')
+  }
+  if (project.hasProperty("OS") || project.hasProperty("prefix")) {
+    command.add('--image')
+    command.add("bigtop/puppet:${_prefix}-${_OS}")
+  }
+  if (project.hasProperty("image")) {
+    command.add('--image')
+    command.add("${image}")
   }
   workingDir 'provisioner/docker'
   commandLine command
@@ -450,10 +488,14 @@ allprojects {
 }
 
 task "bigtop-puppet"(type:Exec,
-    description: 'Build bigtop/puppet images.\n' +
-      'Usage:\n  $ ./gradlew -POS=[centos-7|fedora-26|debian-9|ubuntu-16.04|opensuse-42.3]
-Pprefix=STRING_TO_PREFIX bigtop-puppet\n' +
-      'Example:\n  $ ./gradlew -POS=debian-9 -Pprefix=1.0.0 bigtop-puppet\n' +
-      'The built image name: bigtop/puppet:1.0.0-debian-9',
+    description: '''
+Build bigtop/puppet images
+Usage:
+  $ ./gradlew -POS=[centos-7|fedora-26|debian-9|ubuntu-16.04|opensuse-42.3] -Pprefix=[trunk|1.2.1|1.2.0|1.1.0|...]
bigtop-puppet
+Example:
+  $ ./gradlew -POS=debian-9 -Pprefix=1.0.0 bigtop-puppet
+  The built image name: bigtop/puppet:1.0.0-debian-9
+      ''',
     group: DOCKERBUILD_GROUP) {
   def _prefix = project.hasProperty("prefix") ? prefix : "trunk"
   def _OS = project.hasProperty("OS") ? OS : "centos-7"
@@ -465,10 +507,14 @@ task "bigtop-puppet"(type:Exec,
 }
 
 task "bigtop-slaves"(type:Exec,
-    description: 'Build bigtop/slaves images.\n' +
-      'Usage:\n  $ ./gradlew -POS=[centos-7|fedora-26|debian-9|ubuntu-16.04|opensuse-42.3]
-Pprefix=STRING_TO_PREFIX bigtop-slaves\n' +
-      'Example:\n  $ ./gradlew -POS=debian-9 -Pprefix=1.0.0 bigtop-slaves\n' +
-      'The built image name: bigtop/slaves:1.0.0-debian-9',
+    description: '''
+Build bigtop/slaves images
+Usage:
+  $ ./gradlew -POS=[centos-7|fedora-26|debian-9|ubuntu-16.04|opensuse-42.3] -Pprefix=[trunk|1.2.1|1.2.0|1.1.0|...]
bigtop-slaves
+Example:
+  $ ./gradlew -POS=debian-9 -Pprefix=1.0.0 bigtop-slaves
+  The built image name: bigtop/slaves:1.0.0-debian-9
+      ''',
     group: DOCKERBUILD_GROUP) {
   def _prefix = project.hasProperty("prefix") ? prefix : "trunk"
   def _OS = project.hasProperty("OS") ? OS : "centos-7"
diff --git a/packages.gradle b/packages.gradle
index 202aa9d..2e069c2 100644
--- a/packages.gradle
+++ b/packages.gradle
@@ -629,7 +629,7 @@ def genTasks = { target ->
   task "$target-pkg-ind" (
           description: "Invoking a native binary packaging for $target in Docker. Usage:
\$ ./gradlew " +
                   "-POS=[centos-7|fedora-26|debian-9|ubuntu-16.04|opensuse-42.3] " +
-                  "-Pprefix=[trunk|1.2.1|1.2.0|1.1.0|...] $target-pkg-ind" +
+                  "-Pprefix=[trunk|1.2.1|1.2.0|1.1.0|...] $target-pkg-ind " +
                   "-Pnexus=[true|false]",
           group: PACKAGES_GROUP) doLast {
     def _prefix = project.hasProperty("prefix") ? prefix : "trunk"
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index 2bc6fd9..bf8b53d 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -17,20 +17,29 @@
 
 usage() {
     echo "usage: $PROG [-C file] args"
-    echo "       -C file                                   Use alternate file for config.yaml"
+    echo "       -C file                                   - Use alternate file for config.yaml"
     echo "  commands:"
-    echo "       -c NUM_INSTANCES, --create NUM_INSTANCES  Create a Docker based Bigtop Hadoop
cluster"
-    echo "       -d, --destroy                             Destroy the cluster"
-    echo "       -e, --exec INSTANCE_NO|INSTANCE_NAME      Execute command on a specific
instance. Instance can be specified by name or number."
-    echo "                                                 For example: $PROG --exec 1 bash"
-    echo "                                                              $PROG --exec docker_bigtop_1
bash"
-    echo "       -E, --env-check                           Check whether required tools has
been installed"
-    echo "       -l, --list                                List out container status for
the cluster"
-    echo "       -p, --provision                           Deploy configuration changes"
-    echo "       -n, --nexus [NEXUS_URL]                   Configure Nexus proxy to speed
up test execution"
-    echo "                                                 If no NEXUS_URL specified, default
to http://NEXUS_IP:8081/nexus,"
-    echo "                                                 where NEXUS_IP is the ip of the
container named nexus"
-    echo "       -s, --smoke-tests                         Run Bigtop smoke tests"
+    echo "       -c NUM_INSTANCES, --create NUM_INSTANCES  - Create a Docker based Bigtop
Hadoop cluster"
+    echo "       -d, --destroy                             - Destroy the cluster"
+    echo "       -e, --exec INSTANCE_NO|INSTANCE_NAME      - Execute command on a specific
instance. Instance can be specified by name or number"
+    echo "                                                   For example: $PROG --exec 1
bash"
+    echo "                                                                $PROG --exec docker_bigtop_1
bash"
+    echo "       -E, --env-check                           - Check whether required tools
has been installed"
+    echo "       -k, --stack COMPONENTS                    - Overwrite the components to
deploy defined in config file"
+    echo "                                                   COMPONENTS is a comma separated
string"
+    echo "                                                   For example: $PROG -c 3 --stack
hdfs"
+    echo "                                                                $PROG -c 3 --stack
'hdfs, yarn, spark'"
+    echo "       -l, --list                                - List out container status for
the cluster"
+    echo "       -n, --nexus NEXUS_URL                     - Configure Nexus proxy to speed
up test execution"
+    echo "                                                   NEXUS_URL is optional. If not
specified, default to http://NEXUS_IP:8081/nexus"
+    echo "                                                   Where NEXUS_IP is the ip of
container named nexus"
+    echo "       -p, --provision                           - Deploy configuration changes"
+    echo "       -r, --enable-local-repo                   - Whether to use repo created
at local file system. You can get one by $ ./gradlew repo"
+    echo "       -s, --smoke-tests COMPONENTS              - Run Bigtop smoke tests"
+    echo "                                                   COMPONENTS is optional. If not
specified, default to smoke_test_components in config file"
+    echo "                                                   COMPONENTS is a comma separated
string"
+    echo "                                                   For example: $PROG -c 3 --smoke-tests
hdfs"
+    echo "                                                                $PROG -c 3 --smoke-tests
'hdfs, yarn, mapreduce'"
     echo "       -h, --help"
     exit 1
 }
@@ -47,7 +56,9 @@ create() {
     echo > ./config/hiera.yaml
     echo > ./config/hosts
     # set correct image name based on running architecture
-    image_name=$(get-yaml-config docker image)
+    if [ -z ${image_name+x} ]; then
+        image_name=$(get-yaml-config docker image)
+    fi
     running_arch=$(uname -m)
     if [ "x86_64" == ${running_arch} ]; then
         image_name=${image_name}
@@ -74,7 +85,9 @@ create() {
         components="[`echo $(get-yaml-config components) | sed 's/ /, /g'`]"
     fi
     distro=$(get-yaml-config distro)
-    enable_local_repo=$(get-yaml-config enable_local_repo)
+    if [ -z ${enable_local_repo+x} ]; then
+        enable_local_repo=$(get-yaml-config enable_local_repo)
+    fi
     generate-config "$hadoop_head_node" "$repo" "$components"
 
     # Start provisioning
@@ -268,6 +281,20 @@ while [ $# -gt 0 ]; do
     -E|--env-check)
         env-check
         shift;;
+    -k|--stack)
+        if [ $# -lt 2 ]; then
+          log "No stack specified"
+          usage
+        fi
+        components="[$2]"
+        shift 2;;
+    -i|--image)
+        if [ $# -lt 2 ]; then
+          log "No image specified"
+          usage
+        fi
+        image_name=$2
+        shift 2;;
     -l|--list)
         list
         shift;;
@@ -285,16 +312,12 @@ while [ $# -gt 0 ]; do
             shift 2
         fi
         ;;
-    -k|--stack)
-        if [ $# -lt 2 ]; then
-          log "No stack specified"
-          usage
-        fi
-        components="[$2]"
-        shift 2;;
     -p|--provision)
         provision
         shift;;
+    -r|--enable-local-repo)
+        enable_local_repo=true
+        shift;;
     -s|--smoke-tests)
         if [ $# -lt 2 ] || [[ $2 == -* ]]; then
             shift


Mime
View raw message