bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From evan...@apache.org
Subject [bigtop] branch master updated: BIGTOP-3175. [Build] Support to specify commit SHA1 when building packages (#484)
Date Tue, 26 Feb 2019 10:47:35 GMT
This is an automated email from the ASF dual-hosted git repository.

evansye pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/master by this push:
     new afa1faf  BIGTOP-3175. [Build] Support to specify commit SHA1 when building packages
(#484)
afa1faf is described below

commit afa1fafc7ac81dab5bde6f5c97c7a425bb625bb5
Author: Evans Ye <evansye@apache.org>
AuthorDate: Tue Feb 26 18:47:31 2019 +0800

    BIGTOP-3175. [Build] Support to specify commit SHA1 when building packages (#484)
---
 README.md       | 51 +++++++++++++++++++++++++++++++++++++++++----------
 packages.gradle | 45 +++++++++++++++++++++++++++++++++++++++------
 2 files changed, 80 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index 4e25d6e..460aaa2 100644
--- a/README.md
+++ b/README.md
@@ -220,33 +220,64 @@ The source for the website is located in "project_root/src/site/".
 For Developers: Building a component from Git repository
 --------------------------------------------------------
 
-To fetch source from a Git repository you need to modify `bigtop.bom` and add the
-following JSON snippets to your component/package:
+To fetch source from a Git repository, there're two ways to achieve this:
+a). modify `bigtop.bom` and add JSON snippets to your component/package, or
+b). specify properties at command line
+
+* __bigtop.bom__
+
+Add following JSON snippets to the desired component/package:
 
 ```
-git     { repo = ""; ref = ""; dir = ""}
+git     { repo = ""; ref = ""; dir = ""; commit_hash = "" }
 ```
 
-* `repo` - SSH, HTTP or local path to Git repo.
-* `ref` - branch, tag or commit hash to check out.
-* `dir` - directory name to write source into.
+  * `repo` - SSH, HTTP or local path to Git repo.
+  * `ref` - branch, tag or commit hash to check out.
+  * `dir` - [OPTIONAL] directory name to write source into.
+  * `commit_hash` - [OPTIONAL] a commit hash to reset to.
 
 Some packages have different names for source directory and source tarball
 (`hbase-0.98.5-src.tar.gz` contains `hbase-0.98.5` directory).
-By default source will be fetched in a directory named by `tarball { destination = TARBALL_DST}`
+By default source will be fetched in a directory named by `tarball { source = TARBALL_SRC
}`
 without `.t*` extension.
 To explicitly set directory name use the `dir` option.
 
+When `commit_hash` specified, the repo to build the package will be reset to the commit hash.
+
 Example for HBase:
 
 ```
       name    = 'hbase'
-      version { base = '1.1.9'; pkg = base; release = 1 }
+      version { base = '1.3.2'; pkg = base; release = 1 }
       git     { repo = "https://github.com/apache/hbase.git"
-                ref  = "${version.base}"
-                dir  = "${name}-${version.base}" }
+                ref  = "branch-1.3"
+                dir  = "${name}-${version.base}"
+                commit_hash = "1bedb5bfbb5a99067e7bc54718c3124f632b6e17"
+              }
+```
+
+* __command line__
+
+
+```
+./gradlew COMPONENT-pkg -Pgit_repo="" -Pgit_ref="" -Pgit_dir="" -Pgit_commit_hash="" -Pbase_version=""
 ```
 
+Where `git_repo`, `git_ref`, `git_dir`, and `git_commit_hash` are exactly the same with what
we set in JSON.
+And `base_version` is to overwrite:
+```
+      version { base = ''}
+```
+
+Example for Kafka:
+
+```
+./gradlew kafka-pkg -Pgit_repo=https://github.com/apache/kafka.git -Pgit_ref=1.1 -Pgit_sha1=4dae083af486eaedd27c69c973c74605bffd416b
-Pbase_version=1.1.1 --info
+```
+
+You can mix both ways to build from Git, but command line always overwrites `bigtop.bom`.
+
 
 Contact us
 ----------
diff --git a/packages.gradle b/packages.gradle
index fb02472..9ff524b 100644
--- a/packages.gradle
+++ b/packages.gradle
@@ -173,13 +173,15 @@ def genTasks = { target ->
       description: "Download $target artifacts",
       group: PACKAGES_GROUP) doLast {
 
+    def final TARBALL_SRC = config.bigtop.components[target].tarball.source
     def final TARBALL_DST = config.bigtop.components[target].tarball.destination
     def final DOWNLOAD_DST = config.bigtop.components[target].downloaddst
     def final DOWNLOAD_URL = config.bigtop.components[target].downloadurl
 
-    def final GIT_REPO = config.bigtop.components[target].git.repo
-    def final GIT_REF = config.bigtop.components[target].git.ref
-    def final GIT_DIR = config.bigtop.components[target].git.dir
+    def final GIT_REPO = project.hasProperty('git_repo') ? project.property('git_repo') :
config.bigtop.components[target].git.repo
+    def final GIT_REF = project.hasProperty('git_ref') ? project.property('git_ref') : config.bigtop.components[target].git.ref
+    def final GIT_DIR = project.hasProperty('git_dir') ? project.property('git_dir') : config.bigtop.components[target].git.dir
+    def final GIT_COMMIT_HASH = project.hasProperty('git_commit_hash') ? project.property('git_commit_hash')
: config.bigtop.components[target].git.commit_hash
     def final GIT_USER_NAME = config.bigtop.components[target].git.user ?: config.bigtop.git.user
     def final GIT_ACCESS_TOKEN = config.bigtop.components[target].git.token ?: config.bigtop.git.token
 
@@ -194,9 +196,13 @@ def genTasks = { target ->
     if (GIT_REPO && GIT_REF) {
       def dir = GIT_DIR
       if (dir == null || dir.isEmpty()) {
-        dir = TARBALL_DST.substring(0, TARBALL_DST.lastIndexOf(".t"))
+        dir = TARBALL_SRC.substring(0, TARBALL_SRC.lastIndexOf(".t"))
       }
       delete("${DL_DIR}/${dir}")
+      def depth = "--depth 1"
+      if ( GIT_COMMIT_HASH ) {
+        depth = ""
+      }
       if (GIT_USER_NAME && GIT_ACCESS_TOKEN) {
         def uri = new URI(GIT_REPO)
         if (!GIT_REPO.toLowerCase().startsWith("http")) {
@@ -209,14 +215,14 @@ def genTasks = { target ->
          uri.getFragment())
         exec {
           workingDir DL_DIR
-          commandLine "git clone --depth 1 --branch $GIT_REF ${uri} ${dir}".split()
+          commandLine "git clone ${depth} --branch $GIT_REF ${uri} ${dir}".split()
           errorOutput devNull
           standardOutput devNull
         }
       } else {
         exec {
           workingDir DL_DIR
-          commandLine "git clone --depth 1 --branch $GIT_REF ${GIT_REPO} ${dir}".split()
+          commandLine "git clone ${depth} --branch $GIT_REF ${GIT_REPO} ${dir}".split()
           errorOutput devNull
           standardOutput devNull
         }
@@ -227,6 +233,14 @@ def genTasks = { target ->
         errorOutput devNull
         standardOutput devNull
       }
+      if ( GIT_COMMIT_HASH ) {
+        exec {
+          workingDir "$DL_DIR/${dir}"
+          commandLine "git reset $GIT_COMMIT_HASH --hard".split()
+          errorOutput devNull
+          standardOutput devNull
+        }
+      }
       delete("${DL_DIR}/${dir}/.git")
       exec {
         workingDir DL_DIR
@@ -618,6 +632,12 @@ def genTasks = { target ->
     def _OS = project.hasProperty("OS") ? OS : "centos-7"
     def _nexus = project.hasProperty("nexus") ? nexus : false
     def _target_pkg = "$target-pkg"
+    def additionalConfigKeys = ['git_repo', 'git_ref', 'git_dir', 'git_commit_hash', 'base_version']
+    additionalConfigKeys.each { key ->
+      if (project.hasProperty(key)) {
+        _target_pkg += " -P${key}=" + project.property(key)
+      }
+    }
     def command = [
             'bash', '-x',
             './bigtop-ci/build.sh',
@@ -640,6 +660,19 @@ def genTasks = { target ->
     println "Base: ${config.bigtop.components[target].version.base}"
   }
   task "${target}_vardefines" doLast {
+    if (project.hasProperty('base_version')) {
+      def BASE_VERSION = project.property("base_version")
+      if (config.bigtop.components[target].tarball.source) {
+        config.bigtop.components[target].tarball.source = config.bigtop.components[target].tarball.source
+                .replaceAll(config.bigtop.components[target].version.base, BASE_VERSION)
+      }
+      config.bigtop.components[target].tarball.destination = config.bigtop.components[target].tarball.destination
+              .replaceAll(config.bigtop.components[target].version.base, BASE_VERSION)
+
+      config.bigtop.components[target].version.base = BASE_VERSION
+      config.bigtop.components[target].version.pkg = BASE_VERSION
+    }
+
     setDefaults(config.bigtop.components[target])
 
     config.bigtop.components[target].package.release = '1'


Mime
View raw message