hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bus...@apache.org
Subject [3/7] hbase git commit: HBASE-19124 Move nightly test of source artifact creation from Jenkinsfile to a script.
Date Thu, 02 Nov 2017 20:36:31 GMT
HBASE-19124 Move nightly test of source artifact creation from Jenkinsfile to a script.

Signed-off-by: Apekshit Sharma <appy@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/27d8a6d1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/27d8a6d1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/27d8a6d1

Branch: refs/heads/branch-1
Commit: 27d8a6d126f2d527291bc1b12a4d9319eba6e16b
Parents: de08793
Author: Sean Busbey <busbey@apache.org>
Authored: Wed Nov 1 08:12:03 2017 -0500
Committer: Sean Busbey <busbey@apache.org>
Committed: Thu Nov 2 15:32:48 2017 -0500

----------------------------------------------------------------------
 dev-support/Jenkinsfile                      |  61 +++-----
 dev-support/hbase_nightly_source-artifact.sh | 168 ++++++++++++++++++++++
 2 files changed, 186 insertions(+), 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/27d8a6d1/dev-support/Jenkinsfile
----------------------------------------------------------------------
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 10438a4..dfac5b3 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -258,51 +258,26 @@ curl -L  -o personality.sh "${env.PROJET_PERSONALITY}"
         jdk "JDK 1.7 (latest)"
       }
       steps {
-        sh 'mvn --version --offline'
-        dir ("component") {
-          sh '''#!/bin/bash -e
-            echo "Clean..."
-            mvn --batch-mode -DskipTests clean
-            echo "Step 3 Build the source tarball"
-            mvn -Prelease --batch-mode -Dmaven.repo.local="${WORKSPACE}/.m2-for-repo" install
-DskipTests assembly:single -Dassembly.file=hbase-assembly/src/main/assembly/src.xml
-'''
-        }
-        dir ("unpacked_src_tarball") {
-          sh '''#!/bin/bash -e
-            echo "Unpack the source tarball"
-            tar --strip-components=1 -xzf ../component/hbase-assembly/target/hbase-*-src.tar.gz
-            echo "Building from source artifact."
-            mvn -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${WORKSPACE}/.m2-for-src"
clean install assembly:single
-            echo "Clean up after checking ability to build."
-            mvn -DskipTests --batch-mode clean
-'''
-        }
-        dir ("component") {
-          sh '''#!/bin/bash -e
-            echo "Clean up the source checkout"
-            mvn -DskipTests --batch-mode clean
-            echo "Diff against source tree"
-            diff --binary --recursive . ../unpacked_src_tarball >../diff_output || true
-'''
-        }
-        // expectation check largely based on HBASE-14952
         sh '''#!/bin/bash -e
-          echo "Checking against things we don't expect to include in the source tarball
(git related, hbase-native-client, etc.)"
-          cat >known_excluded <<END
-Only in .: .git
-Only in .: .gitattributes
-Only in .: .gitignore
-Only in .: hbase-native-client
-END
-          if ! diff known_excluded diff_output >/dev/null ; then
-            echo "Any output here are unexpected differences between the source artifact
we'd make for an RC and the current branch."
-            echo "The expected differences are on the < side and the current differences
are on the > side."
-            echo "In a given set of differences, '.' refers to the branch in the repo and
'unpacked_src_tarball' refers to what we pulled out of the tarball."
-            diff known_excluded diff_output
-          else
-            echo "Everything looks as expected."
-          fi
+          echo "Setting up directories"
+          rm -rf "output-srctarball" && mkdir "output-srctarball"
+          rm -rf "unpacked_src_tarball" && mkdir "unpacked_src_tarball"
+          rm -rf ".m2-for-repo" && mkdir ".m2-for-repo"
+          rm -rf ".m2-for-src" && mkdir ".m2-for-src"
 '''
+        sh """#!/bin/bash -e
+          ${env.BASEDIR}/dev-support/hbase_nightly_source-artifact.sh \
+              --intermediate-file-dir output-srctarball \
+              --unpack-temp-dir unpacked_src_tarball \
+              --maven-m2-initial .m2-for-repo \
+              --maven-m2-src-build .m2-for-src \
+              ${env.BASEDIR}
+"""
+      }
+      post {
+        always {
+          archive 'output-srctarball/*'
+        }
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/27d8a6d1/dev-support/hbase_nightly_source-artifact.sh
----------------------------------------------------------------------
diff --git a/dev-support/hbase_nightly_source-artifact.sh b/dev-support/hbase_nightly_source-artifact.sh
new file mode 100755
index 0000000..4e4c6a2
--- /dev/null
+++ b/dev-support/hbase_nightly_source-artifact.sh
@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+function usage {
+  echo "Usage: ${0} [options] /path/to/component/checkout"
+  echo ""
+  echo "    --intermediate-file-dir /path/to/use  Path for writing listings and diffs. must
exist."
+  echo "                                          defaults to making a directory in /tmp."
+  echo "    --unpack-temp-dir /path/to/use        Path for unpacking tarball. default to"
+  echo "                                          'unpacked_src_tarball' in intermediate
directory."
+  echo "    --maven-m2-initial /path/to/use       Path for maven artifacts while building
in"
+  echo "                                          component-dir."
+  echo "    --maven-m2-src-build /path/to/use     Path for maven artifacts while building
from the"
+  echo "                                          unpacked source tarball."
+  exit 1
+}
+# if no args specified, show usage
+if [ $# -lt 1 ]; then
+  usage
+fi
+
+# Get arguments
+declare component_dir
+declare unpack_dir
+declare m2_initial
+declare m2_tarbuild
+declare working_dir
+while [ $# -gt 0 ]
+do
+  case "$1" in
+    --unpack-temp-dir) shift; unpack_dir=$1; shift;;
+    --maven-m2-initial) shift; m2_initial=$1; shift;;
+    --maven-m2-src-build) shift; m2_tarbuild=$1; shift;;
+    --intermediate-file-dir) shift; working_dir=$1; shift;;
+    --) shift; break;;
+    -*) usage ;;
+    *)  break;;  # terminate while loop
+  esac
+done
+
+# should still have where component checkout is.
+if [ $# -lt 1 ]; then
+  usage
+fi
+component_dir="$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
+
+if [ -z "${working_dir}" ]; then
+  working_dir=/tmp
+  while [[ -e ${working_dir} ]]; do
+    working_dir=/tmp/hbase-srctarball-test-${RANDOM}.${RANDOM}
+  done
+  mkdir "${working_dir}"
+else
+  # absolutes please
+  working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")"
+  if [ ! -d "${working_dir}" ]; then
+    echo "passed working directory '${working_dir}' must already exist."
+    exit 1
+  fi
+fi
+
+echo "You'll find logs and temp files in ${working_dir}"
+
+if [ -z "${unpack_dir}" ]; then
+  unpack_dir="${working_dir}/unpacked_src_tarball"
+  mkdir "${unpack_dir}"
+else
+  # absolutes please
+  unpack_dir="$(cd "$(dirname "${unpack_dir}")"; pwd)/$(basename "${unpack_dir}")"
+  if [ ! -d "${unpack_dir}" ]; then
+    echo "passed directory for unpacking the source tarball '${unpack_dir}' must already
exist."
+    exit 1
+  fi
+  rm -rf "${unpack_dir:?}/*"
+fi
+
+if [ -z "${m2_initial}" ]; then
+  m2_initial="${working_dir}/.m2-initial"
+  mkdir "${m2_initial}"
+else
+  # absolutes please
+  m2_initial="$(cd "$(dirname "${m2_initial}")"; pwd)/$(basename "${m2_initial}")"
+  if [ ! -d "${m2_initial}" ]; then
+    echo "passed directory for storing the initial build's maven repo  '${m2_initial}' "
\
+        "must already exist."
+    exit 1
+  fi
+fi
+
+if [ -z "${m2_tarbuild}" ]; then
+  m2_tarbuild="${working_dir}/.m2-tarbuild"
+  mkdir "${m2_tarbuild}"
+else
+  # absolutes please
+  m2_tarbuild="$(cd "$(dirname "${m2_tarbuild}")"; pwd)/$(basename "${m2_tarbuild}")"
+  if [ ! -d "${m2_tarbuild}" ]; then
+    echo "passed directory for storing the build from src tarball's maven repo  '${m2_tarbuild}'
" \
+        "must already exist."
+    exit 1
+  fi
+fi
+
+# This is meant to mimic what a release manager will do to create RCs.
+# See http://hbase.apache.org/book.html#maven.release
+
+echo "Maven details, in case our JDK doesn't match expectations:"
+mvn --version --offline | tee "${working_dir}/maven_version"
+
+echo "Do a clean building of the source artifact using code in ${component_dir}"
+cd "${component_dir}"
+echo "Clean..."
+mvn --batch-mode -DskipTests clean >"${working_dir}/component_clean_before.log" 2>&1
+echo "Step 3 Build the source tarball"
+mvn -Prelease --batch-mode -Dmaven.repo.local="${m2_initial}" install -DskipTests assembly:single
\
+    -Dassembly.file=hbase-assembly/src/main/assembly/src.xml \
+    >"${working_dir}/component_build_src_tarball.log" 2>&1
+
+cd "${unpack_dir}"
+echo "Unpack the source tarball"
+tar --strip-components=1 -xzf "${component_dir}"/hbase-assembly/target/hbase-*-src.tar.gz
\
+    >"${working_dir}/srctarball_unpack.log" 2>&1
+echo "Building from source artifact."
+mvn -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" clean install
\
+    assembly:single >"${working_dir}/srctarball_install.log" 2>&1
+
+echo "Clean up after checking ability to build."
+mvn -DskipTests --batch-mode clean >"${working_dir}/srctarball_clean.log" 2>&1
+
+
+cd "${component_dir}"
+echo "Clean up the source checkout"
+mvn --batch-mode -DskipTests clean >"${working_dir}/component_clean_after.log" 2>&1
+echo "Diff against source tree"
+diff --binary --recursive . "${unpack_dir}" >"${working_dir}/diff_output" || true
+
+cd "${working_dir}"
+# expectation check largely based on HBASE-14952
+echo "Checking against things we don't expect to include in the source tarball (git related,
hbase-native-client, etc.)"
+cat >known_excluded <<END
+Only in .: .git
+Only in .: .gitattributes
+Only in .: .gitignore
+Only in .: hbase-native-client
+END
+if ! diff known_excluded diff_output >"${working_dir}/unexpected.diff" ; then
+  echo "Any output here are unexpected differences between the source artifact we'd make
for an RC and the current branch."
+  echo "The expected differences are on the < side and the current differences are on
the > side."
+  echo "In a given set of differences, '.' refers to the branch in the repo and 'unpacked_src_tarball'
refers to what we pulled out of the tarball."
+  diff known_excluded diff_output
+else
+  echo "Everything looks as expected."
+fi


Mime
View raw message