hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sp...@apache.org
Subject hive git commit: HIVE-14734: Detect ptest profile and submit to ptest-server from jenkins-execute-build.sh (Sergio Pena, reviewed by Siddarth Seth)
Date Sat, 17 Sep 2016 21:16:59 GMT
Repository: hive
Updated Branches:
  refs/heads/master 05e251036 -> c90eed20d


HIVE-14734: Detect ptest profile and submit to ptest-server from jenkins-execute-build.sh
(Sergio Pena, reviewed by Siddarth Seth)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c90eed20
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c90eed20
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c90eed20

Branch: refs/heads/master
Commit: c90eed20d84cf1c86dd6dd6a5dc3f56e0e48d4ff
Parents: 05e2510
Author: Sergio Pena <sergio.pena@cloudera.com>
Authored: Fri Sep 16 23:18:31 2016 -0500
Committer: Sergio Pena <sergio.pena@cloudera.com>
Committed: Sat Sep 17 16:16:39 2016 -0500

----------------------------------------------------------------------
 dev-support/jenkins-common.sh        |  63 ++++++++++++
 dev-support/jenkins-execute-build.sh | 157 ++++++++++++++++++++----------
 2 files changed, 166 insertions(+), 54 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/c90eed20/dev-support/jenkins-common.sh
----------------------------------------------------------------------
diff --git a/dev-support/jenkins-common.sh b/dev-support/jenkins-common.sh
index 214ca1c..0467d11 100644
--- a/dev-support/jenkins-common.sh
+++ b/dev-support/jenkins-common.sh
@@ -13,6 +13,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+JIRA_ROOT_URL="https://issues.apache.org"
+
 fail() {
   echo "$@" 1>&2
   exit 1
@@ -48,6 +51,66 @@ get_branch_name() {
   return 0
 }
 
+# Gets the attachment identifier of a JIRA attachment file
+get_attachment_id() {
+  local jira_attachment_url="$1"
+
+  basename $(dirname $jira_attachment_url)
+}
+
+# Fetchs JIRA information, and save it in a file specified as a parameter
+initialize_jira_info() {
+  local jira_issue="$1" 
+  local output_file="$2"
+
+  curl -s -S --location --retry 3 "${JIRA_ROOT_URL}/jira/browse/${jira_issue}" > $output_file
+}
+
+# Checks if a JIRA is in 'Patch Available' status
+is_patch_available() {
+  grep -q "Patch Available" $1
+}
+
+# Checks if a JIRA has 'NO PRECOMMIT TESTS' enabled
+is_check_no_precommit_tests_set() {
+  grep -q "NO PRECOMMIT TESTS" $1
+}
+
+# Gets the URL for the JIRA patch attached
+get_jira_patch_url() {
+  grep -o '"/jira/secure/attachment/[0-9]*/[^"]*' $1 | grep -v -e 'htm[l]*$' | sort | tail
-1 | \
+    grep -o '/jira/secure/attachment/[0-9]*/[^"]*'
+}
+
+# Checks if the patch was already tested
+is_patch_already_tested() {
+  local attachment_id="$1"
+  local test_handle="$2"
+  local jira_info_file="$3"
+
+  grep -q "ATTACHMENT ID: $attachment_id $test_handle" $jira_info_file
+}
+
+# Gets the branch profile attached to the patch
+get_branch_profile() {
+  local jira_patch_url="$1"
+  local branch_name=`get_branch_name $(basename $jira_patch_url)`
+
+  if [ -n "$branch_name" ]; then
+    shopt -s nocasematch
+    if [[ $branch_name =~ (mr1|mr2)$ ]]; then
+      echo $branch_name
+    else
+      echo ${branch_name}-mr2
+    fi
+  fi
+}
+
+# Checks if a JIRA patch has 'CLEAR LIBRARY CACHE' enabled
+is_clear_cache_set() {
+  grep -q "CLEAR LIBRARY CACHE" $1
+}
+
 # Parses the JIRA/patch to find relavent information.
 # Exports two variables of import:
 # * BUILD_PROFILE - the profile which the ptest server understands

http://git-wip-us.apache.org/repos/asf/hive/blob/c90eed20/dev-support/jenkins-execute-build.sh
----------------------------------------------------------------------
diff --git a/dev-support/jenkins-execute-build.sh b/dev-support/jenkins-execute-build.sh
index 3b41b0b..a9935e1 100644
--- a/dev-support/jenkins-execute-build.sh
+++ b/dev-support/jenkins-execute-build.sh
@@ -13,67 +13,116 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 set -e
-. jenkins-common.sh
-test -n "$BRANCH" || fail "BRANCH must be specified"
-test -n "$API_ENDPOINT" || fail "API_ENDPOINT must be specified"
-test -n "$LOG_ENDPOINT" || fail "LOG_ENDPOINT must be specified"
-test -n "$API_PASSWORD" || fail "API_PASSWORD must be specified"
-if [[ -n "$ISSUE_NUM" ]]
-then
-  export JIRA_NAME="HIVE-${ISSUE_NUM}"
-fi
-export ROOT=$PWD
-export JIRA_ROOT_URL="https://issues.apache.org"
-export BUILD_TAG="${BUILD_TAG##jenkins-}"
-if [[ -n "$JIRA_NAME" ]]
-then
-  echo $JIRA_NAME
-fi
 set -x
-env
 
-if [[ -n "$JIRA_NAME" ]]
-then
-  process_jira
-fi
+. jenkins-common.sh
 
-profile=$BUILD_PROFILE
-if [[ -z "$profile" ]]
-then
-  profile=$DEFAULT_BUILD_PROFILE
-fi
-if [[ -z "$profile" ]]
-then
-  fail "Could not find build profile"
-fi
+# Build the ptest framework locally
+build_ptest_client() {
+	[ -z "$PTEST_BUILD_DIR" ] && echoerr "Error: Cannot build ptest: PTEST_BUILD_DIR
is not defined."
+
+	test -d $PTEST_BUILD_DIR || mkdir -p $PTEST_BUILD_DIR
+	cd $PTEST_BUILD_DIR &&	rm -rf hive
+
+	git clone --depth 1 https://github.com/apache/hive.git
+	cd hive/testutils/ptest2
+	mvn clean package -DskipTests -Drat.numUnapprovedLicenses=1000 -Dmaven.repo.local=$MVN_REPO_LOCAL
+}
+
+# Call the ptest server to run all tests
+call_ptest_server() {
+	[ -z "$PTEST_BUILD_DIR" ] && echoerr "Error: Cannot build ptest: PTEST_BUILD_DIR
is not defined."
+
+	read -s  -p "JIRA password: " JIRA_PASSWORD
+
+	build_ptest_client || return 1
+
+	local PTEST_CLASSPATH="$PTEST_BUILD_DIR/hive/testutils/ptest2/target/hive-ptest-1.0-classes.jar:$PTEST_BUILD_DIR/hive/testutils/ptest2/target/lib/*"
+
+	java -cp "$PTEST_CLASSPATH" org.apache.hive.ptest.api.client.PTestClient --command testStart
\
+		--outputDir "$PTEST_BUILD_DIR/hive/testutils/ptest2/target" --password "$JIRA_PASSWORD"
"$@"
+}
+
+# Unpack all test results
+unpack_test_results() {
+	[ -z "$PTEST_BUILD_DIR" ] && echoerr "Error: Cannot build ptest: PTEST_BUILD_DIR
is not defined."
+
+	cd "$PTEST_BUILD_DIR/hive/testutils/ptest2/target"
+	if [[ -f test-results.tar.gz ]]; then
+	  rm -rf $PTEST_BUILD_DIR/test-results/
+	  tar zxf test-results.tar.gz -C $PTEST_BUILD_DIR
+	fi
+}
+
+# Check required arguments
+test -n "$TEST_HANDLE" || fail "TEST_HANDLE must be specified and cannot be empty."
+test -n "$PTEST_API_ENDPOINT" || fail "PTEST_API_ENDPOINT must be specified and cannot be
empty."
+test -n "$PTEST_LOG_ENDPOINT" || fail "PTEST_LOG_ENDPOINT must be specified and cannot be
empty."
 
-test -d hive/build/ || mkdir -p hive/build/
-cd hive/build/
-rm -rf hive
-git clone --depth 1 https://github.com/apache/hive.git
-cd hive/testutils/ptest2
+# WORKSPACE is an environment variable created by Jenkins, and it is the directory where
the build is executed.
+# If not set, then default to $HOME
+MVN_REPO_LOCAL=${WORKSPACE:-$HOME}/.m2/repository
 
-mvn clean package -DskipTests -Drat.numUnapprovedLicenses=1000 -Dmaven.repo.local=$WORKSPACE/.m2
-set +e
+# Directory where to build the ptest framework
+PTEST_BUILD_DIR="$PWD/hive/build"
+
+# Default profile in case a patch does not have a profile assigned, or 
+# if this script is executed without JIRA_ISSUE defined
+DEFAULT_BUILD_PROFILE="master-mr2"
+
+# Optional parameters that might be passed to the ptest client command
 optionalArgs=()
-if [[ -n "$JIRA_NAME" ]]
-then
-  optionalArgs=(--patch "${JIRA_ROOT_URL}${PATCH_URL}" --jira "$JIRA_NAME")
+
+if [ -n "$JIRA_ISSUE" ]; then
+	JIRA_INFO_FILE=`mktemp`
+	trap "rm -f $JIRA_INFO_FILE" EXIT
+
+	initialize_jira_info $JIRA_ISSUE $JIRA_INFO_FILE || fail "Error: Cannot initialize JIRA
information."
+
+	if ! is_patch_available $JIRA_INFO_FILE; then
+		fail "$JIRA_ISSUE is not 'Patch Available'."
+	fi
+
+	if is_check_no_precommit_tests_set $JIRA_INFO_FILE; then
+		fail "$JIRA_ISSUE has tag NO PRECOMMIT TESTS"
+	fi
+
+	JIRA_PATCH_URL=`get_jira_patch_url $JIRA_INFO_FILE`
+	if [ -z "$JIRA_PATCH_URL" ]; then
+		fail "Unable to find attachment for $JIRA_ISSUE"
+	fi
+
+	attachment_id=`get_attachment_id $JIRA_PATCH_URL`
+	if is_patch_already_tested "$attachment_id" "$TEST_HANDLE" "$JIRA_INFO_FILE"; then
+		fail "attachment $attachment_id is already tested for $JIRA_ISSUE"
+	fi
+
+	BUILD_PROFILE=`get_branch_profile $JIRA_PATCH_URL $JIRA_INFO_FILE`
+	if [ -z "$BUILD_PROFILE" ]; then
+		BUILD_PROFILE="$DEFAULT_BUILD_PROFILE"
+	fi
+
+	if is_clear_cache_set $JIRA_INFO_FILE; then
+		optionalArgs+=(--clearLibraryCache)
+	fi
+
+	optionalArgs+=(--patch "${JIRA_ROOT_URL}${JIRA_PATCH_URL}" --jira "$JIRA_ISSUE")
+
+	echo "ISSUE: $JIRA_ISSUE PROFILE: $BUILD_PROFILE"
+else
+	# If not JIRA is specified, then use a default profile
+	BUILD_PROFILE="$DEFAULT_BUILD_PROFILE"
+
+	echo "ISSUE: unspecified PROFILE: $BUILD_PROFILE"
 fi
-java -cp "target/hive-ptest-1.0-classes.jar:target/lib/*" org.apache.hive.ptest.api.client.PTestClient
--endpoint "$API_ENDPOINT" \
-  --logsEndpoint "$LOG_ENDPOINT" \
-  --command testStart \
-  --profile $profile \
-  --password $API_PASSWORD \
-  --outputDir target/ \
-  --testHandle "$BUILD_TAG" \
-  ${optionalArgs[@]} ${BUILD_OPTS} "$@"
+
+call_ptest_server --testHandle "$TEST_HANDLE" --endpoint "$PTEST_API_ENDPOINT" --logsEndpoint
"$PTEST_LOG_ENDPOINT" \
+	--profile "$BUILD_PROFILE" ${optionalArgs[@]} "$@"
+
 ret=$?
-cd target/
-if [[ -f test-results.tar.gz ]]
-then
-  rm -rf $ROOT/hive/build/test-results/
-  tar zxf test-results.tar.gz -C $ROOT/hive/build/
-fi
+
+unpack_test_results
+
 exit $ret


Mime
View raw message