hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject hbase git commit: Adds exclusion of flaky tests from master nightly runs Fixes findbugs in nightly runs Adds testing for hadoop3 Renames OUTPUTDIR to OUTPUT_DIR Renames OUTPUT_RELATIVE to OUTPUT_DIR_RELATIVE
Date Wed, 20 Dec 2017 11:45:36 GMT
Repository: hbase
Updated Branches:
  refs/heads/HBASE-19491 [created] 44f7ae5f2


Adds exclusion of flaky tests from master nightly runs
Fixes findbugs in nightly runs
Adds testing for hadoop3
Renames OUTPUTDIR to OUTPUT_DIR
Renames OUTPUT_RELATIVE to OUTPUT_DIR_RELATIVE


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/44f7ae5f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/44f7ae5f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/44f7ae5f

Branch: refs/heads/HBASE-19491
Commit: 44f7ae5f215d3bcb56298d822fbb2f817d7f3d94
Parents: 03e79b7
Author: Apekshit Sharma <appy@apache.org>
Authored: Tue Dec 12 14:56:20 2017 -0800
Committer: Apekshit Sharma <appy@apache.org>
Committed: Wed Dec 20 03:45:19 2017 -0800

----------------------------------------------------------------------
 dev-support/Jenkinsfile            | 168 +++++++++++++++++++++-----------
 dev-support/hbase-personality.sh   | 125 ++++++++++++++++--------
 dev-support/hbase_nightly_yetus.sh |  30 ++++--
 3 files changed, 221 insertions(+), 102 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/44f7ae5f/dev-support/Jenkinsfile
----------------------------------------------------------------------
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 83c0afb..1c662d0 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -34,15 +34,9 @@ pipeline {
     // where we check out to across stages
     BASEDIR = "${env.WORKSPACE}/component"
     YETUS_RELEASE = '0.6.0'
-    // where we'll write everything from different steps.
-    OUTPUT_RELATIVE_GENERAL = 'output-general'
-    OUTPUTDIR_GENERAL = "${env.WORKSPACE}/output-general"
-    OUTPUT_RELATIVE_JDK7 = 'output-jdk7'
-    OUTPUTDIR_JDK7 = "${env.WORKSPACE}/output-jdk7"
-    OUTPUT_RELATIVE_JDK8 = 'output-jdk8'
-    OUTPUTDIR_JDK8 = "${env.WORKSPACE}/output-jdk8"
     PROJECT = 'hbase'
-    PROJET_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
+    // TODO: revert back after testing
+    PROJET_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/HBASE-19491/dev-support/hbase-personality.sh'
     // This section of the docs tells folks not to use the javadoc tag. older branches have
our old version of the check for said tag.
     AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
     WHITESPACE_IGNORE_LIST = '.*/generated/.*'
@@ -115,6 +109,12 @@ curl -L  -o personality.sh "${env.PROJET_PERSONALITY}"
       }
     }
     stage ('yetus general check') {
+      // TODO: revert back after testing hadoop3
+      when {
+        not {
+          branch 'HBASE-19491'
+        }
+      }
       environment {
         // TODO does hadoopcheck need to be jdk specific?
         // Should be things that work with multijdk
@@ -122,30 +122,30 @@ curl -L  -o personality.sh "${env.PROJET_PERSONALITY}"
         // on branches that don't support jdk7, this will already be JAVA_HOME, so we'll
end up not
         // doing multijdk there.
         MULTIJDK = '/usr/lib/jvm/java-8-openjdk-amd64'
-        OUTPUT_RELATIVE = "${env.OUTPUT_RELATIVE_GENERAL}"
-        OUTPUTDIR = "${env.OUTPUTDIR_GENERAL}"
+        OUTPUT_DIR_RELATIVE = "output-general"
+        OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}"
       }
       steps {
         unstash 'yetus'
         sh '''#!/usr/bin/env bash
-          rm -rf "${OUTPUTDIR}" && mkdir "${OUTPUTDIR}"
-          rm -rf "${OUTPUTDIR}/machine" && mkdir "${OUTPUTDIR}/machine"
-          "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_RELATIVE}/machine"
+          rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
+          rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
+          "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
 '''
         // TODO should this be a download from master, similar to how the personality is?
         sh "${env.BASEDIR}/dev-support/hbase_nightly_yetus.sh"
       }
       post {
         always {
-          // env variables don't work in archive? or this has to be relative to WORKSPACE.
:(
-          archive 'output-general/*'
-          archive 'output-general/**/*'
+          // Has to be relative to WORKSPACE.
+          archive "${env.OUTPUT_DIR_RELATIVE}/*"
+          archive "${env.OUTPUT_DIR_RELATIVE}/**/*"
           publishHTML target: [
             allowMissing: true,
             keepAll: true,
             alwaysLinkToLastBuild: true,
-            // has to be relative to WORKSPACE :(
-            reportDir: 'output-general',
+            // Has to be relative to WORKSPACE
+            reportDir: "${env.OUTPUT_DIR_RELATIVE}",
             reportFiles: 'console-report.html',
             reportName: 'General Nightly Build Report'
           ]
@@ -158,16 +158,16 @@ curl -L  -o personality.sh "${env.PROJET_PERSONALITY}"
       }
       environment {
         TESTS = 'mvninstall,compile,javac,unit,htmlout'
-        OUTPUT_RELATIVE = "${env.OUTPUT_RELATIVE_JDK7}"
-        OUTPUTDIR = "${env.OUTPUTDIR_JDK7}"
+        OUTPUT_DIR_RELATIVE = "output-jdk7"
+        OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}"
         // On branches where we do jdk7 checks, jdk7 will be JAVA_HOME already.
       }
       steps {
         unstash 'yetus'
         sh '''#!/usr/bin/env bash
-          rm -rf "${OUTPUTDIR}" && mkdir "${OUTPUTDIR}"
-          rm -rf "${OUTPUTDIR}/machine" && mkdir "${OUTPUTDIR}/machine"
-          "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_RELATIVE}/machine"
+          rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
+          rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
+          "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
 '''
         sh """#!/usr/bin/env bash
           # for branch-1.1 we don't do jdk8 findbugs, so do it here
@@ -179,14 +179,14 @@ curl -L  -o personality.sh "${env.PROJET_PERSONALITY}"
       }
       post {
         always {
-          junit testResults: 'output-jdk7/**/target/**/TEST-*.xml', allowEmptyResults: true
+          junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults:
true
           // zip surefire reports.
           sh '''#!/bin/bash -e
-            if [ -d "${OUTPUTDIR}/archiver" ]; then
-              count=$(find "${OUTPUTDIR}/archiver" -type f | wc -l)
+            if [ -d "${OUTPUT_DIR}/archiver" ]; then
+              count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
               if [[ 0 -ne ${count} ]]; then
                 echo "zipping ${count} archived files"
-                zip -q -m -r "${OUTPUTDIR}/test_logs.zip" "${OUTPUTDIR}/archiver"
+                zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
               else
                 echo "No archived files, skipping compressing."
               fi
@@ -194,22 +194,78 @@ curl -L  -o personality.sh "${env.PROJET_PERSONALITY}"
               echo "No archiver directory, skipping compressing."
             fi
 '''
-          // env variables don't work in archive? or this has to be relative to WORKSPACE.
:(
-          archive 'output-jdk7/*'
-          archive 'output-jdk7/**/*'
+          // Has to be relative to WORKSPACE.
+          archive "${env.OUTPUT_DIR_RELATIVE}/*"
+          archive "${env.OUTPUT_DIR_RELATIVE}/**/*"
           publishHTML target: [
-            allowMissing: true,
-            keepAll: true,
+            allowMissing         : true,
+            keepAll              : true,
             alwaysLinkToLastBuild: true,
-            // has to be relative to WORKSPACE :(
-            reportDir: 'output-jdk7',
-            reportFiles: 'console-report.html',
-            reportName: 'JDK7 Nightly Build Report'
+            // Has to be relative to WORKSPACE.
+            reportDir            : "${env.OUTPUT_DIR_RELATIVE}",
+            reportFiles          : 'console-report.html',
+            reportName           : 'JDK7 Nightly Build Report'
+          ]
+        }
+      }
+    }
+    stage ('yetus jdk8 hadoop2 checks') {
+      when {
+        not {
+          // TODO: revert back after testing hadoop3
+          branch 'HBASE-19491'
+        }
+      }
+      environment {
+        TESTS = 'mvninstall,compile,javac,unit,htmlout'
+        OUTPUT_DIR_RELATIVE = "output-jdk8-hadoop2"
+        OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}"
+        // This isn't strictly needed on branches that only support jdk8, but doesn't hurt
+        // and is needed on branches that do both jdk7 and jdk8
+        SET_JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
+      }
+      steps {
+        unstash 'yetus'
+        sh '''#!/usr/bin/env bash
+          rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
+          rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
+          "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+'''
+        sh "${env.BASEDIR}/dev-support/hbase_nightly_yetus.sh"
+      }
+      post {
+        always {
+          junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults:
true
+          // zip surefire reports.
+          sh '''#!/bin/bash -e
+            if [ -d "${OUTPUT_DIR}/archiver" ]; then
+              count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
+              if [[ 0 -ne ${count} ]]; then
+                echo "zipping ${count} archived files"
+                zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
+              else
+                echo "No archived files, skipping compressing."
+              fi
+            else
+              echo "No archiver directory, skipping compressing."
+            fi
+'''
+          // Has to be relative to WORKSPACE.
+          archive "${env.OUTPUT_DIR_RELATIVE}/*"
+          archive "${env.OUTPUT_DIR_RELATIVE}/**/*"
+          publishHTML target: [
+            allowMissing         : true,
+            keepAll              : true,
+            alwaysLinkToLastBuild: true,
+            // Has to be relative to WORKSPACE.
+            reportDir            : "${env.OUTPUT_DIR_RELATIVE}",
+            reportFiles          : 'console-report.html',
+            reportName           : 'JDK8 Nightly Build Report (Hadoop2)'
           ]
         }
       }
     }
-    stage ('yetus jdk8 checks') {
+    stage ('yetus jdk8 hadoop3 checks') {
       when {
         not {
           branch 'branch-1.1*'
@@ -217,31 +273,33 @@ curl -L  -o personality.sh "${env.PROJET_PERSONALITY}"
       }
       environment {
         TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout'
-        OUTPUT_RELATIVE = "${env.OUTPUT_RELATIVE_JDK8}"
-        OUTPUTDIR = "${env.OUTPUTDIR_JDK8}"
+        OUTPUT_DIR_RELATIVE = "output-jdk8-hadoop3"
+        OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}"
         // This isn't strictly needed on branches that only support jdk8, but doesn't hurt
         // and is needed on branches that do both jdk7 and jdk8
         SET_JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
+        HADOOP_PROFILE = '3.0'
       }
       steps {
         unstash 'yetus'
         sh '''#!/usr/bin/env bash
-          rm -rf "${OUTPUTDIR}" && mkdir "${OUTPUTDIR}"
-          rm -rf "${OUTPUTDIR}/machine" && mkdir "${OUTPUTDIR}/machine"
-          "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_RELATIVE}/machine"
+          rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
+          rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
+          "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
 '''
         sh "${env.BASEDIR}/dev-support/hbase_nightly_yetus.sh"
       }
       post {
         always {
-          junit testResults: 'output-jdk8/**/target/**/TEST-*.xml', allowEmptyResults: true
+
+          // junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults:
true
           // zip surefire reports.
           sh '''#!/bin/bash -e
-            if [ -d "${OUTPUTDIR}/archiver" ]; then
-              count=$(find "${OUTPUTDIR}/archiver" -type f | wc -l)
+            if [ -d "${OUTPUT_DIR}/archiver" ]; then
+              count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
               if [[ 0 -ne ${count} ]]; then
                 echo "zipping ${count} archived files"
-                zip -q -m -r "${OUTPUTDIR}/test_logs.zip" "${OUTPUTDIR}/archiver"
+                zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
               else
                 echo "No archived files, skipping compressing."
               fi
@@ -249,17 +307,17 @@ curl -L  -o personality.sh "${env.PROJET_PERSONALITY}"
               echo "No archiver directory, skipping compressing."
             fi
 '''
-          // env variables don't work in archive? or this has to be relative to WORKSPACE.
:(
-          archive 'output-jdk8/*'
-          archive 'output-jdk8/**/*'
+          // Has to be relative to WORKSPACE.
+          archive "${env.OUTPUT_DIR_RELATIVE}/*"
+          archive "${env.OUTPUT_DIR_RELATIVE}/**/*"
           publishHTML target: [
-            allowMissing: true,
-            keepAll: true,
+            allowMissing         : true,
+            keepAll              : true,
             alwaysLinkToLastBuild: true,
-            // has to be relative to WORKSPACE :(
-            reportDir: 'output-jdk8',
-            reportFiles: 'console-report.html',
-            reportName: 'JDK8 Nightly Build Report'
+            // Has to be relative to WORKSPACE
+            reportDir            : "${env.OUTPUT_DIR_RELATIVE}",
+            reportFiles          : 'console-report.html',
+            reportName           : 'JDK8 Nightly Build Report (Hadoop3)'
           ]
         }
       }

http://git-wip-us.apache.org/repos/asf/hbase/blob/44f7ae5f/dev-support/hbase-personality.sh
----------------------------------------------------------------------
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index f33faea..9028277 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -64,7 +64,28 @@ function personality_globals
 
   # Override the maven options
   MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}"
+}
 
+## @description  Parse extra arguments required by personalities, if any.
+## @audience     private
+## @stability    evolving
+function personality_parse_args
+{
+  declare i
+
+  for i in "$@"; do
+    case ${i} in
+      --exclude-tests-url=*)
+        EXCLUDE_TESTS_URL=${i#*=}
+      ;;
+      --include-tests-url=*)
+        INCLUDE_TESTS_URL=${i#*=}
+      ;;
+      --hadoop-profile=*)
+        HADOOP_PROFILE=${i#*=}
+      ;;
+    esac
+  done
 }
 
 ## @description  Queue up modules for this personality
@@ -77,19 +98,39 @@ function personality_modules
   local repostatus=$1
   local testtype=$2
   local extra=""
+  local MODULES=(${CHANGED_MODULES[@]})
 
-  yetus_debug "Personality: ${repostatus} ${testtype}"
+  yetus_info "Personality: ${repostatus} ${testtype}"
 
   clear_personality_queue
 
   extra="-DHBasePatchProcess"
 
-  if [[ ${testtype} == mvninstall ]] || [[ "${BUILDMODE}" == full ]]; then
+  if [[ -n "${HADOOP_PROFILE}" ]]; then
+    extra="${extra} -Dhadoop.profile=${HADOOP_PROFILE}"
+  fi
+
+  # BUILDMODE value is 'full' when there is no patch and we are running checks on full source
code.
+  # In this case, do full compiles, tests, findbugs etc instead of per module.
+  # Used in nightly runs.
+  if [[ "${BUILDMODE}" == "full" ]]; then
+    MODULES=(.)
+  elif [[ "${testtype}" == unit || "${testtype}" == compile ]] && [[ "${MODULES[*]}"
=~ \. ]]; then
+    # For unit and compile testtypes, if root is included, no need to run for individual
modules.
+    # HBASE-18505
+    MODULES=(.)
+  fi
+
+  if [[ ${testtype} == mvninstall ]]; then
     personality_enqueue_module . ${extra}
     return
   fi
 
-  if [[ ${testtype} = findbugs ]]; then
+  if [[ ${testtype} == findbugs ]]; then
+    # Run findbugs on each module individually to diff pre-patch and post-patch results and
+    # report new warnings for changed modules only.
+    # For some reason, findbugs on root is not working, but running on individual modules
is
+    # working. For time being, let it run on original list of CHANGED_MODULES. HBASE-19491
     for module in "${CHANGED_MODULES[@]}"; do
       # skip findbugs on hbase-shell and hbase-it. hbase-it has nothing
       # in src/main/java where findbugs goes to look
@@ -108,40 +149,9 @@ function personality_modules
   # If EXCLUDE_TESTS_URL/INCLUDE_TESTS_URL is set, fetches the url
   # and sets -Dtest.exclude.pattern/-Dtest to exclude/include the
   # tests respectively.
-  if [[ ${testtype} = unit ]]; then
-    # if the modules include root, skip all the submodules HBASE-18505
-    if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
-      CHANGED_MODULES=(.)
-    fi
-
-    extra="${extra} -PrunAllTests"
-    yetus_debug "EXCLUDE_TESTS_URL = ${EXCLUDE_TESTS_URL}"
-    yetus_debug "INCLUDE_TESTS_URL = ${INCLUDE_TESTS_URL}"
-    if [[ -n "$EXCLUDE_TESTS_URL" ]]; then
-        if wget "$EXCLUDE_TESTS_URL" -O "excludes"; then
-          excludes=$(cat excludes)
-          yetus_debug "excludes=${excludes}"
-          if [[ -n "${excludes}" ]]; then
-            extra="${extra} -Dtest.exclude.pattern=${excludes}"
-          fi
-          rm excludes
-        else
-          echo "Wget error $? in fetching excludes file from url" \
-               "${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
-        fi
-    elif [[ -n "$INCLUDE_TESTS_URL" ]]; then
-        if wget "$INCLUDE_TESTS_URL" -O "includes"; then
-          includes=$(cat includes)
-          yetus_debug "includes=${includes}"
-          if [[ -n "${includes}" ]]; then
-            extra="${extra} -Dtest=${includes}"
-          fi
-          rm includes
-        else
-          echo "Wget error $? in fetching includes file from url" \
-               "${INCLUDE_TESTS_URL}. Ignoring and proceeding."
-        fi
-    fi
+  if [[ ${testtype} == unit ]]; then
+    get_include_exclude_tests_arg tests_arg
+    extra="${extra} -PrunAllTests ${tests_arg}"
 
     # Inject the jenkins build-id for our surefire invocations
     # Used by zombie detection stuff, even though we're not including that yet.
@@ -150,12 +160,50 @@ function personality_modules
     fi
   fi
 
-  for module in "${CHANGED_MODULES[@]}"; do
+  for module in "${MODULES[@]}"; do
     # shellcheck disable=SC2086
     personality_enqueue_module ${module} ${extra}
   done
 }
 
+## @description  Uses relevant include/exclude env variable to fetch list of included/excluded
+#                tests and sets given variable to arguments to be passes to maven command.
+## @audience     private
+## @stability    evolving
+## @param        name of variable to set with maven arguments
+function get_include_exclude_tests_arg
+{
+  local  __resultvar=$1
+  yetus_info "EXCLUDE_TESTS_URL=${EXCLUDE_TESTS_URL}"
+  yetus_info "INCLUDE_TESTS_URL=${INCLUDE_TESTS_URL}"
+  eval $__resultvar=""
+  if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
+      if wget "${EXCLUDE_TESTS_URL}" -O "excludes"; then
+        excludes=$(cat excludes)
+        yetus_debug "excludes=${excludes}"
+        if [[ -n "${excludes}" ]]; then
+          eval $__resultvar="-Dtest.exclude.pattern=${excludes}"
+        fi
+        rm excludes
+      else
+        yetus_error "Wget error $? in fetching excludes file from url" \
+             "${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
+      fi
+  elif [[ -n "$INCLUDE_TESTS_URL" ]]; then
+      if wget "$INCLUDE_TESTS_URL" -O "includes"; then
+        includes=$(cat includes)
+        yetus_debug "includes=${includes}"
+        if [[ -n "${includes}" ]]; then
+          eval $__resultvar="-Dtest=${includes}"
+        fi
+        rm includes
+      else
+        yetus_error "Wget error $? in fetching includes file from url" \
+             "${INCLUDE_TESTS_URL}. Ignoring and proceeding."
+      fi
+  fi
+}
+
 ###################################################
 # Below here are our one-off tests specific to hbase.
 # TODO break them into individual files so it's easier to maintain them?
@@ -166,7 +214,6 @@ function personality_modules
 
 add_test_type shadedjars
 
-
 function shadedjars_initialize
 {
   yetus_debug "initializing shaded client checks."

http://git-wip-us.apache.org/repos/asf/hbase/blob/44f7ae5f/dev-support/hbase_nightly_yetus.sh
----------------------------------------------------------------------
diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh
index 007d64a..e1175d2 100755
--- a/dev-support/hbase_nightly_yetus.sh
+++ b/dev-support/hbase_nightly_yetus.sh
@@ -18,8 +18,8 @@
 
 declare -i missing_env=0
 # Validate params
-for required_env in "TESTS" "TOOLS" "BASEDIR" "ARCHIVE_PATTERN_LIST" "OUTPUT_RELATIVE" \
-                    "BRANCH_SPECIFIC_DOCKERFILE" "OUTPUTDIR" "PROJECT" "AUTHOR_IGNORE_LIST"
\
+for required_env in "TESTS" "TOOLS" "BASEDIR" "ARCHIVE_PATTERN_LIST" "OUTPUT_DIR_RELATIVE"
\
+                    "BRANCH_SPECIFIC_DOCKERFILE" "OUTPUT_DIR" "PROJECT" "AUTHOR_IGNORE_LIST"
\
                     "WHITESPACE_IGNORE_LIST" "BRANCH_NAME" "TESTS_FILTER" "DEBUG" \
                     "USE_YETUS_PRERELEASE" "WORKSPACE" "YETUS_RELEASE"; do
   if [ -z "${!required_env}" ]; then
@@ -49,15 +49,16 @@ YETUS_ARGS=("--basedir=${BASEDIR}" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--archive-list=${ARCHIVE_PATTERN_LIST}" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--console-urls" "${YETUS_ARGS[@]}")
 # YETUS-532, repeat this twice in case the fix is to update args rather than docs
-YETUS_ARGS=("--build-url-patchdir=artifact/${OUTPUT_RELATIVE}" "${YETUS_ARGS[@]}")
-YETUS_ARGS=("--build-url-artifacts=artifact/${OUTPUT_RELATIVE}" "${YETUS_ARGS[@]}")
+YETUS_ARGS=("--build-url-patchdir=artifact/${OUTPUT_DIR_RELATIVE}" "${YETUS_ARGS[@]}")
+YETUS_ARGS=("--build-url-artifacts=artifact/${OUTPUT_DIR_RELATIVE}" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--docker" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--dockerfile=${BRANCH_SPECIFIC_DOCKERFILE}" "${YETUS_ARGS[@]}")
+# Yetus sets BUILDMODE env variable to "full" if this arg is passed.
 YETUS_ARGS=("--empty-patch" "${YETUS_ARGS[@]}")
-YETUS_ARGS=("--html-report-file=${OUTPUTDIR}/console-report.html" "${YETUS_ARGS[@]}")
+YETUS_ARGS=("--html-report-file=${OUTPUT_DIR}/console-report.html" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--jenkins" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--mvn-custom-repos" "${YETUS_ARGS[@]}")
-YETUS_ARGS=("--patch-dir=${OUTPUTDIR}" "${YETUS_ARGS[@]}")
+YETUS_ARGS=("--patch-dir=${OUTPUT_DIR}" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--project=${PROJECT}" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--resetrepo" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--author-ignore-list=${AUTHOR_IGNORE_LIST}" "${YETUS_ARGS[@]}")
@@ -67,12 +68,25 @@ YETUS_ARGS=("--sentinel" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--branch=${BRANCH_NAME}" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--tests-filter=${TESTS_FILTER}" "${YETUS_ARGS[@]}")
 
+# Currently, flaky list is calculated only for master branch.
+if [[ -n "${EXCLUDE_TESTS_URL}" && "${BRANCH_NAME}" == "master" ]]; then
+  YETUS_ARGS=("--exclude-tests-url=${EXCLUDE_TESTS_URL}" "${YETUS_ARGS[@]}")
+fi
+if [[ -n "${INCLUDE_TESTS_URL}" && "${BRANCH_NAME}" == "master" ]]; then
+  YETUS_ARGS=("--include-tests-url=${INCLUDE_TESTS_URL}" "${YETUS_ARGS[@]}")
+fi
+
+# For testing with specific hadoop version. Activates corresponding profile in maven runs.
+if [[ -n "${HADOOP_PROFILE}" ]]; then
+  YETUS_ARGS=("--hadoop-profile=${HADOOP_PROFILE}" "${YETUS_ARGS[@]}")
+fi
+
 if [[ true == "${DEBUG}" ]]; then
   YETUS_ARGS=("--debug" "${YETUS_ARGS[@]}")
 fi
 
-rm -rf "${OUTPUTDIR}"
-mkdir -p "${OUTPUTDIR}"
+rm -rf "${OUTPUT_DIR}"
+mkdir -p "${OUTPUT_DIR}"
 if [[ true !=  "${USE_YETUS_PRERELEASE}" ]]; then
   YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs" "${YETUS_ARGS[@]}")
   TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch"


Mime
View raw message