yetus-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject yetus git commit: YETUS-378. flink, hbase, jmeter personalities custom tests need to be made qbt aware
Date Tue, 19 Apr 2016 23:19:33 GMT
Repository: yetus
Updated Branches:
  refs/heads/YETUS-156 3f782d346 -> 64377ca5a


YETUS-378. flink, hbase, jmeter personalities custom tests need to be made qbt aware

Signed-off-by: Allen Wittenauer <aw@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/64377ca5
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/64377ca5
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/64377ca5

Branch: refs/heads/YETUS-156
Commit: 64377ca5a0bacf3f6ed946c2507018dc02d0975e
Parents: 3f782d3
Author: Allen Wittenauer <aw@apache.org>
Authored: Mon Apr 18 14:18:15 2016 -0700
Committer: Allen Wittenauer <aw@apache.org>
Committed: Tue Apr 19 16:18:02 2016 -0700

----------------------------------------------------------------------
 precommit/personality/flink.sh  |  61 +++++++------
 precommit/personality/hbase.sh  | 166 +++++++++++++++++++++++++++++++----
 precommit/personality/jmeter.sh |   3 +
 3 files changed, 180 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/yetus/blob/64377ca5/precommit/personality/flink.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/flink.sh b/precommit/personality/flink.sh
index 6b4f0a8..76fa418 100755
--- a/precommit/personality/flink.sh
+++ b/precommit/personality/flink.sh
@@ -31,6 +31,10 @@ function personality_globals
 
 add_test_type flinklib
 
+## @description  flinklib file filter
+## @audience     private
+## @stability    evolving
+## @param        filename
 function flinklib_filefilter
 {
   local filename=$1
@@ -42,6 +46,9 @@ function flinklib_filefilter
   fi
 }
 
+## @description  count files in the lib dir
+## @audience     private
+## @stability    evolving
 function flinklib_count
 {
   find "${BASEDIR}" \
@@ -50,42 +57,46 @@ function flinklib_count
     | wc -l
 }
 
-function flinklib_preapply
+## @description  check fliblib
+## @audience     private
+## @stability    evolving
+## @param        repostatus
+function flinklib_rebuild
 {
+  declare repostatus=$1
+
   start_clock
   big_console_header "${PATCH_BRANCH} flink library dependencies"
 
   if ! verify_needed_test flinklib; then
-    echo "Patch does not need flinklib testing."
+    echo "${BUILDMODEMSG} does not need flinklib testing."
     return 0
   fi
 
   pushd "${BASEDIR}" >/dev/null
-  echo_and_redirect "${PATCH_DIR}/branch-flinklib-root.txt" \
+  echo_and_redirect "${PATCH_DIR}/${repostatus}-flinklib-root.txt" \
      "${MAVEN}" "${MAVEN_ARGS[@]}" package -DskipTests -Dmaven.javadoc.skip=true -Ptest-patch
   if [[ $? != 0 ]]; then
      add_vote_table -1 flinklib "Unable to determine flink libs in ${PATCH_BRANCH}."
   fi
-  FLINK_PRE_LIB_FILES=$(flinklib_count)
-  popd >/dev/null
-}
 
-function flinklib_postapply
-{
-  start_clock
-  big_console_header "Patch flink library dependencies"
-
-  if ! verify_needed_test flinklib; then
-    echo "Patch does not need flinklib testing."
+  if [[ ${repostatus} = branch ]]; then
+    FLINK_PRE_LIB_FILES=$(flinklib_count)
     return 0
+  else
+    FLINK_POST_LIB_FILES=$(flinklib_count)
   fi
-
-  pushd "${BASEDIR}" >/dev/null
-  echo_and_redirect "${PATCH_DIR}/patch-flinklib-root.txt" \
-     "${MAVEN}" "${MAVEN_ARGS[@]}" package -DskipTests -Dmaven.javadoc.skip=true -Ptest-patch
-  FLINK_POST_LIB_FILES=$(flinklib_count)
   popd >/dev/null
 
+  if [[ "${BUILDMODE}" = full ]]; then
+    if [[ ${FLINK_POST_LIB_FILES} -gt 0 ]]; then
+      add_vote_table -1 flinklib "Lib folder dependencies are currently ${FLINK_POST_LIB_FILES}"
+      return 1
+    else
+      add_vote_table +1 flinklib "No lib folder dependencies!"
+      return 0
+    fi
+  fi
 
   if [[ "${FLINK_POST_LIB_FILES}" -gt "${FLINK_PRE_LIB_FILES}" ]]; then
     add_vote_table -1 flinklib "Patch increases lib folder dependencies from " \
@@ -99,16 +110,4 @@ function flinklib_postapply
       "$((FLINK_PRE_LIB_FILES-FLINK_POST_LIB_FILES))."
   fi
   return 0
-}
-
-function flinklib_rebuild
-{
-  declare repostatus=$1
-
-  if [[ "${repostatus}" = branch ]]; then
-    flinklib_preapply
-  else
-    flinklib_postinstall
-  fi
-}
-
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/yetus/blob/64377ca5/precommit/personality/hbase.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/hbase.sh b/precommit/personality/hbase.sh
index e9e4ea8..5e6f0dc 100755
--- a/precommit/personality/hbase.sh
+++ b/precommit/personality/hbase.sh
@@ -22,17 +22,23 @@ personality_plugins "all"
 function personality_globals
 {
   #shellcheck disable=SC2034
+  PROJECT_NAME=hbase
+  #shellcheck disable=SC2034
   PATCH_BRANCH_DEFAULT=master
   #shellcheck disable=SC2034
   JIRA_ISSUE_RE='^HBASE-[0-9]+$'
   #shellcheck disable=SC2034
   GITHUB_REPO="apache/hbase"
 
+  # TODO use PATCH_BRANCH to select hadoop versions to use.
   # All supported Hadoop versions that we want to test the compilation with
-  HBASE_HADOOP_VERSIONS="2.4.1 2.5.2 2.6.0"
+  HBASE_HADOOP_VERSIONS="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.1 2.6.2 2.6.3 2.7.1"
+
+  # TODO use PATCH_BRANCH to select jdk versions to use.
 
   # Override the maven options
   MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}"
+
 }
 
 ## @description  Queue up modules for this personality
@@ -54,16 +60,19 @@ function personality_modules
 
   if [[ ${repostatus} == branch
      && ${testtype} == mvninstall ]] ||
-     [[ "${BUILDMODE}" == full ]]; then
-     personality_enqueue_module . ${extra}
-     return
-   fi
+     [[ "${BUILDMODE}" == full ]];then
+    personality_enqueue_module . ${extra}
+    return
+  fi
 
   if [[ ${testtype} = findbugs ]]; then
     for module in "${CHANGED_MODULES[@]}"; do
-      # skip findbugs on hbase-shell
+      # skip findbugs on hbase-shell and hbase-it. hbase-it has nothing
+      # in src/main/java where findbugs goes to look
       if [[ ${module} == hbase-shell ]]; then
         continue
+      elif [[ ${module} == hbase-it ]]; then
+        continue
       else
         # shellcheck disable=SC2086
         personality_enqueue_module ${module} ${extra}
@@ -72,6 +81,16 @@ function personality_modules
     return
   fi
 
+  if [[ ${testtype} = unit ]]; then
+    extra="${extra} -PrunAllTests"
+
+    # Inject the jenkins build-id for our surefire invocations
+    # Used by zombie detection stuff, even though we're not including that yet.
+    if [ -n "${BUILD_ID}" ]; then
+      extra="${extra} -Dbuild.id=${BUILD_ID}"
+    fi
+  fi
+
   for module in "${CHANGED_MODULES[@]}"; do
     # shellcheck disable=SC2086
     personality_enqueue_module ${module} ${extra}
@@ -79,9 +98,19 @@ function personality_modules
 }
 
 ###################################################
+# Below here are our one-off tests specific to hbase.
+# TODO break them into individual files so it's easier to maintain them?
+
+# TODO line length check? could ignore all java files since checkstyle gets them.
+
+###################################################
 
 add_test_type hadoopcheck
 
+## @description  hadoopcheck file filter
+## @audience     private
+## @stability    evolving
+## @param        filename
 function hadoopcheck_filefilter
 {
   local filename=$1
@@ -91,6 +120,10 @@ function hadoopcheck_filefilter
   fi
 }
 
+## @description  hadoopcheck test
+## @audience     private
+## @stability    evolving
+## @param        repostatus
 function hadoopcheck_rebuild
 {
   local repostatus=$1
@@ -114,7 +147,7 @@ function hadoopcheck_rebuild
         -Dhadoop-two.version="${hadoopver}"
     count=$(${GREP} -c ERROR "${logfile}")
     if [[ ${count} -gt 0 ]]; then
-      add_vote_table -1 hadoopcheck "Patch causes ${count} errors with Hadoop v${hadoopver}."
+      add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
       ((result=result+1))
     fi
   done
@@ -123,14 +156,19 @@ function hadoopcheck_rebuild
     return 1
   fi
 
-  add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${HBASE_HADOOP_VERSIONS}."
+  add_vote_table +1 hadoopcheck "${BUILDMODEMSG} does not cause any errors with Hadoop ${HBASE_HADOOP_VERSIONS}."
   return 0
 }
 
 ######################################
 
+# TODO if we need the protoc check, we probably need to check building all the modules that
rely on hbase-protocol
 add_test_type hbaseprotoc
 
+## @description  hbaseprotoc file filter
+## @audience     private
+## @stability    evolving
+## @param        filename
 function hbaseprotoc_filefilter
 {
   local filename=$1
@@ -140,33 +178,38 @@ function hbaseprotoc_filefilter
   fi
 }
 
+## @description  hadoopcheck test
+## @audience     private
+## @stability    evolving
+## @param        repostatus
 function hbaseprotoc_rebuild
 {
-  local i=0
-  local fn
-  local module
-  local logfile
-  local count
-  local result
+  declare repostatus=$1
+  declare i=0
+  declare fn
+  declare module
+  declare logfile
+  declare count
+  declare result
 
   if [[ "${repostatus}" = branch ]]; then
     return 0
   fi
 
-  if ! verify_needed_test hbaseprotoc; then
+  verify_needed_test hbaseprotoc
+  if [[ $? == 0 ]]; then
     return 0
   fi
 
-  big_console_header "Patch HBase protoc plugin"
+  big_console_header "HBase protoc plugin: ${BUILDMODE}"
 
   start_clock
 
-
   personality_modules patch hbaseprotoc
   modules_workers patch hbaseprotoc compile -DskipTests -Pcompile-protobuf -X -DHBasePatchProcess
 
   # shellcheck disable=SC2153
-  until [[ $i -eq ${#MODULE[@]} ]]; do
+  until [[ $i -eq "${#MODULE[@]}" ]]; do
     if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
       ((result=result+1))
       ((i=i+1))
@@ -197,6 +240,10 @@ function hbaseprotoc_rebuild
 
 add_test_type hbaseanti
 
+## @description  hbaseanti file filter
+## @audience     private
+## @stability    evolving
+## @param        filename
 function hbaseanti_filefilter
 {
   local filename=$1
@@ -206,13 +253,22 @@ function hbaseanti_filefilter
   fi
 }
 
+## @description  hbaseanti patch file check
+## @audience     private
+## @stability    evolving
+## @param        filename
 function hbaseanti_patchfile
 {
   local patchfile=$1
   local warnings
   local result
 
-  if ! verify_needed_test hbaseanti; then
+  if [[ "${BUILDMODE}" = full ]]; then
+    return 0
+  fi
+
+  verify_needed_test hbaseanti
+  if [[ $? == 0 ]]; then
     return 0
   fi
 
@@ -239,3 +295,75 @@ function hbaseanti_patchfile
   add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
   return 0
 }
+
+
+## @description  hbase custom mvnsite file filter.  See HBASE-15042
+## @audience     private
+## @stability    evolving
+## @param        filename
+function mvnsite_filefilter
+{
+  local filename=$1
+
+  if [[ ${BUILDTOOL} = maven ]]; then
+    if [[ ${filename} =~ src/main/site || ${filename} =~ src/main/asciidoc ]]; then
+      yetus_debug "tests/mvnsite: ${filename}"
+      add_test mvnsite
+    fi
+  fi
+}
+
+## This is named so that yetus will check us right after running tests.
+## Essentially, we check for normal failures and then we look for zombies.
+#function hbase_unit_logfilter
+#{
+#  declare testtype="unit"
+#  declare input=$1
+#  declare output=$2
+#  declare processes
+#  declare process_output
+#  declare zombies
+#  declare zombie_count=0
+#  declare zombie_process
+#
+#  yetus_debug "in hbase-specific unit logfilter."
+#
+#  # pass-through to whatever is counting actual failures
+#  if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then
+#    "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}"
+#  elif declare -f ${testtype}_logfilter >/dev/null; then
+#    "${testtype}_logfilter" "${input}" "${output}"
+#  fi
+#
+#  start_clock
+#  if [ -n "${BUILD_ID}" ]; then
+#    yetus_debug "Checking for zombie test processes."
+#    processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
+#    if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then
+#      yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just
slow to stop."
+#      yetus_debug "${processes}"
+#      sleep 30
+#      #shellcheck disable=SC2016
+#      for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do
+#        # Test our zombie still running (and that it still an hbase build item)
+#        process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
+#        if [[ -n "${process_output}" ]]; then
+#          yetus_error "Zombie: ${process_output}"
+#          ((zombie_count = zombie_count + 1))
+#          zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"|
head -3)
+#          zombies="${zombies} ${zombie_process}"
+#        fi
+#      done
+#    fi
+#    if [ "${zombie_count}" -ne 0 ]; then
+#      add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)"
+#      populate_test_table "zombie unit tests" "${zombies}"
+#    else
+#      yetus_info "Zombie check complete. All test runs exited normally."
+#      stop_clock
+#    fi
+#  else
+#    add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies."
+#  fi
+#
+#}

http://git-wip-us.apache.org/repos/asf/yetus/blob/64377ca5/precommit/personality/jmeter.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/jmeter.sh b/precommit/personality/jmeter.sh
index 194fc4f..b4e61b5 100755
--- a/precommit/personality/jmeter.sh
+++ b/precommit/personality/jmeter.sh
@@ -58,6 +58,9 @@ function jmeter_parse_args
   done
 }
 
+## @description  Download jmetere dependencies
+## @audience     private
+## @stability    evolving
 function jmeter_precheck
 {
   if [[ ${JMETER_DOWNLOAD_JARS} = true ]]; then


Mime
View raw message