hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bus...@apache.org
Subject [1/4] hbase git commit: HBASE-18833 Ensure precommit personality is up to date on all active branches
Date Wed, 20 Sep 2017 11:05:08 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1.1 68cd18176 -> 55f6e3346
  refs/heads/branch-1.2 da25def91 -> 2ea2cee60
  refs/heads/branch-1.3 c7a165cb4 -> ee6131fbd
  refs/heads/branch-1.4 cfbbcf5c1 -> 09956077f


HBASE-18833 Ensure precommit personality is up to date on all active branches


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ee6131fb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ee6131fb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ee6131fb

Branch: refs/heads/branch-1.3
Commit: ee6131fbd21847df5d824ae325e8616678a7dbf7
Parents: c7a165c
Author: Sean Busbey <busbey@apache.org>
Authored: Mon Sep 18 09:00:23 2017 -0500
Committer: Sean Busbey <busbey@apache.org>
Committed: Wed Sep 20 06:01:07 2017 -0500

----------------------------------------------------------------------
 dev-support/hbase-personality.sh | 498 ++++++++++++++++++++++++++++++++++
 1 file changed, 498 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/ee6131fb/dev-support/hbase-personality.sh
----------------------------------------------------------------------
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
new file mode 100755
index 0000000..b7a2f97
--- /dev/null
+++ b/dev-support/hbase-personality.sh
@@ -0,0 +1,498 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# You'll need a local installation of
+# [Apache Yetus' precommit checker](http://yetus.apache.org/documentation/0.1.0/#yetus-precommit)
+# to use this personality.
+#
+# Download from: http://yetus.apache.org/downloads/ . You can either grab the source artifact
and
+# build from it, or use the convenience binaries provided on that download page.
+#
+# To run against, e.g. HBASE-15074 you'd then do
+# ```bash
+# test-patch --personality=dev-support/hbase-personality.sh HBASE-15074
+# ```
+#
+# If you want to skip the ~1 hour it'll take to do all the hadoop API checks, use
+# ```bash
+# test-patch  --plugins=all,-hadoopcheck --personality=dev-support/hbase-personality.sh HBASE-15074
+# ````
+#
+# pass the `--jenkins` flag if you want to allow test-patch to destructively alter local
working
+# directory / branch in order to have things match what the issue patch requests.
+
+personality_plugins "all"
+
+## @description  Globals specific to this personality
+## @audience     private
+## @stability    evolving
+function personality_globals
+{
+  BUILDTOOL=maven
+  #shellcheck disable=SC2034
+  PROJECT_NAME=hbase
+  #shellcheck disable=SC2034
+  PATCH_BRANCH_DEFAULT=master
+  #shellcheck disable=SC2034
+  JIRA_ISSUE_RE='^HBASE-[0-9]+$'
+  #shellcheck disable=SC2034
+  GITHUB_REPO="apache/hbase"
+
+  # All supported Hadoop versions that we want to test the compilation with
+  # See the Hadoop section on prereqs in the HBase Reference Guide
+  if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
+    HBASE_HADOOP2_VERSIONS="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1
2.7.2 2.7.3"
+    HBASE_HADOOP3_VERSIONS=""
+  elif [[ ${PATCH_BRANCH} = branch-2* ]]; then
+    HBASE_HADOOP2_VERSIONS="2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3"
+    HBASE_HADOOP3_VERSIONS="3.0.0-alpha4"
+  else # master or a feature branch
+    HBASE_HADOOP2_VERSIONS="2.6.1 2.6.2 2.6.3 2.6.4 2.6.5 2.7.1 2.7.2 2.7.3"
+    HBASE_HADOOP3_VERSIONS="3.0.0-alpha4"
+  fi
+
+  # TODO use PATCH_BRANCH to select jdk versions to use.
+
+  # Override the maven options
+  MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}"
+
+}
+
+## @description  Queue up modules for this personality
+## @audience     private
+## @stability    evolving
+## @param        repostatus
+## @param        testtype
+function personality_modules
+{
+  local repostatus=$1
+  local testtype=$2
+  local extra=""
+
+  yetus_debug "Personality: ${repostatus} ${testtype}"
+
+  clear_personality_queue
+
+  extra="-DHBasePatchProcess"
+
+  if [[ ${repostatus} == branch
+     && ${testtype} == mvninstall ]] ||
+     [[ "${BUILDMODE}" == full ]];then
+    personality_enqueue_module . ${extra}
+    return
+  fi
+
+  if [[ ${testtype} = findbugs ]]; then
+    for module in "${CHANGED_MODULES[@]}"; do
+      # skip findbugs on hbase-shell and hbase-it. hbase-it has nothing
+      # in src/main/java where findbugs goes to look
+      if [[ ${module} == hbase-shell ]]; then
+        continue
+      elif [[ ${module} == hbase-it ]]; then
+        continue
+      else
+        # shellcheck disable=SC2086
+        personality_enqueue_module ${module} ${extra}
+      fi
+    done
+    return
+  fi
+
+  # If EXCLUDE_TESTS_URL/INCLUDE_TESTS_URL is set, fetches the url
+  # and sets -Dtest.exclude.pattern/-Dtest to exclude/include the
+  # tests respectively.
+  if [[ ${testtype} = unit ]]; then
+    extra="${extra} -PrunAllTests"
+    yetus_debug "EXCLUDE_TESTS_URL = ${EXCLUDE_TESTS_URL}"
+    yetus_debug "INCLUDE_TESTS_URL = ${INCLUDE_TESTS_URL}"
+    if [[ -n "$EXCLUDE_TESTS_URL" ]]; then
+        if wget "$EXCLUDE_TESTS_URL" -O "excludes"; then
+          excludes=$(cat excludes)
+          yetus_debug "excludes=${excludes}"
+          if [[ -n "${excludes}" ]]; then
+            extra="${extra} -Dtest.exclude.pattern=${excludes}"
+          fi
+          rm excludes
+        else
+          echo "Wget error $? in fetching excludes file from url" \
+               "${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
+        fi
+    elif [[ -n "$INCLUDE_TESTS_URL" ]]; then
+        if wget "$INCLUDE_TESTS_URL" -O "includes"; then
+          includes=$(cat includes)
+          yetus_debug "includes=${includes}"
+          if [[ -n "${includes}" ]]; then
+            extra="${extra} -Dtest=${includes}"
+          fi
+          rm includes
+        else
+          echo "Wget error $? in fetching includes file from url" \
+               "${INCLUDE_TESTS_URL}. Ignoring and proceeding."
+        fi
+    fi
+
+    # Inject the jenkins build-id for our surefire invocations
+    # Used by zombie detection stuff, even though we're not including that yet.
+    if [ -n "${BUILD_ID}" ]; then
+      extra="${extra} -Dbuild.id=${BUILD_ID}"
+    fi
+  fi
+
+  for module in "${CHANGED_MODULES[@]}"; do
+    # shellcheck disable=SC2086
+    personality_enqueue_module ${module} ${extra}
+  done
+}
+
+###################################################
+# Below here are our one-off tests specific to hbase.
+# TODO break them into individual files so it's easier to maintain them?
+
+# TODO line length check? could ignore all java files since checkstyle gets them.
+
+###################################################
+
+add_test_type shadedjars
+
+
+function shadedjars_initialize
+{
+  yetus_debug "initializing shaded client checks."
+  maven_add_install shadedjars
+  add_test shadedjars
+}
+
+function shadedjars_clean
+{
+  "${MAVEN}" "${MAVEN_ARGS[@]}" clean -fae -pl hbase_shaded/hbase-shaded-check-invariants
-am -Prelease
+}
+
+## @description test the shaded client artifacts
+## @audience private
+## @stability evolving
+## @param repostatus
+function shadedjars_rebuild
+{
+  local repostatus=$1
+  local logfile="${PATCH_DIR}/${repostatus}-shadedjars.txt"
+
+  big_console_header "Checking shaded client builds on ${repostatus}"
+
+  echo_and_redirect "${logfile}" \
+    "${MAVEN}" "${MAVEN_ARGS[@]}" clean verify -fae --batch-mode \
+      -pl hbase-shaded/hbase-shaded-check-invariants -am \
+      -Dtest=NoUnitTests -DHBasePatchProcess -Prelease \
+      -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true
+
+  count=$(${GREP} -c '\[ERROR\]' "${logfile}")
+  if [[ ${count} -gt 0 ]]; then
+    add_vote_table -1 shadedjars "${repostatus} has ${count} errors when building our shaded
downstream artifacts."
+    return 1
+  fi
+
+  add_vote_table +1 shadedjars "${repostatus} has no errors when building our shaded downstream
artifacts."
+  return 0
+}
+
+###################################################
+
+add_test_type hadoopcheck
+
+## @description  hadoopcheck file filter
+## @audience     private
+## @stability    evolving
+## @param        filename
+function hadoopcheck_filefilter
+{
+  local filename=$1
+
+  if [[ ${filename} =~ \.java$ ]]; then
+    add_test hadoopcheck
+  fi
+}
+
+## @description  hadoopcheck test
+## @audience     private
+## @stability    evolving
+## @param        repostatus
+function hadoopcheck_rebuild
+{
+  local repostatus=$1
+  local hadoopver
+  local logfile
+  local count
+  local result=0
+  local hbase_hadoop2_versions
+  local hbase_hadoop3_versions
+
+  if [[ "${repostatus}" = branch ]]; then
+    return 0
+  fi
+
+  big_console_header "Compiling against various Hadoop versions"
+
+  hbase_hadoop2_versions=${HBASE_HADOOP2_VERSIONS}
+  hbase_hadoop3_versions=${HBASE_HADOOP3_VERSIONS}
+
+
+  export MAVEN_OPTS="${MAVEN_OPTS}"
+  for hadoopver in ${hbase_hadoop2_versions}; do
+    logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
+    echo_and_redirect "${logfile}" \
+      "${MAVEN}" clean install \
+        -DskipTests -DHBasePatchProcess \
+        -Dhadoop-two.version="${hadoopver}"
+    count=$(${GREP} -c '\[ERROR\]' "${logfile}")
+    if [[ ${count} -gt 0 ]]; then
+      add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
+      ((result=result+1))
+    fi
+  done
+
+  for hadoopver in ${hbase_hadoop3_versions}; do
+    logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
+    echo_and_redirect "${logfile}" \
+      "${MAVEN}" clean install \
+        -DskipTests -DHBasePatchProcess \
+        -Dhadoop-three.version="${hadoopver} \
+        -Dhadoop.profile=3.0"
+    count=$(${GREP} -c '\[ERROR\]' "${logfile}")
+    if [[ ${count} -gt 0 ]]; then
+      add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
+      ((result=result+1))
+    fi
+  done
+
+  if [[ ${result} -gt 0 ]]; then
+    return 1
+  fi
+
+  if [[ -n "${hbase_hadoop3_versions}" ]]; then
+    add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions}
or ${hbase_hadoop3_versions}."
+  else
+    add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${hbase_hadoop2_versions}."
+  fi
+  return 0
+}
+
+######################################
+
+# TODO if we need the protoc check, we probably need to check building all the modules that
rely on hbase-protocol
+add_test_type hbaseprotoc
+
+## @description  hbaseprotoc file filter
+## @audience     private
+## @stability    evolving
+## @param        filename
+function hbaseprotoc_filefilter
+{
+  local filename=$1
+
+  if [[ ${filename} =~ \.proto$ ]]; then
+    add_test hbaseprotoc
+  fi
+}
+
+## @description  hadoopcheck test
+## @audience     private
+## @stability    evolving
+## @param        repostatus
+function hbaseprotoc_rebuild
+{
+  declare repostatus=$1
+  declare i=0
+  declare fn
+  declare module
+  declare logfile
+  declare count
+  declare result
+
+  if [[ "${repostatus}" = branch ]]; then
+    return 0
+  fi
+
+  if ! verify_needed_test hbaseprotoc; then
+    return 0
+  fi
+
+  big_console_header "HBase protoc plugin: ${BUILDMODE}"
+
+  start_clock
+
+  personality_modules patch hbaseprotoc
+  # Need to run 'install' instead of 'compile' because shading plugin
+  # is hooked-up to 'install'; else hbase-protocol-shaded is left with
+  # half of its process done.
+  modules_workers patch hbaseprotoc install -DskipTests -Pcompile-protobuf -X -DHBasePatchProcess
+
+  # shellcheck disable=SC2153
+  until [[ $i -eq "${#MODULE[@]}" ]]; do
+    if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
+      ((result=result+1))
+      ((i=i+1))
+      continue
+    fi
+    module=${MODULE[$i]}
+    fn=$(module_file_fragment "${module}")
+    logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt"
+
+    count=$(${GREP} -c '\[ERROR\]' "${logfile}")
+
+    if [[ ${count} -gt 0 ]]; then
+      module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\
+        "${count} new protoc errors in ${module}."
+      ((result=result+1))
+    fi
+    ((i=i+1))
+  done
+
+  modules_messages patch hbaseprotoc true
+  if [[ ${result} -gt 0 ]]; then
+    return 1
+  fi
+  return 0
+}
+
+######################################
+
+add_test_type hbaseanti
+
+## @description  hbaseanti file filter
+## @audience     private
+## @stability    evolving
+## @param        filename
+function hbaseanti_filefilter
+{
+  local filename=$1
+
+  if [[ ${filename} =~ \.java$ ]]; then
+    add_test hbaseanti
+  fi
+}
+
+## @description  hbaseanti patch file check
+## @audience     private
+## @stability    evolving
+## @param        filename
+function hbaseanti_patchfile
+{
+  local patchfile=$1
+  local warnings
+  local result
+
+  if [[ "${BUILDMODE}" = full ]]; then
+    return 0
+  fi
+
+  if ! verify_needed_test hbaseanti; then
+    return 0
+  fi
+
+  big_console_header "Checking for known anti-patterns"
+
+  start_clock
+
+  warnings=$(${GREP} 'new TreeMap<byte.*()' "${patchfile}")
+  if [[ ${warnings} -gt 0 ]]; then
+    add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR
was omitted: ${warnings}."
+    ((result=result+1))
+  fi
+
+  warnings=$(${GREP} 'import org.apache.hadoop.classification' "${patchfile}")
+  if [[ ${warnings} -gt 0 ]]; then
+    add_vote_table -1 hbaseanti "" "The patch appears use Hadoop classification instead of
HBase: ${warnings}."
+    ((result=result+1))
+  fi
+
+  if [[ ${result} -gt 0 ]]; then
+    return 1
+  fi
+
+  add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
+  return 0
+}
+
+
+## @description  hbase custom mvnsite file filter.  See HBASE-15042
+## @audience     private
+## @stability    evolving
+## @param        filename
+function mvnsite_filefilter
+{
+  local filename=$1
+
+  if [[ ${BUILDTOOL} = maven ]]; then
+    if [[ ${filename} =~ src/site || ${filename} =~ src/main/asciidoc ]]; then
+      yetus_debug "tests/mvnsite: ${filename}"
+      add_test mvnsite
+    fi
+  fi
+}
+
+## This is named so that yetus will check us right after running tests.
+## Essentially, we check for normal failures and then we look for zombies.
+#function hbase_unit_logfilter
+#{
+#  declare testtype="unit"
+#  declare input=$1
+#  declare output=$2
+#  declare processes
+#  declare process_output
+#  declare zombies
+#  declare zombie_count=0
+#  declare zombie_process
+#
+#  yetus_debug "in hbase-specific unit logfilter."
+#
+#  # pass-through to whatever is counting actual failures
+#  if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then
+#    "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}"
+#  elif declare -f ${testtype}_logfilter >/dev/null; then
+#    "${testtype}_logfilter" "${input}" "${output}"
+#  fi
+#
+#  start_clock
+#  if [ -n "${BUILD_ID}" ]; then
+#    yetus_debug "Checking for zombie test processes."
+#    processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
+#    if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then
+#      yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just
slow to stop."
+#      yetus_debug "${processes}"
+#      sleep 30
+#      #shellcheck disable=SC2016
+#      for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do
+#        # Test our zombie still running (and that it still an hbase build item)
+#        process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
+#        if [[ -n "${process_output}" ]]; then
+#          yetus_error "Zombie: ${process_output}"
+#          ((zombie_count = zombie_count + 1))
+#          zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"|
head -3)
+#          zombies="${zombies} ${zombie_process}"
+#        fi
+#      done
+#    fi
+#    if [ "${zombie_count}" -ne 0 ]; then
+#      add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)"
+#      populate_test_table "zombie unit tests" "${zombies}"
+#    else
+#      yetus_info "Zombie check complete. All test runs exited normally."
+#      stop_clock
+#    fi
+#  else
+#    add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies."
+#  fi
+#
+#}


Mime
View raw message