hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject [1/4] hbase git commit: HBASE-15009 Update test-patch.sh on branches; to fix curtailed build report
Date Mon, 04 Jan 2016 23:43:24 GMT
Repository: hbase
Updated Branches:
  refs/heads/0.98 4bdd2b2de -> cf38756ef


HBASE-15009 Update test-patch.sh on branches; to fix curtailed build report

Conflicts:
	dev-support/jenkinsEnv.sh
	dev-support/test-patch.sh


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d93b4372
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d93b4372
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d93b4372

Branch: refs/heads/0.98
Commit: d93b43723c2b3fe130ae416faef5c1234d51d4df
Parents: 4bdd2b2
Author: stack <stack@apache.org>
Authored: Fri Dec 18 09:31:22 2015 -0800
Committer: Andrew Purtell <apurtell@apache.org>
Committed: Mon Jan 4 15:18:41 2016 -0800

----------------------------------------------------------------------
 dev-support/findHangingTest.sh  |  40 ---
 dev-support/findHangingTests.py |  82 ++++++
 dev-support/jenkinsEnv.sh       |  36 +++
 dev-support/test-patch.sh       | 492 +++++++++++++++++++++--------------
 dev-support/zombie-detector.sh  | 166 ++++++++++++
 5 files changed, 587 insertions(+), 229 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/d93b4372/dev-support/findHangingTest.sh
----------------------------------------------------------------------
diff --git a/dev-support/findHangingTest.sh b/dev-support/findHangingTest.sh
deleted file mode 100755
index f7ebe47..0000000
--- a/dev-support/findHangingTest.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/bash
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-# script to find hanging test from Jenkins build output
-# usage: ./findHangingTest.sh <url of Jenkins build console>
-#
-`curl -k -o jenkins.out "$1"`
-expecting=Running
-cat jenkins.out | while read line; do
- if [[ "$line" =~ "Running org.apache.hadoop" ]]; then
-  if [[ "$expecting" =~ "Running" ]]; then 
-   expecting=Tests
-  else
-   echo "Hanging test: $prevLine"
-  fi
- fi
- if [[ "$line" =~ "Tests run" ]]; then
-  expecting=Running
- fi
- if [[ "$line" =~ "Forking command line" ]]; then
-  a=$line
- else
-  prevLine=$line
- fi
-done

http://git-wip-us.apache.org/repos/asf/hbase/blob/d93b4372/dev-support/findHangingTests.py
----------------------------------------------------------------------
diff --git a/dev-support/findHangingTests.py b/dev-support/findHangingTests.py
new file mode 100644
index 0000000..deccc8b
--- /dev/null
+++ b/dev-support/findHangingTests.py
@@ -0,0 +1,82 @@
+#!/usr/bin/python
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+# script to find hanging test from Jenkins build output
+# usage: ./findHangingTests.py <url of Jenkins build console>
+#
+import urllib2
+import sys
+import string
+if len(sys.argv) != 2 :
+  print "ERROR : Provide the jenkins job console URL as the only argument."
+  exit(1)
+print "Fetching " + sys.argv[1]
+response = urllib2.urlopen(sys.argv[1])
+i = 0;
+tests = {}
+failed_tests = {}
+summary = 0
+host = False
+patch = False
+branch = False
+while True:
+  n = response.readline()
+  if n == "" :
+    break
+  if not host and n.find("Building remotely on") >= 0:
+    host = True
+    print n.strip()    
+    continue
+  if not patch and n.find("Testing patch for ") >= 0:
+    patch = True
+    print n.strip()    
+    continue
+  if not branch and n.find("Testing patch on branch ") >= 0:
+    branch = True
+    print n.strip()    
+    continue
+  if n.find("PATCH APPLICATION FAILED") >= 0:
+    print "PATCH APPLICATION FAILED"
+    sys.exit(1) 
+  if summary == 0 and n.find("Running tests.") >= 0:
+    summary = summary + 1
+    continue
+  if summary == 1 and n.find("[INFO] Reactor Summary:") >= 0:
+    summary = summary + 1
+    continue
+  if summary == 2 and n.find("[INFO] Apache HBase ") >= 0:
+    sys.stdout.write(n)
+    continue
+  if n.find("org.apache.hadoop.hbase") < 0:
+    continue 
+  test_name = string.strip(n[n.find("org.apache.hadoop.hbase"):len(n)])
+  if n.find("Running org.apache.hadoop.hbase") > -1 :
+    tests[test_name] = False
+  if n.find("Tests run:") > -1 :
+    if n.find("FAILURE") > -1 or n.find("ERROR") > -1:
+      failed_tests[test_name] = True
+    tests[test_name] = True
+response.close()
+
+print "Printing hanging tests"
+for key, value in tests.iteritems():
+  if value == False:
+    print "Hanging test : " + key
+print "Printing Failing tests"
+for key, value in failed_tests.iteritems():
+  print "Failing test : " + key

http://git-wip-us.apache.org/repos/asf/hbase/blob/d93b4372/dev-support/jenkinsEnv.sh
----------------------------------------------------------------------
diff --git a/dev-support/jenkinsEnv.sh b/dev-support/jenkinsEnv.sh
new file mode 100755
index 0000000..6961437
--- /dev/null
+++ b/dev-support/jenkinsEnv.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#   Licensed to the Apache Software Foundation (ASF) under one
+#   or more contributor license agreements.  See the NOTICE file
+#   distributed with this work for additional information
+#   regarding copyright ownership.  The ASF licenses this file
+#   to you under the Apache License, Version 2.0 (the
+#   "License"); you may not use this file except in compliance
+#   with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing,
+#   software distributed under the License is distributed on an
+#   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#   KIND, either express or implied.  See the License for the
+#   specific language governing permissions and limitations
+#   under the License.
+
+#!/bin/bash
+
+##To set Jenkins Environment Variables:
+export JAVA_HOME=/home/jenkins/tools/java/latest
+export ANT_HOME=/home/jenkins/tools/ant/latest
+export XERCES_HOME=/home/jenkins/tools/xerces/latest
+export ECLIPSE_HOME=/home/jenkins/tools/eclipse/latest
+export FORREST_HOME=/home/jenkins/tools/forrest/latest
+export JAVA5_HOME=/home/jenkins/tools/java5/latest
+export FINDBUGS_HOME=/home/jenkins/tools/findbugs/latest
+export CLOVER_HOME=/home/jenkins/tools/clover/latest
+export MAVEN_HOME=/home/jenkins/tools/maven/latest
+
+export PATH=$PATH:$JAVA_HOME/bin:$ANT_HOME/bin:
+export MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M -XX:-UsePerfData -XX:MaxPermSize=256m"}"
+
+ulimit -n
+

http://git-wip-us.apache.org/repos/asf/hbase/blob/d93b4372/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 19a603f..509edeb 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -20,7 +20,7 @@
 #set -x
 
 ### Setup some variables.  
-### SVN_REVISION and BUILD_URL are set by Hudson if it is run by patch process
+### GIT_COMMIT and BUILD_URL are set by Hudson if it is run by patch process
 ### Read variables from properties file
 bindir=$(dirname $0)
 
@@ -31,21 +31,28 @@ else
   MVN=$MAVEN_HOME/bin/mvn
 fi
 
+NEWLINE=$'\n'
+
 PROJECT_NAME=HBase
 JENKINS=false
+MOVE_PATCH_DIR=true
 PATCH_DIR=/tmp
 BASEDIR=$(pwd)
+BRANCH_NAME="master"
+
+. $BASEDIR/dev-support/test-patch.properties
 
 PS=${PS:-ps}
 AWK=${AWK:-awk}
 WGET=${WGET:-wget}
-SVN=${SVN:-svn}
 GREP=${GREP:-grep}
+EGREP=${EGREP:-egrep}
 PATCH=${PATCH:-patch}
 JIRACLI=${JIRA:-jira}
 FINDBUGS_HOME=${FINDBUGS_HOME}
 FORREST_HOME=${FORREST_HOME}
 ECLIPSE_HOME=${ECLIPSE_HOME}
+GIT=${GIT:-git}
 
 ###############################################################################
 printUsage() {
@@ -61,12 +68,12 @@ printUsage() {
   echo "--mvn-cmd=<cmd>        The 'mvn' command to use (default \$MAVEN_HOME/bin/mvn,
or 'mvn')"
   echo "--ps-cmd=<cmd>         The 'ps' command to use (default 'ps')"
   echo "--awk-cmd=<cmd>        The 'awk' command to use (default 'awk')"
-  echo "--svn-cmd=<cmd>        The 'svn' command to use (default 'svn')"
   echo "--grep-cmd=<cmd>       The 'grep' command to use (default 'grep')"
   echo "--patch-cmd=<cmd>      The 'patch' command to use (default 'patch')"
   echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment
variable)"
   echo "--forrest-home=<path>  Forrest home directory (default FORREST_HOME environment
variable)"
-  echo "--dirty-workspace      Allow the local SVN workspace to have uncommitted changes"
+  echo "--dirty-workspace      Allow the local workspace to have uncommitted changes"
+  echo "--git-cmd=<cmd>        The 'git' command to use (default 'git')"
   echo
   echo "Jenkins-only options:"
   echo "--jenkins              Run by Jenkins (runs tests and posts results to JIRA)"
@@ -84,6 +91,9 @@ parseArgs() {
     --jenkins)
       JENKINS=true
       ;;
+    --no-move-patch-dir)
+      MOVE_PATCH_DIR=false
+      ;;
     --patch-dir=*)
       PATCH_DIR=${i#*=}
       ;;
@@ -102,9 +112,6 @@ parseArgs() {
     --wget-cmd=*)
       WGET=${i#*=}
       ;;
-    --svn-cmd=*)
-      SVN=${i#*=}
-      ;;
     --grep-cmd=*)
       GREP=${i#*=}
       ;;
@@ -129,6 +136,9 @@ parseArgs() {
     --dirty-workspace)
       DIRTY_WORKSPACE=true
       ;;
+    --git-cmd=*)
+      GIT=${i#*=}
+      ;;
     *)
       PATCH_OR_DEFECT=$i
       ;;
@@ -179,23 +189,92 @@ checkout () {
   echo ""
   ### When run by a developer, if the workspace contains modifications, do not continue
   ### unless the --dirty-workspace option was set
-  status=`$SVN stat --ignore-externals | sed -e '/^X[ ]*/D'`
   if [[ $JENKINS == "false" ]] ; then
-    if [[ "$status" != "" && -z $DIRTY_WORKSPACE ]] ; then
-      echo "ERROR: can't run in a workspace that contains the following modifications"
-      echo "$status"
-      cleanupAndExit 1
+    if [[ -z $DIRTY_WORKSPACE ]] ; then
+      # Ref http://stackoverflow.com/a/2659808 for details on checking dirty status
+      ${GIT} diff-index --quiet HEAD
+      if [[ $? -ne 0 ]] ; then
+        uncommitted=`${GIT} diff --name-only HEAD`
+        uncommitted="You have the following files with uncommitted changes:${NEWLINE}${uncommitted}"
+      fi
+      untracked="$(${GIT} ls-files --exclude-standard --others)" && test -z "${untracked}"
+      if [[ $? -ne 0 ]] ; then
+        untracked="You have untracked and unignored files:${NEWLINE}${untracked}"
+      fi
+      if [[ $uncommitted || $untracked ]] ; then
+        echo "ERROR: can't run in a workspace that contains modifications."
+        echo "Pass the '--dirty-workspace' flag to bypass."
+        echo ""
+        echo "${uncommitted}"
+        echo ""
+        echo "${untracked}"
+        cleanupAndExit 1
+      fi
     fi
     echo
-  else   
-    cd $BASEDIR
-    $SVN revert -R .
-    rm -rf `$SVN status --no-ignore`
-    $SVN update
   fi
   return $?
 }
 
+findBranchNameFromPatchName() {
+  local patchName=$1
+  for LOCAL_BRANCH_NAME in $BRANCH_NAMES; do
+    if [[ $patchName =~ /jira/secure/attachment/[0-9]*/.*$LOCAL_BRANCH_NAME ]]; then
+      BRANCH_NAME=$LOCAL_BRANCH_NAME
+      break
+    fi
+  done
+  return 0
+}
+
+checkoutBranch() {
+  echo ""
+  echo ""
+  echo "======================================================================"
+  echo "======================================================================"
+  echo "    Testing patch on branch ${BRANCH_NAME}."
+  echo "======================================================================"
+  echo "======================================================================"
+  echo ""
+  echo ""
+  if [[ $JENKINS == "true" ]] ; then
+    if [[ "$BRANCH_NAME" != "master" ]]; then
+      echo "origin/${BRANCH_NAME} HEAD is commit `${GIT} rev-list origin/${BRANCH_NAME} -1`"
+      echo "${GIT} checkout -f  `${GIT} rev-list origin/${BRANCH_NAME} -1`"
+      ${GIT} checkout -f  `${GIT} rev-list origin/${BRANCH_NAME} -1`
+      echo "${GIT} status"
+      ${GIT} status
+    fi
+  fi
+}
+
+###############################################################################
+###  Collect findbugs reports
+collectFindbugsReports() {
+  name=$1
+  basedir=$2
+  patch_dir=$3
+  for file in $(find $basedir -name findbugsXml.xml)
+  do
+    relative_file=${file#$basedir/} # strip leading $basedir prefix
+    if [ ! $relative_file == "target/findbugsXml.xml" ]; then
+      module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path
+      module_suffix=`basename ${module_suffix}`
+    fi
+
+    cp $file $patch_dir/${name}FindbugsWarnings${module_suffix}.xml
+    $FINDBUGS_HOME/bin/setBugDatabaseInfo -name $name \
+      $patch_dir/${name}FindbugsWarnings${module_suffix}.xml \
+      $patch_dir/${name}FindbugsWarnings${module_suffix}.xml
+  done
+  xml_file=$patch_dir/${name}FindbugsWarnings.xml
+  html_file=$patch_dir/${name}FindbugsWarnings.html
+  $FINDBUGS_HOME/bin/unionBugs -withMessages \
+	 -output $xml_file $patch_dir/${name}FindbugsWarnings*.xml
+  $FINDBUGS_HOME/bin/convertXmlToText -html $xml_file $html_file
+  file $xml_file $html_file
+}
+
 ###############################################################################
 setup () {
   ### Download latest patch file (ignoring .htm and .html) when run from patch process
@@ -205,16 +284,26 @@ setup () {
       echo "$defect is not \"Patch Available\".  Exiting."
       cleanupAndExit 0
     fi
-    relativePatchURL=`$GREP -o '"/jira/secure/attachment/[0-9]*/[^"]*' $PATCH_DIR/jira |
$GREP -v -e 'htm[l]*$' | sort | tail -1 | $GREP -o '/jira/secure/attachment/[0-9]*/[^"]*'`
+    relativePatchURL=`$GREP -o '"/jira/secure/attachment/[0-9]*/[^"]*' $PATCH_DIR/jira |
$EGREP '(\.txt$|\.patch$|\.diff$)' | sort | tail -1 | $GREP -o '/jira/secure/attachment/[0-9]*/[^"]*'`
     patchURL="http://issues.apache.org${relativePatchURL}"
     patchNum=`echo $patchURL | $GREP -o '[0-9]*/' | $GREP -o '[0-9]*'`
+    # ensure attachment has not already been tested
+    ATTACHMENT_ID=$(basename $(dirname $patchURL))
+    if grep -q "ATTACHMENT ID: $ATTACHMENT_ID" $PATCH_DIR/jira
+    then
+      echo "Attachment $ATTACHMENT_ID is already tested for $defect"
+      exit 1
+    fi
     echo "$defect patch is being downloaded at `date` from"
     echo "$patchURL"
     $WGET -q -O $PATCH_DIR/patch $patchURL
-    VERSION=${SVN_REVISION}_${defect}_PATCH-${patchNum}
+    VERSION=${GIT_COMMIT}_${defect}_PATCH-${patchNum}
+    findBranchNameFromPatchName ${relativePatchURL}
+    checkoutBranch
     JIRA_COMMENT="Here are the results of testing the latest attachment 
   $patchURL
-  against trunk revision ${SVN_REVISION}."
+  against ${BRANCH_NAME} branch at commit ${GIT_COMMIT}.
+  ATTACHMENT ID: ${ATTACHMENT_ID}"
 
   ### Copy the patch file to $PATCH_DIR
   else
@@ -227,11 +316,9 @@ setup () {
       cleanupAndExit 0
     fi
   fi
-  . $BASEDIR/dev-support/test-patch.properties
   ### exit if warnings are NOT defined in the properties file
-  if [ -z "$OK_FINDBUGS_WARNINGS" ] || [[ -z "$OK_JAVADOC_WARNINGS" ]] || [[ -z $OK_RELEASEAUDIT_WARNINGS
]] ; then
+  if [[ -z "$OK_JAVADOC_WARNINGS" ]] || [[ -z $OK_RELEASEAUDIT_WARNINGS ]] ; then
     echo "Please define the following properties in test-patch.properties file"
-	 echo  "OK_FINDBUGS_WARNINGS"
 	 echo  "OK_RELEASEAUDIT_WARNINGS"
 	 echo  "OK_JAVADOC_WARNINGS"
     cleanupAndExit 1
@@ -240,22 +327,28 @@ setup () {
   echo ""
   echo "======================================================================"
   echo "======================================================================"
-  echo " Pre-build trunk to verify trunk stability and javac warnings" 
+  echo " Pre-build master to verify stability and javac warnings"
   echo "======================================================================"
   echo "======================================================================"
   echo ""
   echo ""
-  echo "$MVN clean package checkstyle:checkstyle-aggregate -DskipTests -D${PROJECT_NAME}PatchProcess
> $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
+  echo "$MVN clean package checkstyle:checkstyle-aggregate findbugs:findbugs -DskipTests
\
+ -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
   export MAVEN_OPTS="${MAVEN_OPTS}"
   # build core and tests
-  $MVN clean package checkstyle:checkstyle-aggregate -DskipTests -D${PROJECT_NAME}PatchProcess
> $PATCH_DIR/trunkJavacWarnings.txt 2>&1
+  $MVN clean package checkstyle:checkstyle-aggregate findbugs:findbugs -DskipTests \
+	 -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1
   if [[ $? != 0 ]] ; then
+    echo "mvn exit code was $?"
     ERR=`$GREP -A 5 'Compilation failure' $PATCH_DIR/trunkJavacWarnings.txt`
-    echo "Trunk compilation is broken?
-    {code}$ERR{code}"
-    cleanupAndExit 1
+    if [[ ${#ERR} -ge 1 ]] ; then
+      echo "Trunk compilation is broken?
+      {code}$ERR{code}"
+      cleanupAndExit 1
+    fi
   fi
   mv target/checkstyle-result.xml $PATCH_DIR/trunkCheckstyle.xml
+  collectFindbugsReports trunk $BASEDIR $PATCH_DIR
 }
 
 ###############################################################################
@@ -309,6 +402,16 @@ checkTests () {
         return 0
       fi
     fi
+    srcReferences=`${GREP} "diff --git" "${PATCH_DIR}/patch" | ${GREP} "src/main" | \
+        ${GREP} -v "src/main/asciidoc" | ${GREP} -v "src/main/site" -c`
+    if [[ $srcReferences == 0 ]] ; then
+      echo "The patch doesn't appear to alter any code that requires tests."
+      JIRA_COMMENT="$JIRA_COMMENT
+
+    {color:green}+0 tests included{color}.  The patch appears to be a documentation, build,
+                        or dev-support patch that doesn't require tests."
+      return 0
+    fi
     JIRA_COMMENT="$JIRA_COMMENT
 
     {color:red}-1 tests included{color}.  The patch doesn't appear to include any new or
modified tests.
@@ -323,97 +426,81 @@ checkTests () {
 }
 
 ###############################################################################
-### Attempt to apply the patch
-applyPatch () {
-  echo ""
-  echo ""
-  echo "======================================================================"
-  echo "======================================================================"
-  echo "    Applying patch."
-  echo "======================================================================"
-  echo "======================================================================"
-  echo ""
-  echo ""
- 
-  export PATCH
-  $BASEDIR/dev-support/smart-apply-patch.sh $PATCH_DIR/patch
-  if [[ $? != 0 ]] ; then
-    echo "PATCH APPLICATION FAILED"
+### Check there are no compilation errors, passing a file to be parsed.
+checkCompilationErrors() {
+  local file=$1
+  hadoopVersion=""
+  if [ "$#" -ne 1 ]; then
+    hadoopVersion="with Hadoop version $2"
+  fi
+  COMPILATION_ERROR=false
+  eval $(awk '/ERROR/ {print "COMPILATION_ERROR=true"}' $file)
+  if $COMPILATION_ERROR ; then
+    ERRORS=$($AWK '/ERROR/ { print $0 }' $file)
+    echo "======================================================================"
+    echo "There are compilation errors $hadoopVersion."
+    echo "======================================================================"
+    echo "$ERRORS"
     JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:red}-1 patch{color}.  The patch command could not apply the patch."
-    return 1
+    {color:red}-1 javac{color}.  The patch appears to cause mvn compile goal to fail $hadoopVersion.
+
+    Compilation errors resume:
+    $ERRORS
+    "
+    submitJiraComment 1
+    cleanupAndExit 1
   fi
-  return 0
 }
 
 ###############################################################################
-### Attempt to compile against the hadoop 1.1
-checkHadoop20Compile () {
-  echo ""
-  echo ""
-  echo "======================================================================"
-  echo "======================================================================"
-  echo "    Checking against hadoop 1.1 build"
-  echo "======================================================================"
-  echo "======================================================================"
-  echo ""
-  echo ""
-
-  patchForHadoop2=`$GREP -c -i 'hadoop.profile=1.1' $PATCH_DIR/patch`
-  if [[ $patchForHadoop2 != 0 ]] ; then
-    return 0;
-  fi
-
-  export MAVEN_OPTS="${MAVEN_OPTS}"
-  # build core and tests
-  $MVN clean test help:active-profiles -X -DskipTests -Dhadoop.profile=1.1 -D${PROJECT_NAME}PatchProcess
> $PATCH_DIR/trunk1.1JavacWarnings.txt 2>&1
-  if [[ $? != 0 ]] ; then
+### Check there are no protoc compilation errors, passing a file to be parsed.
+checkProtocCompilationErrors() {
+  local file=$1
+  COMPILATION_ERROR=false
+  eval $(awk '/\[ERROR/ {print "COMPILATION_ERROR=true"}' $file)
+  if $COMPILATION_ERROR ; then
+    ERRORS=$($AWK '/\[ERROR/ { print $0 }' $file)
+    echo "======================================================================"
+    echo "There are Protoc compilation errors."
+    echo "======================================================================"
+    echo "$ERRORS"
     JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:red}-1 hadoop1.1{color}.  The patch failed to compile against the hadoop 1.1 profile."
-    submitJiraComment 1
+    {color:red}-1 javac{color}.  The patch appears to cause mvn compile-protobuf profile
to fail.
+
+    Protoc Compilation errors resume:
+    $ERRORS
+    "
     cleanupAndExit 1
   fi
-  JIRA_COMMENT="$JIRA_COMMENT
-
-    {color:green}+1 hadoop1.1{color}.  The patch compiles against the hadoop 1.1 profile."
-  return 0
 }
 
 ###############################################################################
-### Attempt to compile against the hadoop 1.0
-checkHadoop10Compile () {
+### Attempt to apply the patch
+applyPatch () {
   echo ""
   echo ""
   echo "======================================================================"
   echo "======================================================================"
-  echo "    Checking against hadoop 1.0 build"
+  echo "    Applying patch."
   echo "======================================================================"
   echo "======================================================================"
   echo ""
   echo ""
-
-  export MAVEN_OPTS="${MAVEN_OPTS}"
-  # build core and tests
-  $MVN clean test help:active-profiles -X -DskipTests -Dhadoop.profile=1.0 -D${PROJECT_NAME}PatchProcess
> $PATCH_DIR/trunk1.0JavacWarnings.txt 2>&1
+ 
+  export PATCH
+  $BASEDIR/dev-support/smart-apply-patch.sh $PATCH_DIR/patch
   if [[ $? != 0 ]] ; then
-    ERR=`$GREP -A 5 'Compilation failure' $PATCH_DIR/trunk1.0JavacWarnings.txt`
+    echo "PATCH APPLICATION FAILED"
     JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:red}-1 hadoop1.0{color}.  The patch failed to compile against the hadoop 1.0 profile.
-    Here is snippet of errors:
-    {code}$ERR{code}"
-    submitJiraComment 1
-    cleanupAndExit 1
+    {color:red}-1 patch{color}.  The patch command could not apply the patch."
+    return 1
   fi
-  JIRA_COMMENT="$JIRA_COMMENT
-
-    {color:green}+1 hadoop1.0{color}.  The patch compiles against the hadoop 1.0 profile."
   return 0
 }
 
-
 ###############################################################################
 ### Check against known anti-patterns
 checkAntiPatterns () {
@@ -430,9 +517,8 @@ checkAntiPatterns () {
   if [[ $warnings != "" ]]; then
     JIRA_COMMENT="$JIRA_COMMENT
 
-		    {color:red}-1 Anti-pattern{color}.  The patch appears to have anti-pattern where BYTES_COMPARATOR
was omitted:
-             $warnings."
-	  return 1
+    {color:red}-1 Anti-pattern{color}.  The patch appears to have anti-pattern where BYTES_COMPARATOR
was omitted: $warnings."
+    return 1
   fi
   return 0
 }
@@ -453,9 +539,8 @@ checkInterfaceAudience () {
   if [[ $warnings != "" ]]; then
     JIRA_COMMENT="$JIRA_COMMENT
 
-		    {color:red}-1 InterfaceAudience{color}.  The patch appears to contain InterfaceAudience
from hadoop rather than hbase:
-             $warnings."
-	  return 1
+    {color:red}-1 InterfaceAudience{color}.  The patch appears to contain InterfaceAudience
from hadoop rather than hbase: $warnings."
+    return 1
   fi
   return 0
 }
@@ -485,6 +570,9 @@ checkJavadocWarnings () {
     JIRA_COMMENT="$JIRA_COMMENT
 
     {color:red}-1 javadoc{color}.  The javadoc tool appears to have generated `expr $(($javadocWarnings-$OK_JAVADOC_WARNINGS))`
warning messages."
+    # Add javadoc output url
+    JIRA_COMMENT_FOOTER="Javadoc warnings: $BUILD_URL/artifact/patchprocess/patchJavadocWarnings.txt
+$JIRA_COMMENT_FOOTER"
     return 1
   fi
   JIRA_COMMENT="$JIRA_COMMENT
@@ -493,6 +581,31 @@ checkJavadocWarnings () {
   return 0
 }
 
+checkBuildWithHadoopVersions() {
+  echo ""
+  echo ""
+  echo "======================================================================"
+  echo "======================================================================"
+  echo "    Building with all supported Hadoop versions ."
+  echo "======================================================================"
+  echo "======================================================================"
+  echo ""
+  echo ""
+  export MAVEN_OPTS="${MAVEN_OPTS}"
+  for HADOOP2_VERSION in $HADOOP2_VERSIONS ; do
+    echo "$MVN clean install -DskipTests -D${PROJECT_NAME}PatchProcess -Dhadoop-two.version=$HADOOP2_VERSION
> $PATCH_DIR/patchJavacWithHadoop-$HADOOP2_VERSION.txt 2>&1"
+    $MVN clean install -DskipTests -D${PROJECT_NAME}PatchProcess -Dhadoop-two.version=$HADOOP2_VERSION
> $PATCH_DIR/patchJavacWithHadoop-$HADOOP2_VERSION.txt 2>&1
+    checkCompilationErrors $PATCH_DIR/patchJavacWithHadoop-$HADOOP2_VERSION.txt $HADOOP2_VERSION
+  done
+
+  # TODO: add Hadoop3 versions and compilation here when we get the hadoop.profile=3.0 working
+
+  JIRA_COMMENT="$JIRA_COMMENT
+
+    {color:green}+1 hadoop versions{color}. The patch compiles with all supported hadoop
versions ($HADOOP2_VERSIONS)"
+  return 0
+}
+
 ###############################################################################
 ### Check there are no changes in the number of Javac warnings
 checkJavacWarnings () {
@@ -508,12 +621,7 @@ checkJavacWarnings () {
   echo "$MVN clean package -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavacWarnings.txt
2>&1"
   export MAVEN_OPTS="${MAVEN_OPTS}"
   $MVN clean package -DskipTests -D${PROJECT_NAME}PatchProcess  > $PATCH_DIR/patchJavacWarnings.txt
2>&1
-  if [[ $? != 0 ]] ; then
-    JIRA_COMMENT="$JIRA_COMMENT
-
-    {color:red}-1 javac{color}.  The patch appears to cause mvn compile goal to fail."
-    return 1
-  fi
+  checkCompilationErrors $PATCH_DIR/patchJavacWarnings.txt
   ### Compare trunk and patch javac warning numbers
   if [[ -f $PATCH_DIR/patchJavacWarnings.txt ]] ; then
     trunkJavacWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/trunkJavacWarnings.txt | $AWK 'BEGIN
{total = 0} {total += 1} END {print total}'`
@@ -523,7 +631,7 @@ checkJavacWarnings () {
       if [[ $patchJavacWarnings -gt $trunkJavacWarnings ]] ; then
         JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:red}-1 javac{color}.  The applied patch generated $patchJavacWarnings javac compiler
warnings (more than the trunk's current $trunkJavacWarnings warnings)."
+    {color:red}-1 javac{color}.  The applied patch generated $patchJavacWarnings javac compiler
warnings (more than the master's current $trunkJavacWarnings warnings)."
         return 1
       fi
     fi
@@ -549,27 +657,50 @@ checkCheckstyleErrors() {
     mv target/checkstyle-result.xml $PATCH_DIR/patchCheckstyle.xml
     mv target/site/checkstyle-aggregate.html $PATCH_DIR
     mv target/site/checkstyle.css $PATCH_DIR
-    trunkCheckstyleErrors=`$GREP '<error' $PATCH_DIR/trunkCheckstyle.xml | $AWK 'BEGIN
{total = 0} {total += 1} END {print total}'`
-    patchCheckstyleErrors=`$GREP '<error' $PATCH_DIR/patchCheckstyle.xml | $AWK 'BEGIN
{total = 0} {total += 1} END {print total}'`
-    if [[ $patchCheckstyleErrors -gt $trunkCheckstyleErrors ]] ; then
+    $BASEDIR/dev-support/checkstyle_report.py $PATCH_DIR/trunkCheckstyle.xml $PATCH_DIR/patchCheckstyle.xml
+    if [[ $? -eq 1 ]] ; then
                 JIRA_COMMENT_FOOTER="Checkstyle Errors: $BUILD_URL/artifact/patchprocess/checkstyle-aggregate.html
 
                 $JIRA_COMMENT_FOOTER"
 
                 JIRA_COMMENT="$JIRA_COMMENT
 
-                {color:red}-1 javac{color}.  The applied patch generated $patchCheckstyleErrors
checkstyle errors (more than the trunk's current $trunkCheckstyleErrors errors)."
+                {color:red}-1 checkstyle{color}.  The applied patch generated new checkstyle
errors. Check build console for list of new errors."
         return 1
     fi
-    echo "There were $patchCheckstyleErrors checkstyle errors in this patch compared to $trunkCheckstyleErrors
on master."
   fi
+  JIRA_COMMENT_FOOTER="Checkstyle Errors: $BUILD_URL/artifact/patchprocess/checkstyle-aggregate.html
+
+  $JIRA_COMMENT_FOOTER"
+
   JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:green}+1 javac{color}.  The applied patch does not increase the total number of
checkstyle errors"
+    {color:green}+1 checkstyle{color}. The applied patch does not generate new checkstyle
errors."
   return 0
 
 }
 ###############################################################################
+checkProtocErrors () {
+  echo ""
+  echo ""
+  echo "======================================================================"
+  echo "======================================================================"
+  echo "    Determining whether there is patched protoc error."
+  echo "======================================================================"
+  echo "======================================================================"
+  echo ""
+  echo ""
+  echo "$MVN clean install -DskipTests -Pcompile-protobuf -X -D${PROJECT_NAME}PatchProcess
> $PATCH_DIR/patchProtocErrors.txt 2>&1"
+  export MAVEN_OPTS="${MAVEN_OPTS}"
+  $MVN clean install -DskipTests -Pcompile-protobuf -X -D${PROJECT_NAME}PatchProcess  >
$PATCH_DIR/patchProtocErrors.txt 2>&1
+  checkProtocCompilationErrors $PATCH_DIR/patchProtocErrors.txt
+  JIRA_COMMENT="$JIRA_COMMENT
+
+    {color:green}+1 protoc{color}.  The applied patch does not increase the total number
of protoc compiler warnings."
+  return 0
+}
+
+###############################################################################
 ### Check there are no changes in the number of release audit (RAT) warnings
 checkReleaseAuditWarnings () {
   echo ""
@@ -596,7 +727,7 @@ checkReleaseAuditWarnings () {
       if [[ $patchReleaseAuditWarnings -gt $OK_RELEASEAUDIT_WARNINGS ]] ; then
         JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:red}-1 release audit{color}.  The applied patch generated $patchReleaseAuditWarnings
release audit warnings (more than the trunk's current $OK_RELEASEAUDIT_WARNINGS warnings)."
+    {color:red}-1 release audit{color}.  The applied patch generated $patchReleaseAuditWarnings
release audit warnings (more than the master's current $OK_RELEASEAUDIT_WARNINGS warnings)."
         $GREP '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt > $PATCH_DIR/patchReleaseAuditProblems.txt
         echo "Lines that start with ????? in the release audit report indicate files that
do not have an Apache license header." >> $PATCH_DIR/patchReleaseAuditProblems.txt
         JIRA_COMMENT_FOOTER="Release audit warnings: $BUILD_URL/artifact/patchprocess/patchReleaseAuditWarnings.txt
@@ -634,41 +765,36 @@ checkFindbugsWarnings () {
     {color:red}-1 findbugs{color}.  The patch appears to cause Findbugs (version ${findbugs_version})
to fail."
     return 1
   fi
-    
-  findbugsWarnings=0
-  for file in $(find $BASEDIR -name findbugsXml.xml)
-  do
-    relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix
-    if [ ! $relative_file == "target/findbugsXml.xml" ]; then
-      module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path
-      module_suffix=`basename ${module_suffix}`
-    fi
-    
-    cp $file $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml
-    $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \
-      $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \
-      $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml
-    newFindbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml
\
-      $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml | $AWK '{print $1}'`
-    echo "Found $newFindbugsWarnings Findbugs warnings ($file)"
-    findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings))
-    $FINDBUGS_HOME/bin/convertXmlToText -html \
-      $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \
-      $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html
-    JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/trunk/patchprocess/newPatchFindbugsWarnings${module_suffix}.html
-$JIRA_COMMENT_FOOTER"
-  done
 
-  ### if current warnings greater than OK_FINDBUGS_WARNINGS
-  if [[ $findbugsWarnings -gt $OK_FINDBUGS_WARNINGS ]] ; then
+  collectFindbugsReports patch $BASEDIR $PATCH_DIR 
+  #this files are generated by collectFindbugsReports() named with its first argument
+  patch_xml=$PATCH_DIR/patchFindbugsWarnings.xml
+  trunk_xml=$PATCH_DIR/trunkFindbugsWarnings.xml
+  # combine them to one database
+  combined_xml=$PATCH_DIR/combinedFindbugsWarnings.xml
+  new_xml=$PATCH_DIR/newFindbugsWarnings.xml
+  new_html=$PATCH_DIR/newFindbugsWarnings.html
+  $FINDBUGS_HOME/bin/computeBugHistory -useAnalysisTimes -withMessages \
+	-output $combined_xml $trunk_xml $patch_xml
+  findbugsWarnings=$($FINDBUGS_HOME/bin/filterBugs -first patch $combined_xml $new_xml)
+  findbugsFixedWarnings=$($FINDBUGS_HOME/bin/filterBugs -fixed patch $combined_xml $new_xml)
+  $FINDBUGS_HOME/bin/convertXmlToText -html  $new_xml $new_html
+  file $new_xml $new_html
+  JIRA_COMMENT_FOOTER="Release Findbugs (version ${findbugs_version}) \
+	warnings: $BUILD_URL/artifact/patchprocess/newFindbugsWarnings.html
+$JIRA_COMMENT_FOOTER"
+  ### if current warnings greater than 0, fail
+  if [[ $findbugsWarnings -gt 0 ]] ; then
     JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:red}-1 findbugs{color}.  The patch appears to introduce `expr $(($findbugsWarnings-$OK_FINDBUGS_WARNINGS))`
new Findbugs (version ${findbugs_version}) warnings."
+    {color:red}-1 findbugs{color}.  The patch appears to introduce $findbugsWarnings \
+ new Findbugs (version ${findbugs_version}) warnings."
     return 1
   fi
   JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:green}+1 findbugs{color}.  The patch does not introduce any new Findbugs (version
${findbugs_version}) warnings."
+    {color:green}+1 findbugs{color}.  The patch does not introduce any \
+ new Findbugs (version ${findbugs_version}) warnings."
   return 0
 }
 
@@ -687,9 +813,9 @@ checkLineLengths () {
   #see http://en.wikipedia.org/wiki/Diff#Unified_format
 
   MAX_LINE_LENGTH_PATCH=`expr $MAX_LINE_LENGTH + 1`
-  lines=`cat $PATCH_DIR/patch | grep "^+" | grep -v "^@@" | grep -v "^+++" | grep -v "import"
| grep -v "hbase.protobuf.generated" | awk -v len="$MAX_LINE_LENGTH_PATCH" 'length ($0) >
len' | head -n 10`
+  lines=`cat $PATCH_DIR/patch | grep "^+" | grep -v "^@@" | grep -v "^+++" | grep -v "import"
| grep -v "org.apache.thrift." | grep -v "com.google.protobuf." | grep -v "protobuf.generated"
| awk -v len="$MAX_LINE_LENGTH_PATCH" 'length ($0) > len' | head -n 10`
   ll=`echo "$lines" | wc -l`
-  if [[ "$ll" -gt "$MAX_LINE_LENGTH_PATCH" ]]; then
+  if [[ "$ll" -gt "1" ]]; then
     JIRA_COMMENT="$JIRA_COMMENT
 
     {color:red}-1 lineLengths{color}.  The patch introduces the following lines longer than
$MAX_LINE_LENGTH:
@@ -715,59 +841,30 @@ runTests () {
   echo "======================================================================"
   echo ""
   echo ""
-
-
-  ### kill any process remaining from another test, maybe even another project
-  jps | grep surefirebooter | cut -d ' ' -f 1 | xargs kill -9 2>/dev/null
-  
   failed_tests=""
-  ### Kill any rogue build processes from the last attempt
-  condemnedCount=`$PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | $AWK
'BEGIN {total = 0} {total += 1} END {print total}'`
-  echo "WARNING: $condemnedCount rogue build processes detected, terminating."
-  $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t
-I {} /bin/kill -9 {} > /dev/null
-  echo "$MVN clean test -P runAllTests -D${PROJECT_NAME}PatchProcess"
+  echo "$MVN clean test -Dsurefire.rerunFailingTestsCount=2 -P runAllTests -D${PROJECT_NAME}PatchProcess"
   export MAVEN_OPTS="${MAVEN_OPTS}"
   ulimit -a
-  $MVN clean test -P runAllTests -D${PROJECT_NAME}PatchProcess
+  # Need to export this so the zombie subshell picks up current content
+  export JIRA_COMMENT
+  $MVN clean test -Dsurefire.rerunFailingTestsCount=2 -P runAllTests -D${PROJECT_NAME}PatchProcess
   if [[ $? != 0 ]] ; then
      ### Find and format names of failed tests
      failed_tests=`find . -name 'TEST*.xml' | xargs $GREP  -l -E "<failure|<error"
| sed -e "s|.*target/surefire-reports/TEST-|                  |g" | sed -e "s|\.xml||g"`
  
      JIRA_COMMENT="$JIRA_COMMENT
 
-     {color:red}-1 core tests{color}.  The patch failed these unit tests:
+    {color:red}-1 core tests{color}.  The patch failed these unit tests:
      $failed_tests"
      BAD=1
+     JIRA_COMMENT=`$BASEDIR/dev-support/zombie-detector.sh ${BUILD_ID}`
   else
     JIRA_COMMENT="$JIRA_COMMENT
 
     {color:green}+1 core tests{color}.  The patch passed unit tests in $modules."
-    BAD=0
+     JIRA_COMMENT=`$BASEDIR/dev-support/zombie-detector.sh ${BUILD_ID}`
+    BAD=$?
   fi
-  ZOMBIE_TESTS_COUNT=`jps | grep surefirebooter | wc -l`
-  if [[ $ZOMBIE_TESTS_COUNT != 0 ]] ; then
-    #It seems sometimes the tests are not dying immediately. Let's give them 30s
-    echo "Suspicious java process found - waiting 30s to see if there are just slow to stop"
-    sleep 30
-    ZOMBIE_TESTS_COUNT=`jps | grep surefirebooter | wc -l`
-    if [[ $ZOMBIE_TESTS_COUNT != 0 ]] ; then
-      echo "There are $ZOMBIE_TESTS_COUNT zombie tests, they should have been killed by surefire
but survived"
-      echo "************ BEGIN zombies jstack extract"
-      ZB_STACK=`jps | grep surefirebooter | cut -d ' ' -f 1 | xargs -n 1 jstack | grep ".test"
| grep "\.java"`
-      jps | grep surefirebooter | cut -d ' ' -f 1 | xargs -n 1 jstack
-      echo "************ END  zombies jstack extract"
-      JIRA_COMMENT="$JIRA_COMMENT
-
-     {color:red}-1 core zombie tests{color}.  There are ${ZOMBIE_TESTS_COUNT} zombie test(s):
${ZB_STACK}"
-      BAD=1
-      jps | grep surefirebooter | cut -d ' ' -f 1 | xargs kill -9
-    else
-      echo "We're ok: there is no zombie test, but some tests took some time to stop"
-    fi
-  else
-    echo "We're ok: there is no zombie test"
-  fi
-  return $BAD
 }
 
 ###############################################################################
@@ -783,18 +880,18 @@ checkSiteXml () {
   echo ""
   echo ""
 
-  echo "$MVN package site -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchSiteOutput.txt
2>&1"
+  echo "$MVN package post-site -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchSiteOutput.txt
2>&1"
   export MAVEN_OPTS="${MAVEN_OPTS}"
-  $MVN package site -DskipTests -D${PROJECT_NAME}PatchProcess  > $PATCH_DIR/patchSiteOutput.txt
2>&1
+  $MVN package post-site -DskipTests -D${PROJECT_NAME}PatchProcess  > $PATCH_DIR/patchSiteOutput.txt
2>&1
   if [[ $? != 0 ]] ; then
     JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:red}-1 site{color}.  The patch appears to cause mvn site goal to fail."
+    {color:red}-1 site{color}.  The patch appears to cause mvn post-site goal to fail."
     return 1
   fi
   JIRA_COMMENT="$JIRA_COMMENT
 
-  {color:green}+1 site{color}.  The mvn site goal succeeds with this patch."
+    {color:green}+1 site{color}.  The mvn post-site goal succeeds with this patch."
   return 0
 }
 
@@ -875,8 +972,9 @@ $comment"
 ### Cleanup files
 cleanupAndExit () {
   local result=$1
-  if [[ $JENKINS == "true" ]] ; then
+  if [[ ${JENKINS} == "true" && ${MOVE_PATCH_DIR} == "true" ]] ; then
     if [ -e "$PATCH_DIR" ] ; then
+      echo "Relocating patch dir into ${BASEDIR}"
       mv $PATCH_DIR $BASEDIR
     fi
   fi
@@ -905,8 +1003,10 @@ This message is automatically generated."
 parseArgs $@
 cd $BASEDIR
 
+echo "Version of this script: Wed Oct 14 00:29:04 PDT 2015"
 checkout
 RESULT=$?
+echo "RESULT = " $RESULT
 if [[ $JENKINS == "true" ]] ; then
   if [[ $RESULT != 0 ]] ; then
     exit 100
@@ -915,8 +1015,10 @@ fi
 setup
 checkAuthor
 RESULT=$?
+echo "RESULT = " $RESULT
 checkTests
 (( RESULT = RESULT + $? ))
+echo "RESULT = " $RESULT
 applyPatch
 if [[ $? != 0 ]] ; then
   submitJiraComment 1
@@ -925,30 +1027,42 @@ fi
 
 checkAntiPatterns
 (( RESULT = RESULT + $? ))
-checkHadoop10Compile
+echo "RESULT = " $RESULT
+checkBuildWithHadoopVersions
 (( RESULT = RESULT + $? ))
-checkHadoop20Compile
+echo "RESULT = " $RESULT
+checkJavacWarnings
 (( RESULT = RESULT + $? ))
-checkJavadocWarnings
+echo "RESULT = " $RESULT
+checkProtocErrors
 (( RESULT = RESULT + $? ))
-checkJavacWarnings
+echo "RESULT = " $RESULT
+checkJavadocWarnings
 (( RESULT = RESULT + $? ))
+echo "RESULT = " $RESULT
 checkCheckstyleErrors
 (( RESULT = RESULT + $? ))
+echo "RESULT = " $RESULT
 checkInterfaceAudience
 (( RESULT = RESULT + $? ))
+echo "RESULT = " $RESULT
 checkFindbugsWarnings
 (( RESULT = RESULT + $? ))
+echo "RESULT = " $RESULT
 checkReleaseAuditWarnings
 (( RESULT = RESULT + $? ))
+echo "RESULT = " $RESULT
 checkLineLengths
 (( RESULT = RESULT + $? ))
+echo "RESULT = " $RESULT
 checkSiteXml
 (( RESULT = RESULT + $?))
-### Do not call these when run by a developer 
+echo "RESULT = " $RESULT
+### Do not call these when run by a developer
 if [[ $JENKINS == "true" ]] ; then
   runTests
   (( RESULT = RESULT + $? ))
+  echo "RESULT = " $RESULT
 JIRA_COMMENT_FOOTER="Test results: $BUILD_URL/testReport/
 $JIRA_COMMENT_FOOTER"
 fi

http://git-wip-us.apache.org/repos/asf/hbase/blob/d93b4372/dev-support/zombie-detector.sh
----------------------------------------------------------------------
diff --git a/dev-support/zombie-detector.sh b/dev-support/zombie-detector.sh
new file mode 100644
index 0000000..df4c197
--- /dev/null
+++ b/dev-support/zombie-detector.sh
@@ -0,0 +1,166 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Looks for any running zombies left over from old build runs.
+# Will report and try to do stack trace on stale processes so can
+# figure how they are hung. Echos state as the script runs
+# on STDERR but prints final output on STDOUT formatted so it
+# will fold into the test result formatting done by test-patch.sh.
+# This script is called from test-patch.sh but also after tests
+# have run up on builds.apache.org.
+
+# TODO: format output to suit context -- test-patch, jenkins or dev env
+
+#set -x
+# printenv
+
+### Setup some variables.  
+bindir=$(dirname $0)
+
+# This key is set by our surefire configuration up in the main pom.xml
+# This key needs to match the key we set up there.
+HBASE_BUILD_ID_KEY="hbase.build.id="
+JENKINS=
+
+PS=${PS:-ps}
+AWK=${AWK:-awk}
+WGET=${WGET:-wget}
+GREP=${GREP:-grep}
+JIRACLI=${JIRA:-jira}
+
+###############################################################################
+printUsage() {
+  echo "Usage: $0 [options]" BUILD_ID
+  echo
+  echo "Where:"
+  echo "  BUILD_ID is build id to look for in process listing"
+  echo
+  echo "Options:"
+  echo "--ps-cmd=<cmd>         The 'ps' command to use (default 'ps')"
+  echo "--awk-cmd=<cmd>        The 'awk' command to use (default 'awk')"
+  echo "--grep-cmd=<cmd>       The 'grep' command to use (default 'grep')"
+  echo
+  echo "Jenkins-only options:"
+  echo "--jenkins              Run by Jenkins (runs tests and posts results to JIRA)"
+  echo "--wget-cmd=<cmd>       The 'wget' command to use (default 'wget')"
+  echo "--jira-cmd=<cmd>       The 'jira' command to use (default 'jira')"
+}
+
+###############################################################################
+parseArgs() {
+  for i in $*
+  do
+    case $i in
+    --jenkins)
+      JENKINS=true
+      ;;
+    --ps-cmd=*)
+      PS=${i#*=}
+      ;;
+    --awk-cmd=*)
+      AWK=${i#*=}
+      ;;
+    --wget-cmd=*)
+      WGET=${i#*=}
+      ;;
+    --grep-cmd=*)
+      GREP=${i#*=}
+      ;;
+    --jira-cmd=*)
+      JIRACLI=${i#*=}
+      ;;
+    *)
+      BUILD_ID=$i
+      ;;
+    esac
+  done
+  if [ -z "$BUILD_ID" ]; then
+    printUsage
+    exit 1
+  fi
+}
+
+### Return list of the processes found with passed build id.
+find_processes () {
+  jps -v | grep surefirebooter | grep -e "${HBASE_BUILD_TAG}"
+}
+
+### Look for zombies
+zombies () {
+  ZOMBIES=`find_processes`
+  if [[ -z ${ZOMBIES} ]]
+  then
+    ZOMBIE_TESTS_COUNT=0
+  else
+    ZOMBIE_TESTS_COUNT=`echo "${ZOMBIES}"| wc -l| xargs`
+  fi
+  if [[ $ZOMBIE_TESTS_COUNT != 0 ]] ; then
+    wait=30
+    echo "`date` Found ${ZOMBIE_TESTS_COUNT} suspicious java process(es) listed below; waiting
${wait}s to see if just slow to stop" >&2
+    echo ${ZOMBIES} >&2
+    sleep ${wait}
+    PIDS=`echo "${ZOMBIES}"|${AWK} '{print $1}'`
+    ZOMBIE_TESTS_COUNT=0
+    for pid in $PIDS
+    do
+      # Test our zombie still running (and that it still an hbase build item)
+      PS_OUTPUT=`ps -p $pid | tail +2 | grep -e "${HBASE_BUILD_TAG}"`
+      if [[ ! -z "${PS_OUTPUT}" ]]
+      then
+        echo "`date` Zombie: $PS_OUTPUT" >&2
+        let "ZOMBIE_TESTS_COUNT+=1"
+        PS_STACK=`jstack $pid | grep -e "\.Test" | grep -e "\.java"| head -3`
+        echo "${PS_STACK}" >&2
+        ZB_STACK="${ZB_STACK}\nPID=${pid} ${PS_STACK}"
+      fi
+    done
+    if [[ $ZOMBIE_TESTS_COUNT != 0 ]]
+    then
+      echo "`date` There are ${ZOMBIE_TESTS_COUNT} possible zombie test(s)." >&2
+      # If JIRA_COMMENT in environment, append our findings to it
+      echo -e "$JIRA_COMMENT
+
+    {color:red}+1 zombies{red}. There are ${ZOMBIE_TESTS_COUNT} possible zombie test(s)
+        ${ZB_STACK}"
+      # Exit with exit code of 1.
+      exit 1
+    else
+      echo "`date` We're ok: there was a zombie candidate but it went away" >&2
+      echo "$JIRA_COMMENT
+
+    {color:green}+1 zombies{color}. No zombie tests found running at the end of the build
(There were candidates but they seem to have gone away)."
+    fi
+  else
+      echo "`date` We're ok: there is no zombie test" >&2
+      echo "$JIRA_COMMENT
+
+    {color:green}+1 zombies{color}. No zombie tests found running at the end of the build."
+  fi
+}
+
+### Check if arguments to the script have been specified properly or not
+parseArgs $@
+HBASE_BUILD_TAG="${HBASE_BUILD_ID_KEY}${BUILD_ID}"
+zombies
+RESULT=$?
+if [[ $JENKINS == "true" ]] ; then
+  if [[ $RESULT != 0 ]] ; then
+    exit 100
+  fi
+fi
+RESULT=$?


Mime
View raw message