Return-Path: 
 <commits-return-30509-apmail-hbase-commits-archive=hbase.apache.org@hbase.apache.org>
X-Original-To: apmail-hbase-commits-archive@www.apache.org
Delivered-To: apmail-hbase-commits-archive@www.apache.org
Received: from mail.apache.org (hermes.apache.org [140.211.11.3])
	by minotaur.apache.org (Postfix) with SMTP id 48F921832C
	for <apmail-hbase-commits-archive@www.apache.org>;
 Wed, 23 Dec 2015 03:54:46 +0000 (UTC)
Received: (qmail 33491 invoked by uid 500); 23 Dec 2015 03:54:46 -0000
Delivered-To: apmail-hbase-commits-archive@hbase.apache.org
Received: (qmail 33452 invoked by uid 500); 23 Dec 2015 03:54:46 -0000
Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:commits-help@hbase.apache.org>
List-Unsubscribe: <mailto:commits-unsubscribe@hbase.apache.org>
List-Post: <mailto:commits@hbase.apache.org>
List-Id: <commits.hbase.apache.org>
Reply-To: dev@hbase.apache.org
Delivered-To: mailing list commits@hbase.apache.org
Received: (qmail 33442 invoked by uid 99); 23 Dec 2015 03:54:46 -0000
Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org)
 (140.211.11.23)
    by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 23 Dec 2015 03:54:46 +0000
Received: by git1-us-west.apache.org (ASF Mail Server at
 git1-us-west.apache.org, from userid 33)
	id CE8A5E0663; Wed, 23 Dec 2015 03:54:45 +0000 (UTC)
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: stack@apache.org
To: commits@hbase.apache.org
Message-Id: <80da5b7125c9458f97146a94801c4dd7@git.apache.org>
X-Mailer: ASF-Git Admin Mailer
Subject: hbase git commit: HBASE-15021 hadoopqa doing false positives
Date: Wed, 23 Dec 2015 03:54:45 +0000 (UTC)

Repository: hbase
Updated Branches:
  refs/heads/branch-1.0 55eb15111 -> f02a9fa6a


HBASE-15021 hadoopqa doing false positives


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f02a9fa6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f02a9fa6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f02a9fa6

Branch: refs/heads/branch-1.0
Commit: f02a9fa6a02b0ea98c4d2a183c70016b678e34bd
Parents: 55eb151
Author: stack <stack@apache.org>
Authored: Tue Dec 22 19:54:21 2015 -0800
Committer: stack <stack@apache.org>
Committed: Tue Dec 22 19:54:21 2015 -0800

----------------------------------------------------------------------
 dev-support/test-patch.sh      | 346 ++++++++++++++++++++++++------------
 dev-support/zombie-detector.sh | 166 +++++++++++++++++
 2 files changed, 395 insertions(+), 117 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/f02a9fa6/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index e6a377a..509edeb 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -20,7 +20,7 @@
 #set -x
 
 ### Setup some variables.  
-### SVN_REVISION and BUILD_URL are set by Hudson if it is run by patch process
+### GIT_COMMIT and BUILD_URL are set by Hudson if it is run by patch process
 ### Read variables from properties file
 bindir=$(dirname $0)
 
@@ -31,15 +31,20 @@ else
   MVN=$MAVEN_HOME/bin/mvn
 fi
 
+NEWLINE=$'\n'
+
 PROJECT_NAME=HBase
 JENKINS=false
+MOVE_PATCH_DIR=true
 PATCH_DIR=/tmp
 BASEDIR=$(pwd)
+BRANCH_NAME="master"
+
+. $BASEDIR/dev-support/test-patch.properties
 
 PS=${PS:-ps}
 AWK=${AWK:-awk}
 WGET=${WGET:-wget}
-SVN=${SVN:-svn}
 GREP=${GREP:-grep}
 EGREP=${EGREP:-egrep}
 PATCH=${PATCH:-patch}
@@ -47,6 +52,7 @@ JIRACLI=${JIRA:-jira}
 FINDBUGS_HOME=${FINDBUGS_HOME}
 FORREST_HOME=${FORREST_HOME}
 ECLIPSE_HOME=${ECLIPSE_HOME}
+GIT=${GIT:-git}
 
 ###############################################################################
 printUsage() {
@@ -62,12 +68,12 @@ printUsage() {
   echo "--mvn-cmd=<cmd>        The 'mvn' command to use (default \$MAVEN_HOME/bin/mvn, or 'mvn')"
   echo "--ps-cmd=<cmd>         The 'ps' command to use (default 'ps')"
   echo "--awk-cmd=<cmd>        The 'awk' command to use (default 'awk')"
-  echo "--svn-cmd=<cmd>        The 'svn' command to use (default 'svn')"
   echo "--grep-cmd=<cmd>       The 'grep' command to use (default 'grep')"
   echo "--patch-cmd=<cmd>      The 'patch' command to use (default 'patch')"
   echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
   echo "--forrest-home=<path>  Forrest home directory (default FORREST_HOME environment variable)"
-  echo "--dirty-workspace      Allow the local SVN workspace to have uncommitted changes"
+  echo "--dirty-workspace      Allow the local workspace to have uncommitted changes"
+  echo "--git-cmd=<cmd>        The 'git' command to use (default 'git')"
   echo
   echo "Jenkins-only options:"
   echo "--jenkins              Run by Jenkins (runs tests and posts results to JIRA)"
@@ -85,6 +91,9 @@ parseArgs() {
     --jenkins)
       JENKINS=true
       ;;
+    --no-move-patch-dir)
+      MOVE_PATCH_DIR=false
+      ;;
     --patch-dir=*)
       PATCH_DIR=${i#*=}
       ;;
@@ -103,9 +112,6 @@ parseArgs() {
     --wget-cmd=*)
       WGET=${i#*=}
       ;;
-    --svn-cmd=*)
-      SVN=${i#*=}
-      ;;
     --grep-cmd=*)
       GREP=${i#*=}
       ;;
@@ -130,6 +136,9 @@ parseArgs() {
     --dirty-workspace)
       DIRTY_WORKSPACE=true
       ;;
+    --git-cmd=*)
+      GIT=${i#*=}
+      ;;
     *)
       PATCH_OR_DEFECT=$i
       ;;
@@ -180,23 +189,92 @@ checkout () {
   echo ""
   ### When run by a developer, if the workspace contains modifications, do not continue
   ### unless the --dirty-workspace option was set
-  status=`$SVN stat --ignore-externals | sed -e '/^X[ ]*/D'`
   if [[ $JENKINS == "false" ]] ; then
-    if [[ "$status" != "" && -z $DIRTY_WORKSPACE ]] ; then
-      echo "ERROR: can't run in a workspace that contains the following modifications"
-      echo "$status"
-      cleanupAndExit 1
+    if [[ -z $DIRTY_WORKSPACE ]] ; then
+      # Ref http://stackoverflow.com/a/2659808 for details on checking dirty status
+      ${GIT} diff-index --quiet HEAD
+      if [[ $? -ne 0 ]] ; then
+        uncommitted=`${GIT} diff --name-only HEAD`
+        uncommitted="You have the following files with uncommitted changes:${NEWLINE}${uncommitted}"
+      fi
+      untracked="$(${GIT} ls-files --exclude-standard --others)" && test -z "${untracked}"
+      if [[ $? -ne 0 ]] ; then
+        untracked="You have untracked and unignored files:${NEWLINE}${untracked}"
+      fi
+      if [[ $uncommitted || $untracked ]] ; then
+        echo "ERROR: can't run in a workspace that contains modifications."
+        echo "Pass the '--dirty-workspace' flag to bypass."
+        echo ""
+        echo "${uncommitted}"
+        echo ""
+        echo "${untracked}"
+        cleanupAndExit 1
+      fi
     fi
     echo
-  else   
-    cd $BASEDIR
-    $SVN revert -R .
-    rm -rf `$SVN status --no-ignore`
-    $SVN update
   fi
   return $?
 }
 
+findBranchNameFromPatchName() {
+  local patchName=$1
+  for LOCAL_BRANCH_NAME in $BRANCH_NAMES; do
+    if [[ $patchName =~ /jira/secure/attachment/[0-9]*/.*$LOCAL_BRANCH_NAME ]]; then
+      BRANCH_NAME=$LOCAL_BRANCH_NAME
+      break
+    fi
+  done
+  return 0
+}
+
+checkoutBranch() {
+  echo ""
+  echo ""
+  echo "======================================================================"
+  echo "======================================================================"
+  echo "    Testing patch on branch ${BRANCH_NAME}."
+  echo "======================================================================"
+  echo "======================================================================"
+  echo ""
+  echo ""
+  if [[ $JENKINS == "true" ]] ; then
+    if [[ "$BRANCH_NAME" != "master" ]]; then
+      echo "origin/${BRANCH_NAME} HEAD is commit `${GIT} rev-list origin/${BRANCH_NAME} -1`"
+      echo "${GIT} checkout -f  `${GIT} rev-list origin/${BRANCH_NAME} -1`"
+      ${GIT} checkout -f  `${GIT} rev-list origin/${BRANCH_NAME} -1`
+      echo "${GIT} status"
+      ${GIT} status
+    fi
+  fi
+}
+
+###############################################################################
+###  Collect findbugs reports
+collectFindbugsReports() {
+  name=$1
+  basedir=$2
+  patch_dir=$3
+  for file in $(find $basedir -name findbugsXml.xml)
+  do
+    relative_file=${file#$basedir/} # strip leading $basedir prefix
+    if [ ! $relative_file == "target/findbugsXml.xml" ]; then
+      module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path
+      module_suffix=`basename ${module_suffix}`
+    fi
+
+    cp $file $patch_dir/${name}FindbugsWarnings${module_suffix}.xml
+    $FINDBUGS_HOME/bin/setBugDatabaseInfo -name $name \
+      $patch_dir/${name}FindbugsWarnings${module_suffix}.xml \
+      $patch_dir/${name}FindbugsWarnings${module_suffix}.xml
+  done
+  xml_file=$patch_dir/${name}FindbugsWarnings.xml
+  html_file=$patch_dir/${name}FindbugsWarnings.html
+  $FINDBUGS_HOME/bin/unionBugs -withMessages \
+	 -output $xml_file $patch_dir/${name}FindbugsWarnings*.xml
+  $FINDBUGS_HOME/bin/convertXmlToText -html $xml_file $html_file
+  file $xml_file $html_file
+}
+
 ###############################################################################
 setup () {
   ### Download latest patch file (ignoring .htm and .html) when run from patch process
@@ -219,10 +297,12 @@ setup () {
     echo "$defect patch is being downloaded at `date` from"
     echo "$patchURL"
     $WGET -q -O $PATCH_DIR/patch $patchURL
-    VERSION=${SVN_REVISION}_${defect}_PATCH-${patchNum}
+    VERSION=${GIT_COMMIT}_${defect}_PATCH-${patchNum}
+    findBranchNameFromPatchName ${relativePatchURL}
+    checkoutBranch
     JIRA_COMMENT="Here are the results of testing the latest attachment 
   $patchURL
-  against trunk revision ${SVN_REVISION}.
+  against ${BRANCH_NAME} branch at commit ${GIT_COMMIT}.
   ATTACHMENT ID: ${ATTACHMENT_ID}"
 
   ### Copy the patch file to $PATCH_DIR
@@ -236,11 +316,9 @@ setup () {
       cleanupAndExit 0
     fi
   fi
-  . $BASEDIR/dev-support/test-patch.properties
   ### exit if warnings are NOT defined in the properties file
-  if [ -z "$OK_FINDBUGS_WARNINGS" ] || [[ -z "$OK_JAVADOC_WARNINGS" ]] || [[ -z $OK_RELEASEAUDIT_WARNINGS ]] ; then
+  if [[ -z "$OK_JAVADOC_WARNINGS" ]] || [[ -z $OK_RELEASEAUDIT_WARNINGS ]] ; then
     echo "Please define the following properties in test-patch.properties file"
-	 echo  "OK_FINDBUGS_WARNINGS"
 	 echo  "OK_RELEASEAUDIT_WARNINGS"
 	 echo  "OK_JAVADOC_WARNINGS"
     cleanupAndExit 1
@@ -249,22 +327,28 @@ setup () {
   echo ""
   echo "======================================================================"
   echo "======================================================================"
-  echo " Pre-build trunk to verify trunk stability and javac warnings" 
+  echo " Pre-build master to verify stability and javac warnings"
   echo "======================================================================"
   echo "======================================================================"
   echo ""
   echo ""
-  echo "$MVN clean package checkstyle:checkstyle-aggregate -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
+  echo "$MVN clean package checkstyle:checkstyle-aggregate findbugs:findbugs -DskipTests \
+ -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
   export MAVEN_OPTS="${MAVEN_OPTS}"
   # build core and tests
-  $MVN clean package checkstyle:checkstyle-aggregate -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1
+  $MVN clean package checkstyle:checkstyle-aggregate findbugs:findbugs -DskipTests \
+	 -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1
   if [[ $? != 0 ]] ; then
+    echo "mvn exit code was $?"
     ERR=`$GREP -A 5 'Compilation failure' $PATCH_DIR/trunkJavacWarnings.txt`
-    echo "Trunk compilation is broken?
-    {code}$ERR{code}"
-    cleanupAndExit 1
+    if [[ ${#ERR} -ge 1 ]] ; then
+      echo "Trunk compilation is broken?
+      {code}$ERR{code}"
+      cleanupAndExit 1
+    fi
   fi
   mv target/checkstyle-result.xml $PATCH_DIR/trunkCheckstyle.xml
+  collectFindbugsReports trunk $BASEDIR $PATCH_DIR
 }
 
 ###############################################################################
@@ -318,6 +402,16 @@ checkTests () {
         return 0
       fi
     fi
+    srcReferences=`${GREP} "diff --git" "${PATCH_DIR}/patch" | ${GREP} "src/main" | \
+        ${GREP} -v "src/main/asciidoc" | ${GREP} -v "src/main/site" -c`
+    if [[ $srcReferences == 0 ]] ; then
+      echo "The patch doesn't appear to alter any code that requires tests."
+      JIRA_COMMENT="$JIRA_COMMENT
+
+    {color:green}+0 tests included{color}.  The patch appears to be a documentation, build,
+                        or dev-support patch that doesn't require tests."
+      return 0
+    fi
     JIRA_COMMENT="$JIRA_COMMENT
 
     {color:red}-1 tests included{color}.  The patch doesn't appear to include any new or modified tests.
@@ -335,21 +429,26 @@ checkTests () {
 ### Check there are no compilation errors, passing a file to be parsed.
 checkCompilationErrors() {
   local file=$1
+  hadoopVersion=""
+  if [ "$#" -ne 1 ]; then
+    hadoopVersion="with Hadoop version $2"
+  fi
   COMPILATION_ERROR=false
   eval $(awk '/ERROR/ {print "COMPILATION_ERROR=true"}' $file)
   if $COMPILATION_ERROR ; then
     ERRORS=$($AWK '/ERROR/ { print $0 }' $file)
     echo "======================================================================"
-    echo "There are compilation errors."
+    echo "There are compilation errors $hadoopVersion."
     echo "======================================================================"
     echo "$ERRORS"
     JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:red}-1 javac{color}.  The patch appears to cause mvn compile goal to fail.
+    {color:red}-1 javac{color}.  The patch appears to cause mvn compile goal to fail $hadoopVersion.
 
     Compilation errors resume:
     $ERRORS
     "
+    submitJiraComment 1
     cleanupAndExit 1
   fi
 }
@@ -418,9 +517,8 @@ checkAntiPatterns () {
   if [[ $warnings != "" ]]; then
     JIRA_COMMENT="$JIRA_COMMENT
 
-		    {color:red}-1 Anti-pattern{color}.  The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted:
-             $warnings."
-	  return 1
+    {color:red}-1 Anti-pattern{color}.  The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted: $warnings."
+    return 1
   fi
   return 0
 }
@@ -441,9 +539,8 @@ checkInterfaceAudience () {
   if [[ $warnings != "" ]]; then
     JIRA_COMMENT="$JIRA_COMMENT
 
-		    {color:red}-1 InterfaceAudience{color}.  The patch appears to contain InterfaceAudience from hadoop rather than hbase:
-             $warnings."
-	  return 1
+    {color:red}-1 InterfaceAudience{color}.  The patch appears to contain InterfaceAudience from hadoop rather than hbase: $warnings."
+    return 1
   fi
   return 0
 }
@@ -473,6 +570,9 @@ checkJavadocWarnings () {
     JIRA_COMMENT="$JIRA_COMMENT
 
     {color:red}-1 javadoc{color}.  The javadoc tool appears to have generated `expr $(($javadocWarnings-$OK_JAVADOC_WARNINGS))` warning messages."
+    # Add javadoc output url
+    JIRA_COMMENT_FOOTER="Javadoc warnings: $BUILD_URL/artifact/patchprocess/patchJavadocWarnings.txt
+$JIRA_COMMENT_FOOTER"
     return 1
   fi
   JIRA_COMMENT="$JIRA_COMMENT
@@ -481,6 +581,31 @@ checkJavadocWarnings () {
   return 0
 }
 
+checkBuildWithHadoopVersions() {
+  echo ""
+  echo ""
+  echo "======================================================================"
+  echo "======================================================================"
+  echo "    Building with all supported Hadoop versions ."
+  echo "======================================================================"
+  echo "======================================================================"
+  echo ""
+  echo ""
+  export MAVEN_OPTS="${MAVEN_OPTS}"
+  for HADOOP2_VERSION in $HADOOP2_VERSIONS ; do
+    echo "$MVN clean install -DskipTests -D${PROJECT_NAME}PatchProcess -Dhadoop-two.version=$HADOOP2_VERSION > $PATCH_DIR/patchJavacWithHadoop-$HADOOP2_VERSION.txt 2>&1"
+    $MVN clean install -DskipTests -D${PROJECT_NAME}PatchProcess -Dhadoop-two.version=$HADOOP2_VERSION > $PATCH_DIR/patchJavacWithHadoop-$HADOOP2_VERSION.txt 2>&1
+    checkCompilationErrors $PATCH_DIR/patchJavacWithHadoop-$HADOOP2_VERSION.txt $HADOOP2_VERSION
+  done
+
+  # TODO: add Hadoop3 versions and compilation here when we get the hadoop.profile=3.0 working
+
+  JIRA_COMMENT="$JIRA_COMMENT
+
+    {color:green}+1 hadoop versions{color}. The patch compiles with all supported hadoop versions ($HADOOP2_VERSIONS)"
+  return 0
+}
+
 ###############################################################################
 ### Check there are no changes in the number of Javac warnings
 checkJavacWarnings () {
@@ -506,7 +631,7 @@ checkJavacWarnings () {
       if [[ $patchJavacWarnings -gt $trunkJavacWarnings ]] ; then
         JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:red}-1 javac{color}.  The applied patch generated $patchJavacWarnings javac compiler warnings (more than the trunk's current $trunkJavacWarnings warnings)."
+    {color:red}-1 javac{color}.  The applied patch generated $patchJavacWarnings javac compiler warnings (more than the master's current $trunkJavacWarnings warnings)."
         return 1
       fi
     fi
@@ -532,23 +657,25 @@ checkCheckstyleErrors() {
     mv target/checkstyle-result.xml $PATCH_DIR/patchCheckstyle.xml
     mv target/site/checkstyle-aggregate.html $PATCH_DIR
     mv target/site/checkstyle.css $PATCH_DIR
-    trunkCheckstyleErrors=`$GREP '<error' $PATCH_DIR/trunkCheckstyle.xml | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
-    patchCheckstyleErrors=`$GREP '<error' $PATCH_DIR/patchCheckstyle.xml | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
-    if [[ $patchCheckstyleErrors -gt $trunkCheckstyleErrors ]] ; then
+    $BASEDIR/dev-support/checkstyle_report.py $PATCH_DIR/trunkCheckstyle.xml $PATCH_DIR/patchCheckstyle.xml
+    if [[ $? -eq 1 ]] ; then
                 JIRA_COMMENT_FOOTER="Checkstyle Errors: $BUILD_URL/artifact/patchprocess/checkstyle-aggregate.html
 
                 $JIRA_COMMENT_FOOTER"
 
                 JIRA_COMMENT="$JIRA_COMMENT
 
-                {color:red}-1 javac{color}.  The applied patch generated $patchCheckstyleErrors checkstyle errors (more than the trunk's current $trunkCheckstyleErrors errors)."
+                {color:red}-1 checkstyle{color}.  The applied patch generated new checkstyle errors. Check build console for list of new errors."
         return 1
     fi
-    echo "There were $patchCheckstyleErrors checkstyle errors in this patch compared to $trunkCheckstyleErrors on master."
   fi
+  JIRA_COMMENT_FOOTER="Checkstyle Errors: $BUILD_URL/artifact/patchprocess/checkstyle-aggregate.html
+
+  $JIRA_COMMENT_FOOTER"
+
   JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:green}+1 javac{color}.  The applied patch does not increase the total number of checkstyle errors"
+    {color:green}+1 checkstyle{color}. The applied patch does not generate new checkstyle errors."
   return 0
 
 }
@@ -569,7 +696,7 @@ checkProtocErrors () {
   checkProtocCompilationErrors $PATCH_DIR/patchProtocErrors.txt
   JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:green}+1 javac{color}.  The applied patch does not increase the total number of javac compiler warnings."
+    {color:green}+1 protoc{color}.  The applied patch does not increase the total number of protoc compiler warnings."
   return 0
 }
 
@@ -600,7 +727,7 @@ checkReleaseAuditWarnings () {
       if [[ $patchReleaseAuditWarnings -gt $OK_RELEASEAUDIT_WARNINGS ]] ; then
         JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:red}-1 release audit{color}.  The applied patch generated $patchReleaseAuditWarnings release audit warnings (more than the trunk's current $OK_RELEASEAUDIT_WARNINGS warnings)."
+    {color:red}-1 release audit{color}.  The applied patch generated $patchReleaseAuditWarnings release audit warnings (more than the master's current $OK_RELEASEAUDIT_WARNINGS warnings)."
         $GREP '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt > $PATCH_DIR/patchReleaseAuditProblems.txt
         echo "Lines that start with ????? in the release audit report indicate files that do not have an Apache license header." >> $PATCH_DIR/patchReleaseAuditProblems.txt
         JIRA_COMMENT_FOOTER="Release audit warnings: $BUILD_URL/artifact/patchprocess/patchReleaseAuditWarnings.txt
@@ -638,41 +765,36 @@ checkFindbugsWarnings () {
     {color:red}-1 findbugs{color}.  The patch appears to cause Findbugs (version ${findbugs_version}) to fail."
     return 1
   fi
-    
-  findbugsWarnings=0
-  for file in $(find $BASEDIR -name findbugsXml.xml)
-  do
-    relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix
-    if [ ! $relative_file == "target/findbugsXml.xml" ]; then
-      module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path
-      module_suffix=`basename ${module_suffix}`
-    fi
-    
-    cp $file $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml
-    $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \
-      $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \
-      $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml
-    newFindbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \
-      $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml | $AWK '{print $1}'`
-    echo "Found $newFindbugsWarnings Findbugs warnings ($file)"
-    findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings))
-    $FINDBUGS_HOME/bin/convertXmlToText -html \
-      $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \
-      $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html
-    JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/trunk/patchprocess/newPatchFindbugsWarnings${module_suffix}.html
-$JIRA_COMMENT_FOOTER"
-  done
 
-  ### if current warnings greater than OK_FINDBUGS_WARNINGS
-  if [[ $findbugsWarnings -gt $OK_FINDBUGS_WARNINGS ]] ; then
+  collectFindbugsReports patch $BASEDIR $PATCH_DIR 
+  #this files are generated by collectFindbugsReports() named with its first argument
+  patch_xml=$PATCH_DIR/patchFindbugsWarnings.xml
+  trunk_xml=$PATCH_DIR/trunkFindbugsWarnings.xml
+  # combine them to one database
+  combined_xml=$PATCH_DIR/combinedFindbugsWarnings.xml
+  new_xml=$PATCH_DIR/newFindbugsWarnings.xml
+  new_html=$PATCH_DIR/newFindbugsWarnings.html
+  $FINDBUGS_HOME/bin/computeBugHistory -useAnalysisTimes -withMessages \
+	-output $combined_xml $trunk_xml $patch_xml
+  findbugsWarnings=$($FINDBUGS_HOME/bin/filterBugs -first patch $combined_xml $new_xml)
+  findbugsFixedWarnings=$($FINDBUGS_HOME/bin/filterBugs -fixed patch $combined_xml $new_xml)
+  $FINDBUGS_HOME/bin/convertXmlToText -html  $new_xml $new_html
+  file $new_xml $new_html
+  JIRA_COMMENT_FOOTER="Release Findbugs (version ${findbugs_version}) \
+	warnings: $BUILD_URL/artifact/patchprocess/newFindbugsWarnings.html
+$JIRA_COMMENT_FOOTER"
+  ### if current warnings greater than 0, fail
+  if [[ $findbugsWarnings -gt 0 ]] ; then
     JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:red}-1 findbugs{color}.  The patch appears to introduce `expr $(($findbugsWarnings-$OK_FINDBUGS_WARNINGS))` new Findbugs (version ${findbugs_version}) warnings."
+    {color:red}-1 findbugs{color}.  The patch appears to introduce $findbugsWarnings \
+ new Findbugs (version ${findbugs_version}) warnings."
     return 1
   fi
   JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:green}+1 findbugs{color}.  The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
+    {color:green}+1 findbugs{color}.  The patch does not introduce any \
+ new Findbugs (version ${findbugs_version}) warnings."
   return 0
 }
 
@@ -691,7 +813,7 @@ checkLineLengths () {
   #see http://en.wikipedia.org/wiki/Diff#Unified_format
 
   MAX_LINE_LENGTH_PATCH=`expr $MAX_LINE_LENGTH + 1`
-  lines=`cat $PATCH_DIR/patch | grep "^+" | grep -v "^@@" | grep -v "^+++" | grep -v "import" | grep -v "org.apache.thrift." | grep -v "com.google.protobuf." | grep -v "hbase.protobuf.generated" | awk -v len="$MAX_LINE_LENGTH_PATCH" 'length ($0) > len' | head -n 10`
+  lines=`cat $PATCH_DIR/patch | grep "^+" | grep -v "^@@" | grep -v "^+++" | grep -v "import" | grep -v "org.apache.thrift." | grep -v "com.google.protobuf." | grep -v "protobuf.generated" | awk -v len="$MAX_LINE_LENGTH_PATCH" 'length ($0) > len' | head -n 10`
   ll=`echo "$lines" | wc -l`
   if [[ "$ll" -gt "1" ]]; then
     JIRA_COMMENT="$JIRA_COMMENT
@@ -719,59 +841,30 @@ runTests () {
   echo "======================================================================"
   echo ""
   echo ""
-
-
-  ### kill any process remaining from another test, maybe even another project
-  jps | grep surefirebooter | cut -d ' ' -f 1 | xargs kill -9 2>/dev/null
-  
   failed_tests=""
-  ### Kill any rogue build processes from the last attempt
-  condemnedCount=`$PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
-  echo "WARNING: $condemnedCount rogue build processes detected, terminating."
-  $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
-  echo "$MVN clean test -P runAllTests -D${PROJECT_NAME}PatchProcess"
+  echo "$MVN clean test -Dsurefire.rerunFailingTestsCount=2 -P runAllTests -D${PROJECT_NAME}PatchProcess"
   export MAVEN_OPTS="${MAVEN_OPTS}"
   ulimit -a
-  $MVN clean test -P runAllTests -D${PROJECT_NAME}PatchProcess
+  # Need to export this so the zombie subshell picks up current content
+  export JIRA_COMMENT
+  $MVN clean test -Dsurefire.rerunFailingTestsCount=2 -P runAllTests -D${PROJECT_NAME}PatchProcess
   if [[ $? != 0 ]] ; then
      ### Find and format names of failed tests
      failed_tests=`find . -name 'TEST*.xml' | xargs $GREP  -l -E "<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-|                  |g" | sed -e "s|\.xml||g"`
  
      JIRA_COMMENT="$JIRA_COMMENT
 
-     {color:red}-1 core tests{color}.  The patch failed these unit tests:
+    {color:red}-1 core tests{color}.  The patch failed these unit tests:
      $failed_tests"
      BAD=1
+     JIRA_COMMENT=`$BASEDIR/dev-support/zombie-detector.sh ${BUILD_ID}`
   else
     JIRA_COMMENT="$JIRA_COMMENT
 
     {color:green}+1 core tests{color}.  The patch passed unit tests in $modules."
-    BAD=0
+     JIRA_COMMENT=`$BASEDIR/dev-support/zombie-detector.sh ${BUILD_ID}`
+    BAD=$?
   fi
-  ZOMBIE_TESTS_COUNT=`jps | grep surefirebooter | wc -l`
-  if [[ $ZOMBIE_TESTS_COUNT != 0 ]] ; then
-    #It seems sometimes the tests are not dying immediately. Let's give them 30s
-    echo "Suspicious java process found - waiting 30s to see if there are just slow to stop"
-    sleep 30
-    ZOMBIE_TESTS_COUNT=`jps | grep surefirebooter | wc -l`
-    if [[ $ZOMBIE_TESTS_COUNT != 0 ]] ; then
-      echo "There are $ZOMBIE_TESTS_COUNT zombie tests, they should have been killed by surefire but survived"
-      echo "************ BEGIN zombies jstack extract"
-      ZB_STACK=`jps | grep surefirebooter | cut -d ' ' -f 1 | xargs -n 1 jstack | grep ".test" | grep "\.java"`
-      jps | grep surefirebooter | cut -d ' ' -f 1 | xargs -n 1 jstack
-      echo "************ END  zombies jstack extract"
-      JIRA_COMMENT="$JIRA_COMMENT
-
-     {color:red}-1 core zombie tests{color}.  There are ${ZOMBIE_TESTS_COUNT} zombie test(s): ${ZB_STACK}"
-      BAD=1
-      jps | grep surefirebooter | cut -d ' ' -f 1 | xargs kill -9
-    else
-      echo "We're ok: there is no zombie test, but some tests took some time to stop"
-    fi
-  else
-    echo "We're ok: there is no zombie test"
-  fi
-  return $BAD
 }
 
 ###############################################################################
@@ -787,18 +880,18 @@ checkSiteXml () {
   echo ""
   echo ""
 
-  echo "$MVN package site -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchSiteOutput.txt 2>&1"
+  echo "$MVN package post-site -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchSiteOutput.txt 2>&1"
   export MAVEN_OPTS="${MAVEN_OPTS}"
-  $MVN package site -DskipTests -D${PROJECT_NAME}PatchProcess  > $PATCH_DIR/patchSiteOutput.txt 2>&1
+  $MVN package post-site -DskipTests -D${PROJECT_NAME}PatchProcess  > $PATCH_DIR/patchSiteOutput.txt 2>&1
   if [[ $? != 0 ]] ; then
     JIRA_COMMENT="$JIRA_COMMENT
 
-    {color:red}-1 site{color}.  The patch appears to cause mvn site goal to fail."
+    {color:red}-1 site{color}.  The patch appears to cause mvn post-site goal to fail."
     return 1
   fi
   JIRA_COMMENT="$JIRA_COMMENT
 
-  {color:green}+1 site{color}.  The mvn site goal succeeds with this patch."
+    {color:green}+1 site{color}.  The mvn post-site goal succeeds with this patch."
   return 0
 }
 
@@ -879,8 +972,9 @@ $comment"
 ### Cleanup files
 cleanupAndExit () {
   local result=$1
-  if [[ $JENKINS == "true" ]] ; then
+  if [[ ${JENKINS} == "true" && ${MOVE_PATCH_DIR} == "true" ]] ; then
     if [ -e "$PATCH_DIR" ] ; then
+      echo "Relocating patch dir into ${BASEDIR}"
       mv $PATCH_DIR $BASEDIR
     fi
   fi
@@ -909,8 +1003,10 @@ This message is automatically generated."
 parseArgs $@
 cd $BASEDIR
 
+echo "Version of this script: Wed Oct 14 00:29:04 PDT 2015"
 checkout
 RESULT=$?
+echo "RESULT = " $RESULT
 if [[ $JENKINS == "true" ]] ; then
   if [[ $RESULT != 0 ]] ; then
     exit 100
@@ -919,8 +1015,10 @@ fi
 setup
 checkAuthor
 RESULT=$?
+echo "RESULT = " $RESULT
 checkTests
 (( RESULT = RESULT + $? ))
+echo "RESULT = " $RESULT
 applyPatch
 if [[ $? != 0 ]] ; then
   submitJiraComment 1
@@ -929,28 +1027,42 @@ fi
 
 checkAntiPatterns
 (( RESULT = RESULT + $? ))
+echo "RESULT = " $RESULT
+checkBuildWithHadoopVersions
+(( RESULT = RESULT + $? ))
+echo "RESULT = " $RESULT
 checkJavacWarnings
 (( RESULT = RESULT + $? ))
+echo "RESULT = " $RESULT
 checkProtocErrors
 (( RESULT = RESULT + $? ))
+echo "RESULT = " $RESULT
 checkJavadocWarnings
 (( RESULT = RESULT + $? ))
+echo "RESULT = " $RESULT
 checkCheckstyleErrors
 (( RESULT = RESULT + $? ))
+echo "RESULT = " $RESULT
 checkInterfaceAudience
 (( RESULT = RESULT + $? ))
+echo "RESULT = " $RESULT
 checkFindbugsWarnings
 (( RESULT = RESULT + $? ))
+echo "RESULT = " $RESULT
 checkReleaseAuditWarnings
 (( RESULT = RESULT + $? ))
+echo "RESULT = " $RESULT
 checkLineLengths
 (( RESULT = RESULT + $? ))
+echo "RESULT = " $RESULT
 checkSiteXml
 (( RESULT = RESULT + $?))
-### Do not call these when run by a developer 
+echo "RESULT = " $RESULT
+### Do not call these when run by a developer
 if [[ $JENKINS == "true" ]] ; then
   runTests
   (( RESULT = RESULT + $? ))
+  echo "RESULT = " $RESULT
 JIRA_COMMENT_FOOTER="Test results: $BUILD_URL/testReport/
 $JIRA_COMMENT_FOOTER"
 fi

http://git-wip-us.apache.org/repos/asf/hbase/blob/f02a9fa6/dev-support/zombie-detector.sh
----------------------------------------------------------------------
diff --git a/dev-support/zombie-detector.sh b/dev-support/zombie-detector.sh
new file mode 100755
index 0000000..df4c197
--- /dev/null
+++ b/dev-support/zombie-detector.sh
@@ -0,0 +1,166 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Looks for any running zombies left over from old build runs.
+# Will report and try to do stack trace on stale processes so can
+# figure how they are hung. Echos state as the script runs
+# on STDERR but prints final output on STDOUT formatted so it
+# will fold into the test result formatting done by test-patch.sh.
+# This script is called from test-patch.sh but also after tests
+# have run up on builds.apache.org.
+
+# TODO: format output to suit context -- test-patch, jenkins or dev env
+
+#set -x
+# printenv
+
+### Setup some variables.  
+bindir=$(dirname $0)
+
+# This key is set by our surefire configuration up in the main pom.xml
+# This key needs to match the key we set up there.
+HBASE_BUILD_ID_KEY="hbase.build.id="
+JENKINS=
+
+PS=${PS:-ps}
+AWK=${AWK:-awk}
+WGET=${WGET:-wget}
+GREP=${GREP:-grep}
+JIRACLI=${JIRA:-jira}
+
+###############################################################################
+printUsage() {
+  echo "Usage: $0 [options]" BUILD_ID
+  echo
+  echo "Where:"
+  echo "  BUILD_ID is build id to look for in process listing"
+  echo
+  echo "Options:"
+  echo "--ps-cmd=<cmd>         The 'ps' command to use (default 'ps')"
+  echo "--awk-cmd=<cmd>        The 'awk' command to use (default 'awk')"
+  echo "--grep-cmd=<cmd>       The 'grep' command to use (default 'grep')"
+  echo
+  echo "Jenkins-only options:"
+  echo "--jenkins              Run by Jenkins (runs tests and posts results to JIRA)"
+  echo "--wget-cmd=<cmd>       The 'wget' command to use (default 'wget')"
+  echo "--jira-cmd=<cmd>       The 'jira' command to use (default 'jira')"
+}
+
+###############################################################################
+parseArgs() {
+  for i in $*
+  do
+    case $i in
+    --jenkins)
+      JENKINS=true
+      ;;
+    --ps-cmd=*)
+      PS=${i#*=}
+      ;;
+    --awk-cmd=*)
+      AWK=${i#*=}
+      ;;
+    --wget-cmd=*)
+      WGET=${i#*=}
+      ;;
+    --grep-cmd=*)
+      GREP=${i#*=}
+      ;;
+    --jira-cmd=*)
+      JIRACLI=${i#*=}
+      ;;
+    *)
+      BUILD_ID=$i
+      ;;
+    esac
+  done
+  if [ -z "$BUILD_ID" ]; then
+    printUsage
+    exit 1
+  fi
+}
+
+### Return list of the processes found with passed build id.
+find_processes () {
+  jps -v | grep surefirebooter | grep -e "${HBASE_BUILD_TAG}"
+}
+
+### Look for zombies
+zombies () {
+  ZOMBIES=`find_processes`
+  if [[ -z ${ZOMBIES} ]]
+  then
+    ZOMBIE_TESTS_COUNT=0
+  else
+    ZOMBIE_TESTS_COUNT=`echo "${ZOMBIES}"| wc -l| xargs`
+  fi
+  if [[ $ZOMBIE_TESTS_COUNT != 0 ]] ; then
+    wait=30
+    echo "`date` Found ${ZOMBIE_TESTS_COUNT} suspicious java process(es) listed below; waiting ${wait}s to see if just slow to stop" >&2
+    echo ${ZOMBIES} >&2
+    sleep ${wait}
+    PIDS=`echo "${ZOMBIES}"|${AWK} '{print $1}'`
+    ZOMBIE_TESTS_COUNT=0
+    for pid in $PIDS
+    do
+      # Test our zombie still running (and that it still an hbase build item)
+      PS_OUTPUT=`ps -p $pid | tail +2 | grep -e "${HBASE_BUILD_TAG}"`
+      if [[ ! -z "${PS_OUTPUT}" ]]
+      then
+        echo "`date` Zombie: $PS_OUTPUT" >&2
+        let "ZOMBIE_TESTS_COUNT+=1"
+        PS_STACK=`jstack $pid | grep -e "\.Test" | grep -e "\.java"| head -3`
+        echo "${PS_STACK}" >&2
+        ZB_STACK="${ZB_STACK}\nPID=${pid} ${PS_STACK}"
+      fi
+    done
+    if [[ $ZOMBIE_TESTS_COUNT != 0 ]]
+    then
+      echo "`date` There are ${ZOMBIE_TESTS_COUNT} possible zombie test(s)." >&2
+      # If JIRA_COMMENT in environment, append our findings to it
+      echo -e "$JIRA_COMMENT
+
+    {color:red}+1 zombies{red}. There are ${ZOMBIE_TESTS_COUNT} possible zombie test(s)
+        ${ZB_STACK}"
+      # Exit with exit code of 1.
+      exit 1
+    else
+      echo "`date` We're ok: there was a zombie candidate but it went away" >&2
+      echo "$JIRA_COMMENT
+
+    {color:green}+1 zombies{color}. No zombie tests found running at the end of the build (There were candidates but they seem to have gone away)."
+    fi
+  else
+      echo "`date` We're ok: there is no zombie test" >&2
+      echo "$JIRA_COMMENT
+
+    {color:green}+1 zombies{color}. No zombie tests found running at the end of the build."
+  fi
+}
+
+### Check if arguments to the script have been specified properly or not
+parseArgs $@
+HBASE_BUILD_TAG="${HBASE_BUILD_ID_KEY}${BUILD_ID}"
+zombies
+RESULT=$?
+if [[ $JENKINS == "true" ]] ; then
+  if [[ $RESULT != 0 ]] ; then
+    exit 100
+  fi
+fi
+RESULT=$?