Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id C63CF11A4A for ; Wed, 24 Sep 2014 20:53:01 +0000 (UTC) Received: (qmail 93164 invoked by uid 500); 24 Sep 2014 20:53:01 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 93068 invoked by uid 500); 24 Sep 2014 20:53:01 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 92411 invoked by uid 99); 24 Sep 2014 20:53:01 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 24 Sep 2014 20:53:01 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 28DAE9A3BDF; Wed, 24 Sep 2014 20:53:01 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jonathanhurley@apache.org To: commits@ambari.apache.org Date: Wed, 24 Sep 2014 20:53:19 -0000 Message-Id: <763b20fa267a499393c6afa1cf4681d6@git.apache.org> In-Reply-To: <998119e1ae8c4d4598070a58f013b4f6@git.apache.org> References: <998119e1ae8c4d4598070a58f013b4f6@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [20/37] git commit: AMBARI-7209. Build a test patch Jenkins job on builds.a.o. (Jun Aoki via yusaku) AMBARI-7209. Build a test patch Jenkins job on builds.a.o. (Jun Aoki via yusaku) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6e73363b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6e73363b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6e73363b Branch: refs/heads/branch-alerts-dev Commit: 6e73363bb31221cc47d301a65a729049a0475b2f Parents: b749dd5 Author: Yusaku Sako Authored: Tue Sep 23 17:22:17 2014 -0700 Committer: Yusaku Sako Committed: Tue Sep 23 17:22:17 2014 -0700 ---------------------------------------------------------------------- ...test-patch-find-new-patch-available-jiras.sh | 126 ++ dev-support/test-patch.sh | 1077 ++++++++++++++++++ 2 files changed, 1203 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6e73363b/dev-support/test-patch-find-new-patch-available-jiras.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch-find-new-patch-available-jiras.sh b/dev-support/test-patch-find-new-patch-available-jiras.sh new file mode 100644 index 0000000..09d3924 --- /dev/null +++ b/dev-support/test-patch-find-new-patch-available-jiras.sh @@ -0,0 +1,126 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "${TESTPATCHDEBUG}" == "true" ] ; then + set -x +fi + +BASEDIR=$(pwd) +TEMPDIR=${BASEDIR}/tmp + +JIRAAVAILPATCHQUERY="https://issues.apache.org/jira/sr/jira.issueviews:searchrequest-xml/temp/SearchRequest.xml?jqlQuery=project+in+%28AMBARI%29+AND+status+%3D+%22Patch+Available%22+ORDER+BY+updated+DESC&tempMax=1000" +TESTPATCHJOBURL="https://builds.apache.org/job/Ambari-trunk-test-patch" +# TESTPATCHJOBURL="http://10.103.219.57:8080/job/Ambari-trunk-test-patch" +TOKEN="" +SUBMIT="false" +DELETEHISTORYFILE="false" + +RUNTESTSFILE=${BASEDIR}/TESTED_PATCHES.txt + +printUsage() { + echo "Usage: $0 " + echo " --submit --token=" + echo " [--delete-history-file]" + echo " [--script-debug]" + echo +} +############################################################################### +parseArgs() { + for i in $* + do + case $i in + --submit) + SUBMIT="true" + ;; + --token=*) + TOKEN=${i#*=} + ;; + --script-debug) + DEBUG="-x" + ;; + --delete-history-file) + DELETEHISTORYFILE="true" + ;; + *) + echo "Invalid option" + echo + printUsage + exit 1 + ;; + esac + done + if [[ "$SUBMIT" == "true" && "${TOKEN}" == "" ]] ; then + echo "Token has not been specified" + echo + printUsage + exit 1 + fi +} +############################################################################### +findAndSubmitAvailablePatches() { + +## Grab all the key (issue numbers) and largest attachment id for each item in the XML +curl --fail --location --retry 3 "${JIRAAVAILPATCHQUERY}" > ${TEMPDIR}/patch-availables.xml +if [ "$?" != "0" ] ; then + echo "Could not retrieve available patches from JIRA" + exit 1 +fi +xpath -q -e "//item/key/text() | //item/attachments/attachment[not(../attachment/@id > @id)]/@id" ${TEMPDIR}/patch-availables.xml > ${TEMPDIR}/patch-attachments.element + +### Replace newlines with nothing, then replace id=" with =, then replace " with newline +### to yield lines with pairs (issueNumber,largestAttachmentId). Example: AMBARI-123,456984 +cat ${TEMPDIR}/patch-attachments.element | awk '{ if ( $1 ~ /^AMBARI\-/) {JIRA=$1 }; if ($1 ~ /id=/) { print JIRA","$1} }' | sed 's/id\="//' | sed 's/"//' > ${TEMPDIR}/patch-availables.pair + +### Iterate through issue list and find the (issueNumber,largestAttachmentId) pairs that have +### not been tested (ie don't already exist in the patch_tested.txt file +touch ${RUNTESTSFILE} +cat ${TEMPDIR}/patch-availables.pair | while read PAIR ; do + set +e + COUNT=`grep -c "$PAIR" ${RUNTESTSFILE}` + set -e + if [ "$COUNT" -lt "1" ] ; then + ### Parse $PAIR into project, issue number, and attachment id + ISSUE=`echo $PAIR | sed -e "s/,.*$//"` + echo "Found new patch for issue $ISSUE" + if [ "$SUBMIT" == "true" ]; then + ### Kick off job + echo "Submitting job for issue $ISSUE" + curl --fail --location --retry 3 "${TESTPATCHJOBURL}/buildWithParameters?token=${TOKEN}&JIRA_NUMBER=${ISSUE}" > /dev/null + if [ "$?" != "0" ] ; then + echo "Could not submit precommit job for $ISSUE" + exit 1 + fi + fi + ### Mark this pair as tested by appending to file + echo "$PAIR" >> ${RUNTESTSFILE} + fi +done +} +############################################################################### + +mkdir ${TEMPDIR} 2>&1 $STDOUT + +parseArgs "$@" + +if [ -n "${DEBUG}" ] ; then + set -x +fi + +if [ "${DELETEHISTORYFILE}" == "true" ] ; then + rm ${RUNTESTSFILE} +fi + +findAndSubmitAvailablePatches + +exit 0 http://git-wip-us.apache.org/repos/asf/ambari/blob/6e73363b/dev-support/test-patch.sh ---------------------------------------------------------------------- diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh new file mode 100644 index 0000000..c0b60d2 --- /dev/null +++ b/dev-support/test-patch.sh @@ -0,0 +1,1077 @@ +#!/usr/bin/env bash +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +#set -x +ulimit -n 2048 + +### Setup some variables. +### SVN_REVISION and BUILD_URL are set by Hudson if it is run by patch process +### Read variables from properties file +bindir=$(dirname $0) + +# Defaults +if [ -z "$MAVEN_HOME" ]; then + MVN=mvn +else + MVN=$MAVEN_HOME/bin/mvn +fi + +PROJECT_NAME=Ambari +JENKINS=false +PATCH_DIR=/tmp +SUPPORT_DIR=/tmp +BASEDIR=$(pwd) +PS=${PS:-ps} +AWK=${AWK:-awk} +WGET=${WGET:-wget} +GIT=${GIT:-git} +GREP=${GREP:-grep} +PATCH=${PATCH:-patch} +DIFF=${DIFF:-diff} +JIRACLI=${JIRA:-jira} +FINDBUGS_HOME=${FINDBUGS_HOME} +FORREST_HOME=${FORREST_HOME} +ECLIPSE_HOME=${ECLIPSE_HOME} + +############################################################################### +printUsage() { + echo "Usage: $0 [options] patch-file | defect-number" + echo + echo "Where:" + echo " patch-file is a local patch file containing the changes to test" + echo " defect-number is a JIRA defect number (e.g. 'HADOOP-1234') to test (Jenkins only)" + echo + echo "Options:" + echo "--patch-dir= The directory for working and output files (default '/tmp')" + echo "--basedir= The directory to apply the patch to (default current directory)" + echo "--mvn-cmd= The 'mvn' command to use (default \$MAVEN_HOME/bin/mvn, or 'mvn')" + echo "--ps-cmd= The 'ps' command to use (default 'ps')" + echo "--awk-cmd= The 'awk' command to use (default 'awk')" + echo "--git-cmd= The 'git' command to use (default 'git')" + echo "--grep-cmd= The 'grep' command to use (default 'grep')" + echo "--patch-cmd= The 'patch' command to use (default 'patch')" + echo "--diff-cmd= The 'diff' command to use (default 'diff')" + echo "--findbugs-home= Findbugs home directory (default FINDBUGS_HOME environment variable)" + echo "--forrest-home= Forrest home directory (default FORREST_HOME environment variable)" + echo "--dirty-workspace Allow the local SVN workspace to have uncommitted changes" + echo "--run-tests Run all tests below the base directory" + echo + echo "Jenkins-only options:" + echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)" + echo "--support-dir= The directory to find support files in" + echo "--wget-cmd= The 'wget' command to use (default 'wget')" + echo "--jira-cmd= The 'jira' command to use (default 'jira')" + echo "--jira-password= The password for the 'jira' command" + echo "--eclipse-home= Eclipse home directory (default ECLIPSE_HOME environment variable)" +} + +############################################################################### +parseArgs() { + for i in $* + do + case $i in + --jenkins) + JENKINS=true + ;; + --patch-dir=*) + PATCH_DIR=${i#*=} + ;; + --support-dir=*) + SUPPORT_DIR=${i#*=} + ;; + --basedir=*) + BASEDIR=${i#*=} + ;; + --mvn-cmd=*) + MVN=${i#*=} + ;; + --ps-cmd=*) + PS=${i#*=} + ;; + --awk-cmd=*) + AWK=${i#*=} + ;; + --wget-cmd=*) + WGET=${i#*=} + ;; + --git-cmd=*) + GIT=${i#*=} + ;; + --grep-cmd=*) + GREP=${i#*=} + ;; + --patch-cmd=*) + PATCH=${i#*=} + ;; + --diff-cmd=*) + DIFF=${i#*=} + ;; + --jira-cmd=*) + JIRACLI=${i#*=} + ;; + --jira-password=*) + JIRA_PASSWD=${i#*=} + ;; + --findbugs-home=*) + FINDBUGS_HOME=${i#*=} + ;; + --forrest-home=*) + FORREST_HOME=${i#*=} + ;; + --eclipse-home=*) + ECLIPSE_HOME=${i#*=} + ;; + --dirty-workspace) + DIRTY_WORKSPACE=true + ;; + --run-tests) + RUN_TESTS=true + ;; + *) + PATCH_OR_DEFECT=$i + ;; + esac + done + if [ -z "$PATCH_OR_DEFECT" ]; then + printUsage + exit 1 + fi + if [[ $JENKINS == "true" ]] ; then + echo "Running in Jenkins mode" + defect=$PATCH_OR_DEFECT + ECLIPSE_PROPERTY="-Declipse.home=$ECLIPSE_HOME" + else + echo "Running in developer mode" + JENKINS=false + ### PATCH_FILE contains the location of the patchfile + PATCH_FILE=$PATCH_OR_DEFECT + if [[ ! -e "$PATCH_FILE" ]] ; then + echo "Unable to locate the patch file $PATCH_FILE" + cleanupAndExit 0 + fi + ### Obtain the patch filename to append it to the version number + defect=`basename $PATCH_FILE` + fi +} + +############################################################################### +checkout () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Testing patch for ${defect}." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + ### When run by a developer, if the workspace contains modifications, do not continue + ### unless the --dirty-workspace option was set + # status=`$GIT status -s | sed -e '/^X[ ]*/D'` + status=`$GIT status -s ` + if [[ $JENKINS == "false" ]] ; then + if [[ "$status" != "" && -z $DIRTY_WORKSPACE ]] ; then + echo "ERROR: can't run in a workspace that contains the following modifications" + echo "$status" + cleanupAndExit 1 + fi + echo + else + cd $BASEDIR + $GIT checkout -- . + $GIT clean -x -f -d + $GIT pull + fi + return $? +} + +############################################################################### +downloadPatch () { + ### Download latest patch file (ignoring .htm and .html) when run from patch process + if [[ $JENKINS == "true" ]] ; then + $WGET -q -O $PATCH_DIR/jira http://issues.apache.org/jira/browse/$defect + if [[ `$GREP -c 'Patch Available' $PATCH_DIR/jira` == 0 ]] ; then + echo "$defect is not \"Patch Available\". Exiting." + cleanupAndExit 0 + fi + relativePatchURL=`$GREP -o '"/jira/secure/attachment/[0-9]*/[^"]*' $PATCH_DIR/jira | $GREP -v -e 'htm[l]*$' | sort | tail -1 | $GREP -o '/jira/secure/attachment/[0-9]*/[^"]*'` + patchURL="http://issues.apache.org${relativePatchURL}" + patchNum=`echo $patchURL | $GREP -o '[0-9]*/' | $GREP -o '[0-9]*'` + echo "$defect patch is being downloaded at `date` from" + echo "$patchURL" + $WGET -q -O $PATCH_DIR/patch $patchURL + VERSION=${SVN_REVISION}_${defect}_PATCH-${patchNum} + JIRA_COMMENT="Here are the results of testing the latest attachment + $patchURL + against trunk revision ${SVN_REVISION}." + + ### Copy in any supporting files needed by this process + # cp -r $SUPPORT_DIR/lib/* ./lib + #PENDING: cp -f $SUPPORT_DIR/etc/checkstyle* ./src/test + ### Copy the patch file to $PATCH_DIR + else + VERSION=PATCH-${defect} + cp $PATCH_FILE $PATCH_DIR/patch + if [[ $? == 0 ]] ; then + echo "Patch file $PATCH_FILE copied to $PATCH_DIR" + else + echo "Could not copy $PATCH_FILE to $PATCH_DIR" + cleanupAndExit 0 + fi + fi +} + +############################################################################### +verifyPatch () { + # Before building, check to make sure that the patch is valid + # $bindir/smart-apply-patch.sh $PATCH_DIR/patch dryrun + $GIT apply --check $PATCH_DIR/patch + if [[ $? != 0 ]] ; then + echo "PATCH APPLICATION FAILED" + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 patch{color}. The patch command could not apply the patch." + return 1 + else + return 0 + fi +} + +############################################################################### +prebuildWithoutPatch () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Pre-build trunk to verify trunk stability and javac warnings" + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + if [[ ! -d hadoop-common-project ]]; then + cd $bindir/.. + echo "Compiling $(pwd)" + echo "$MVN clean test -DskipTests > $PATCH_DIR/trunkCompile.txt 2>&1" + $MVN clean test -DskipTests > $PATCH_DIR/trunkCompile.txt 2>&1 + if [[ $? != 0 ]] ; then + echo "Top-level trunk compilation is broken?" + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 patch{color}. Top-level trunk compilation may be broken." + return 1 + fi + cd - + fi + echo "Compiling $(pwd)" + echo "$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1" + $MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1 + if [[ $? != 0 ]] ; then + echo "Trunk compilation is broken?" + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 patch{color}. Trunk compilation may be broken." + return 1 + fi + + echo "$MVN clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavadocWarnings.txt 2>&1" + $MVN clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavadocWarnings.txt 2>&1 + if [[ $? != 0 ]] ; then + echo "Trunk javadoc compilation is broken?" + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 patch{color}. Trunk compilation may be broken." + return 1 + fi + + return 0 +} + +############################################################################### +### Check for @author tags in the patch +checkAuthor () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Checking there are no @author tags in the patch." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + authorTags=`$GREP -c -i '@author' $PATCH_DIR/patch` + echo "There appear to be $authorTags @author tags in the patch." + if [[ $authorTags != 0 ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 @author{color}. The patch appears to contain $authorTags @author tags which the Hadoop community has agreed to not allow in code contributions." + return 1 + fi + JIRA_COMMENT="$JIRA_COMMENT + + {color:green}+1 @author{color}. The patch does not contain any @author tags." + return 0 +} + +############################################################################### +### Check for tests in the patch +checkTests () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Checking there are new or changed tests in the patch." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + testReferences=`$GREP -c -i -e '^+++.*/test' $PATCH_DIR/patch` + echo "There appear to be $testReferences test files referenced in the patch." + if [[ $testReferences == 0 ]] ; then + if [[ $JENKINS == "true" ]] ; then + patchIsDoc=`$GREP -c -i 'title="documentation' $PATCH_DIR/jira` + if [[ $patchIsDoc != 0 ]] ; then + echo "The patch appears to be a documentation patch that doesn't require tests." + JIRA_COMMENT="$JIRA_COMMENT + + {color:green}+0 tests included{color}. The patch appears to be a documentation patch that doesn't require tests." + return 0 + fi + fi + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 tests included{color}. The patch doesn't appear to include any new or modified tests. + Please justify why no new tests are needed for this patch. + Also please list what manual steps were performed to verify this patch." + return 1 + fi + JIRA_COMMENT="$JIRA_COMMENT + + {color:green}+1 tests included{color}. The patch appears to include $testReferences new or modified test files." + return 0 +} + +cleanUpXml () { + cd $BASEDIR/conf + for file in `ls *.xml.template` + do + rm -f `basename $file .template` + done + cd $BASEDIR +} + +############################################################################### +### Attempt to apply the patch +applyPatch () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Applying patch." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + export PATCH + # $bindir/smart-apply-patch.sh $PATCH_DIR/patch + $GIT apply $PATCH_DIR/patch + if [[ $? != 0 ]] ; then + echo "PATCH APPLICATION FAILED" + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 patch{color}. The patch command could not apply the patch." + return 1 + fi + return 0 +} + +############################################################################### +calculateJavadocWarnings() { + WARNING_FILE="$1" + RET=$(egrep "^[0-9]+ warnings$" "$WARNING_FILE" | awk '{sum+=$1} END {print sum}') +} + +### Check there are no javadoc warnings +checkJavadocWarnings () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Determining number of patched javadoc warnings." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + echo "$MVN clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1" + if [ -d hadoop-project ]; then + (cd hadoop-project; $MVN install > /dev/null 2>&1) + fi + if [ -d hadoop-common-project/hadoop-annotations ]; then + (cd hadoop-common-project/hadoop-annotations; $MVN install > /dev/null 2>&1) + fi + $MVN clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1 + calculateJavadocWarnings "$PATCH_DIR/trunkJavadocWarnings.txt" + numTrunkJavadocWarnings=$RET + calculateJavadocWarnings "$PATCH_DIR/patchJavadocWarnings.txt" + numPatchJavadocWarnings=$RET + grep -i warning "$PATCH_DIR/trunkJavadocWarnings.txt" > "$PATCH_DIR/trunkJavadocWarningsFiltered.txt" + grep -i warning "$PATCH_DIR/patchJavadocWarnings.txt" > "$PATCH_DIR/patchJavadocWarningsFiltered.txt" + diff -u "$PATCH_DIR/trunkJavadocWarningsFiltered.txt" \ + "$PATCH_DIR/patchJavadocWarningsFiltered.txt" > \ + "$PATCH_DIR/diffJavadocWarnings.txt" + rm -f "$PATCH_DIR/trunkJavadocWarningsFiltered.txt" "$PATCH_DIR/patchJavadocWarningsFiltered.txt" + echo "There appear to be $numTrunkJavadocWarnings javadoc warnings before the patch and $numPatchJavadocWarnings javadoc warnings after applying the patch." + if [[ $numTrunkJavadocWarnings != "" && $numPatchJavadocWarnings != "" ]] ; then + if [[ $numPatchJavadocWarnings -gt $numTrunkJavadocWarnings ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 javadoc{color}. The javadoc tool appears to have generated `expr $(($numPatchJavadocWarnings-$numTrunkJavadocWarnings))` warning messages. + See $BUILD_URL/artifact/trunk/patchprocess/diffJavadocWarnings.txt for details." + return 1 + fi + fi + JIRA_COMMENT="$JIRA_COMMENT + + {color:green}+1 javadoc{color}. There were no new javadoc warning messages." + return 0 +} + +############################################################################### +### Check there are no changes in the number of Javac warnings +checkJavacWarnings () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Determining number of patched javac warnings." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + echo "$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavacWarnings.txt 2>&1" + $MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavacWarnings.txt 2>&1 + if [[ $? != 0 ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 javac{color:red}. The patch appears to cause the build to fail." + return 2 + fi + ### Compare trunk and patch javac warning numbers + if [[ -f $PATCH_DIR/patchJavacWarnings.txt ]] ; then + $GREP '\[WARNING\]' $PATCH_DIR/trunkJavacWarnings.txt > $PATCH_DIR/filteredTrunkJavacWarnings.txt + $GREP '\[WARNING\]' $PATCH_DIR/patchJavacWarnings.txt > $PATCH_DIR/filteredPatchJavacWarnings.txt + trunkJavacWarnings=`cat $PATCH_DIR/filteredTrunkJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` + patchJavacWarnings=`cat $PATCH_DIR/filteredPatchJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` + echo "There appear to be $trunkJavacWarnings javac compiler warnings before the patch and $patchJavacWarnings javac compiler warnings after applying the patch." + if [[ $patchJavacWarnings != "" && $trunkJavacWarnings != "" ]] ; then + if [[ $patchJavacWarnings -gt $trunkJavacWarnings ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 javac{color}. The applied patch generated $patchJavacWarnings javac compiler warnings (more than the trunk's current $trunkJavacWarnings warnings)." + + $DIFF $PATCH_DIR/filteredTrunkJavacWarnings.txt $PATCH_DIR/filteredPatchJavacWarnings.txt > $PATCH_DIR/diffJavacWarnings.txt + JIRA_COMMENT_FOOTER="Javac warnings: $BUILD_URL/artifact/trunk/patchprocess/diffJavacWarnings.txt +$JIRA_COMMENT_FOOTER" + + return 1 + fi + fi + fi + JIRA_COMMENT="$JIRA_COMMENT + + {color:green}+1 javac{color}. The applied patch does not increase the total number of javac compiler warnings." + return 0 +} + +############################################################################### +### Check there are no changes in the number of release audit (RAT) warnings +checkReleaseAuditWarnings () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Determining number of patched release audit warnings." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + echo "$MVN apache-rat:check -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchReleaseAuditOutput.txt 2>&1" + $MVN apache-rat:check -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchReleaseAuditOutput.txt 2>&1 + find $BASEDIR -name rat.txt | xargs cat > $PATCH_DIR/patchReleaseAuditWarnings.txt + + ### Compare trunk and patch release audit warning numbers + if [[ -f $PATCH_DIR/patchReleaseAuditWarnings.txt ]] ; then + patchReleaseAuditWarnings=`$GREP -c '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt` + echo "" + echo "" + echo "There appear to be $patchReleaseAuditWarnings release audit warnings after applying the patch." + if [[ $patchReleaseAuditWarnings != "" ]] ; then + if [[ $patchReleaseAuditWarnings -gt 0 ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 release audit{color}. The applied patch generated $patchReleaseAuditWarnings release audit warnings." + $GREP '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt > $PATCH_DIR/patchReleaseAuditProblems.txt + echo "Lines that start with ????? in the release audit report indicate files that do not have an Apache license header." >> $PATCH_DIR/patchReleaseAuditProblems.txt + JIRA_COMMENT_FOOTER="Release audit warnings: $BUILD_URL/artifact/trunk/patchprocess/patchReleaseAuditProblems.txt +$JIRA_COMMENT_FOOTER" + return 1 + fi + fi + fi + JIRA_COMMENT="$JIRA_COMMENT + + {color:green}+1 release audit{color}. The applied patch does not increase the total number of release audit warnings." + return 0 +} + +############################################################################### +### Check there are no changes in the number of Checkstyle warnings +checkStyle () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Determining number of patched checkstyle warnings." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + echo "THIS IS NOT IMPLEMENTED YET" + echo "" + echo "" + echo "$MVN test checkstyle:checkstyle -DskipTests -D${PROJECT_NAME}PatchProcess" + $MVN test checkstyle:checkstyle -DskipTests -D${PROJECT_NAME}PatchProcess + + JIRA_COMMENT_FOOTER="Checkstyle results: $BUILD_URL/artifact/trunk/build/test/checkstyle-errors.html +$JIRA_COMMENT_FOOTER" + ### TODO: calculate actual patchStyleErrors +# patchStyleErrors=0 +# if [[ $patchStyleErrors != 0 ]] ; then +# JIRA_COMMENT="$JIRA_COMMENT +# +# {color:red}-1 checkstyle{color}. The patch generated $patchStyleErrors code style errors." +# return 1 +# fi +# JIRA_COMMENT="$JIRA_COMMENT +# +# {color:green}+1 checkstyle{color}. The patch generated 0 code style errors." + return 0 +} + +############################################################################### +### Install the new jars so tests and findbugs can find all of the updated jars +buildAndInstall () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Installing all of the jars" + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + echo "$MVN install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess" + $MVN install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess + return $? +} + + +############################################################################### +### Check there are no changes in the number of Findbugs warnings +checkFindbugsWarnings () { + findbugs_version=`${FINDBUGS_HOME}/bin/findbugs -version` + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Determining number of patched Findbugs warnings." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + + modules=$(findModules) + rc=0 + for module in $modules; + do + cd $module + echo " Running findbugs in $module" + module_suffix=`basename ${module}` + echo "$MVN clean test findbugs:findbugs -DskipTests -D${PROJECT_NAME}PatchProcess < /dev/null > $PATCH_DIR/patchFindBugsOutput${module_suffix}.txt 2>&1" + $MVN clean test findbugs:findbugs -DskipTests -D${PROJECT_NAME}PatchProcess < /dev/null > $PATCH_DIR/patchFindBugsOutput${module_suffix}.txt 2>&1 + (( rc = rc + $? )) + cd - + done + + if [ $rc != 0 ] ; then + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 findbugs{color}. The patch appears to cause Findbugs (version ${findbugs_version}) to fail." + return 1 + fi + + findbugsWarnings=0 + for file in $(find $BASEDIR -name findbugsXml.xml) + do + relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix + if [ ! $relative_file == "target/findbugsXml.xml" ]; then + module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path + module_suffix=`basename ${module_suffix}` + fi + + cp $file $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml + $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \ + $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \ + $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml + newFindbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \ + $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml | $AWK '{print $1}'` + echo "Found $newFindbugsWarnings Findbugs warnings ($file)" + findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings)) + $FINDBUGS_HOME/bin/convertXmlToText -html \ + $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \ + $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html + if [[ $newFindbugsWarnings > 0 ]] ; then + JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/trunk/patchprocess/newPatchFindbugsWarnings${module_suffix}.html +$JIRA_COMMENT_FOOTER" + fi + done + + if [[ $findbugsWarnings -gt 0 ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 findbugs{color}. The patch appears to introduce $findbugsWarnings new Findbugs (version ${findbugs_version}) warnings." + return 1 + fi + JIRA_COMMENT="$JIRA_COMMENT + + {color:green}+1 findbugs{color}. The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings." + return 0 +} + +############################################################################### +### Verify eclipse:eclipse works +checkEclipseGeneration () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Running mvn eclipse:eclipse." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + + echo "$MVN eclipse:eclipse -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchEclipseOutput.txt 2>&1" + $MVN eclipse:eclipse -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchEclipseOutput.txt 2>&1 + if [[ $? != 0 ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 eclipse:eclipse{color}. The patch failed to build with eclipse:eclipse." + return 1 + fi + JIRA_COMMENT="$JIRA_COMMENT + + {color:green}+1 eclipse:eclipse{color}. The patch built with eclipse:eclipse." + return 0 +} + + + +############################################################################### +### Run the tests +runTests () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Running tests." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + + failed_tests="" + modules=$(findModules) + # + # If we are building hadoop-hdfs-project, we must build the native component + # of hadoop-common-project first. In order to accomplish this, we move the + # hadoop-hdfs subprojects to the end of the list so that common will come + # first. + # + # Of course, we may not be building hadoop-common at all-- in this case, we + # explicitly insert a mvn compile -Pnative of common, to ensure that the + # native libraries show up where we need them. + # + building_common=0 + for module in $modules; do + if [[ $module == hadoop-hdfs-project* ]]; then + hdfs_modules="$hdfs_modules $module" + elif [[ $module == hadoop-common-project* ]]; then + ordered_modules="$ordered_modules $module" + building_common=1 + else + ordered_modules="$ordered_modules $module" + fi + done + if [ -n "$hdfs_modules" ]; then + ordered_modules="$ordered_modules $hdfs_modules" + if [[ $building_common -eq 0 ]]; then + echo " Building hadoop-common with -Pnative in order to provide \ +libhadoop.so to the hadoop-hdfs unit tests." + echo " $MVN compile -D${PROJECT_NAME}PatchProcess" + if ! $MVN compile -D${PROJECT_NAME}PatchProcess; then + JIRA_COMMENT="$JIRA_COMMENT + {color:red}-1 core tests{color}. Failed to build the native portion \ +of hadoop-common prior to running the unit tests in $ordered_modules" + return 1 + fi + fi + fi + failed_test_builds="" + test_timeouts="" + for module in $ordered_modules; do + cd $module + module_suffix=`basename ${module}` + test_logfile=$PATCH_DIR/testrun_${module_suffix}.txt + echo " Running tests in $module" + echo " $MVN clean install -fn -D${PROJECT_NAME}PatchProcess" + $MVN clean install -fae -D${PROJECT_NAME}PatchProcess > $test_logfile 2>&1 + test_build_result=$? + cat $test_logfile + # module_test_timeouts=`$AWK '/^Running / { if (last) { print last } last=$2 } /^Tests run: / { last="" }' $test_logfile` + module_test_timeouts="" + if [[ -n "$module_test_timeouts" ]] ; then + test_timeouts="$test_timeouts +$module_test_timeouts" + fi + module_failed_tests=`find . -name 'TEST*.xml' | xargs $GREP -l -E " $TMP + + # if all of the lines start with a/ or b/, then this is a git patch that + # was generated without --no-prefix + if ! $GREP -qv '^a/\|^b/' $TMP ; then + sed -i -e 's,^[ab]/,,' $TMP + fi + + # Now find all the modules that were changed + TMP_MODULES=/tmp/tmp.modules.$$ + for file in $(cut -f 1 $TMP | sort | uniq); do + echo $(findModule $file) >> $TMP_MODULES + done + rm $TMP + + # Filter out modules without code + CHANGED_MODULES="" + for module in $(cat $TMP_MODULES | sort | uniq); do + $GREP "pom" $module/pom.xml > /dev/null + if [ "$?" != 0 ]; then + CHANGED_MODULES="$CHANGED_MODULES $module" + fi + done + rm $TMP_MODULES + echo $CHANGED_MODULES +} +############################################################################### +### Run the test-contrib target +runContribTests () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Running contrib tests." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + + if [[ `$GREP -c 'test-contrib' build.xml` == 0 ]] ; then + echo "No contrib tests in this project." + return 0 + fi + + ### Kill any rogue build processes from the last attempt + $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null + + #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no test-contrib" + #$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no test-contrib + echo "NOP" + if [[ $? != 0 ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 contrib tests{color}. The patch failed contrib unit tests." + return 1 + fi + JIRA_COMMENT="$JIRA_COMMENT + + {color:green}+1 contrib tests{color}. The patch passed contrib unit tests." + return 0 +} + +############################################################################### +### Run the inject-system-faults target +checkInjectSystemFaults () { + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Checking the integrity of system test framework code." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + + ### Kill any rogue build processes from the last attempt + $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null + + #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults" + #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults + echo "NOP" + return 0 + if [[ $? != 0 ]] ; then + JIRA_COMMENT="$JIRA_COMMENT + + {color:red}-1 system test framework{color}. The patch failed system test framework compile." + return 1 + fi + JIRA_COMMENT="$JIRA_COMMENT + + {color:green}+1 system test framework{color}. The patch passed system test framework compile." + return 0 +} + +############################################################################### +### Submit a comment to the defect's Jira +submitJiraComment () { + local result=$1 + ### Do not output the value of JIRA_COMMENT_FOOTER when run by a developer + if [[ $JENKINS == "false" ]] ; then + JIRA_COMMENT_FOOTER="" + fi + if [[ $result == 0 ]] ; then + comment="{color:green}+1 overall{color}. $JIRA_COMMENT + +$JIRA_COMMENT_FOOTER" + else + comment="{color:red}-1 overall{color}. $JIRA_COMMENT + +$JIRA_COMMENT_FOOTER" + fi + ### Output the test result to the console + echo " + + + +$comment" + + if [[ $JENKINS == "true" ]] ; then + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Adding comment to Jira." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + ### Update Jira with a comment + export USER=hudson + $JIRACLI -s https://issues.apache.org/jira -a addcomment -u hadoopqa -p $JIRA_PASSWD --comment "$comment" --issue $defect + $JIRACLI -s https://issues.apache.org/jira -a logout -u hadoopqa -p $JIRA_PASSWD + fi +} + +############################################################################### +### Cleanup files +cleanupAndExit () { + local result=$1 + echo "" + echo "" + echo "======================================================================" + echo "======================================================================" + echo " Finished build." + echo "======================================================================" + echo "======================================================================" + echo "" + echo "" + exit $result +} + +############################################################################### +############################################################################### +############################################################################### + +JIRA_COMMENT="" +JIRA_COMMENT_FOOTER="Console output: $BUILD_URL/console + +This message is automatically generated." + +### Check if arguments to the script have been specified properly or not +parseArgs $@ +cd $BASEDIR + +checkout +RESULT=$? +if [[ $JENKINS == "true" ]] ; then + if [[ $RESULT != 0 ]] ; then + exit 100 + fi +fi + +### Check if $PATCH_DIR exists. If it does not exist, create a new directory +if [[ ! -e "$PATCH_DIR" ]] ; then + mkdir "$PATCH_DIR" + if [[ $? == 0 ]] ; then + echo "$PATCH_DIR has been created" + else + echo "Unable to create $PATCH_DIR" + cleanupAndExit 0 + fi +fi + +downloadPatch +verifyPatch +(( RESULT = RESULT + $? )) +if [[ $RESULT != 0 ]] ; then + submitJiraComment 1 + cleanupAndExit 1 +fi +prebuildWithoutPatch +(( RESULT = RESULT + $? )) +if [[ $RESULT != 0 ]] ; then + submitJiraComment 1 + cleanupAndExit 1 +fi +checkAuthor +(( RESULT = RESULT + $? )) + +# if [[ $JENKINS == "true" ]] ; then + # cleanUpXml +# fi + +echo "before checkTests $RESULT" + +checkTests +(( RESULT = RESULT + $? )) +applyPatch +APPLY_PATCH_RET=$? +(( RESULT = RESULT + $APPLY_PATCH_RET )) + +echo "after applyPatch $RESULT" + +if [[ $APPLY_PATCH_RET != 0 ]] ; then + submitJiraComment 1 + cleanupAndExit 1 +fi +checkJavacWarnings +JAVAC_RET=$? +#2 is returned if the code could not compile +if [[ $JAVAC_RET == 2 ]] ; then + submitJiraComment 1 + cleanupAndExit 1 +fi +(( RESULT = RESULT + $JAVAC_RET )) +checkJavadocWarnings +(( RESULT = RESULT + $? )) +### Checkstyle not implemented yet +#checkStyle +#(( RESULT = RESULT + $? )) + +echo "before buildAndInstall $RESULT" + +buildAndInstall +# checkEclipseGeneration +# (( RESULT = RESULT + $? )) +# checkFindbugsWarnings +# (( RESULT = RESULT + $? )) + +echo "before checkReleaseAuditWarnings $RESULT" + +checkReleaseAuditWarnings +(( RESULT = RESULT + $? )) + +echo "before runTests $RESULT" +### Run tests for Jenkins or if explictly asked for by a developer +if [[ $JENKINS == "true" || $RUN_TESTS == "true" ]] ; then + runTests + (( RESULT = RESULT + $? )) + # runContribTests + # (( RESULT = RESULT + $? )) +fi +checkInjectSystemFaults +(( RESULT = RESULT + $? )) +JIRA_COMMENT_FOOTER="Test results: $BUILD_URL/testReport/ +$JIRA_COMMENT_FOOTER" + +submitJiraComment $RESULT +cleanupAndExit $RESULT