hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ni...@apache.org
Subject svn commit: r575463 - in /lucene/hadoop/nightly: hudsonBuildHadoopPatch.sh processHadoopPatchEmail.sh
Date Thu, 13 Sep 2007 21:27:34 GMT
Author: nigel
Date: Thu Sep 13 14:27:34 2007
New Revision: 575463

URL: http://svn.apache.org/viewvc?rev=575463&view=rev
Log:
Add hudson patch build scripts to svn.  Contributed by nigel.

Added:
    lucene/hadoop/nightly/hudsonBuildHadoopPatch.sh   (with props)
    lucene/hadoop/nightly/processHadoopPatchEmail.sh   (with props)

Added: lucene/hadoop/nightly/hudsonBuildHadoopPatch.sh
URL: http://svn.apache.org/viewvc/lucene/hadoop/nightly/hudsonBuildHadoopPatch.sh?rev=575463&view=auto
==============================================================================
--- lucene/hadoop/nightly/hudsonBuildHadoopPatch.sh (added)
+++ lucene/hadoop/nightly/hudsonBuildHadoopPatch.sh Thu Sep 13 14:27:34 2007
@@ -0,0 +1,408 @@
+#!/bin/bash
+
+#set -x
+
+### Setup global variables.  BUILD_ID is set by Hudson
+baseDir=/export/home/hudson/hudson/jobs/Hadoop-Patch
+GREP=/usr/local/bin/grep
+WGET=/usr/sfw/bin/wget
+PATCH=/usr/local/bin/patch
+PS=/usr/ucb/ps
+JIRA=/export/home/hudson/tools/jira_cli/src/cli/jira
+FINDBUGS_HOME=/export/home/hudson/tools/findbugs/latest
+patchNum=""
+revisionNum=""
+JIRA_COMMENT=""
+JIRA_COMMENT_FOOTER="Console output: http://lucene.zones.apache.org:8080/hudson/job/Hadoop-Patch/$BUILD_NUMBER/console
+
+This message is automatically generated."
+
+### Retrieve the defect number
+if [ ! -e $baseDir/current/defectNum ] ; then
+  echo "Could not determine the patch to test.  Exiting."
+  cleanupAndExit 0
+fi
+defect=`cat $baseDir/current/defectNum`
+if [ -z "$defect" ] ; then
+  echo "Could not determine the patch to test.  Exiting."
+  cleanupAndExit 0
+fi
+
+###############################################################################
+setup () {
+  echo ""
+  echo ""
+  echo "======================================================================"
+  echo "======================================================================"
+  echo "    Testing patch for ${defect}."
+  echo "======================================================================"
+  echo "======================================================================"
+  echo ""
+  echo ""
+
+  ### Download latest patch file (ignoring .htm and .html)
+  $WGET -q -O $baseDir/current/jira http://issues.apache.org/jira/browse/$defect
+  if [[ `$GREP -c 'Patch Available' $baseDir/current/jira` == 0 ]] ; then
+    echo "$defect is not \"Patch Available\".  Exiting."
+    cleanupAndExit 0
+  fi
+  relativePatchURL=`$GREP -o '"/jira/secure/attachment/[0-9]*/[^"]*' $baseDir/current/jira
| $GREP -v -e 'htm[l]*$' | sort | tail -1 | $GREP -o '/jira/secure/attachment/[0-9]*/[^"]*'`
+  patchURL="http://issues.apache.org${relativePatchURL}"
+  patchNum=`echo $patchURL | $GREP -o '[0-9]*/' | $GREP -o '[0-9]*'`
+  echo "$defect patch is being downloaded at `date` from"
+  echo "$patchURL"
+  $WGET -q -O $baseDir/current/patch $patchURL
+
+  ### Retrieve Subversion revision number
+  cd $baseDir/workspace/trunk
+  revisionNum=`svn log --limit 1 -q . | $GREP -o "^r[0-9]*"`
+  JIRA_COMMENT="Here are the results of testing the latest attachment 
+$patchURL
+against trunk revision $revisionNum."
+
+  ### Copy in any supporting files needed by this process
+  cp -r $baseDir/lib/* ./lib
+  #PENDING: cp -f $baseDir/bin/checkstyle* ./src/test
+
+  echo ""
+  echo ""
+  echo "======================================================================"
+  echo "======================================================================"
+  echo "    Pre-building trunk to determine current"
+  echo "    number of javac and Findbugs warnings."
+  echo "======================================================================"
+  echo "======================================================================"
+  echo ""
+  echo ""
+  $ANT_HOME/bin/ant -Dversion=${revisionNum}_${defect}_PATCH-${patchNum} -Dfindbugs.home=$FINDBUGS_HOME
-Djavac.args="-Xlint -Xmaxwarns 1000" -DHadoopPatchProcess= clean tar findbugs &> $baseDir/current/currentJavacWarnings
+  cp $baseDir/workspace/trunk/build/test/findbugs/*.xml $baseDir/current/currentFindbugsWarnings.xml
+}
+
+###############################################################################
+### Check for @author tags in the patch
+checkAuthor () {
+  echo ""
+  echo ""
+  echo "======================================================================"
+  echo "======================================================================"
+  echo "    Checking there are no @author tags in the patch."
+  echo "======================================================================"
+  echo "======================================================================"
+  echo ""
+  echo ""
+  authorTags=`$GREP -c -i '@author' $baseDir/current/patch`
+  echo "There appear to be $authorTags @author tags in the patch."
+  if [[ $authorTags != 0 ]] ; then
+    JIRA_COMMENT="$JIRA_COMMENT
+
+    @author -1.  The patch appears to contain $authorTags @author tags which the Hadoop community
has agreed to not allow in code contributions."
+    return 1
+  fi
+  JIRA_COMMENT="$JIRA_COMMENT
+
+    @author +1.  The patch does not contain any @author tags."
+  return 0
+}
+
+###############################################################################
+### Attempt to apply the patch
+applyPatch () {
+  echo ""
+  echo ""
+  echo "======================================================================"
+  echo "======================================================================"
+  echo "    Applying patch."
+  echo "======================================================================"
+  echo "======================================================================"
+  echo ""
+  echo ""
+  $PATCH -E -p0 < $baseDir/current/patch
+  if [[ $? != 0 ]] ; then
+    echo "PATCH APPLICATION FAILED"
+    JIRA_COMMENT="$JIRA_COMMENT
+
+    patch -1.  The patch command could not apply the patch."
+    return 1
+  fi
+  return 0
+}
+
+###############################################################################
+### Check there are no javadoc warnings
+checkJavadocWarnings () {
+  echo ""
+  echo ""
+  echo "======================================================================"
+  echo "======================================================================"
+  echo "    Determining number of patched javadoc warnings."
+  echo "======================================================================"
+  echo "======================================================================"
+  echo ""
+  echo ""
+  $ANT_HOME/bin/ant -Dversion=${revisionNum}_${defect}_PATCH-${patchNum} -DHadoopPatchProcess=
clean javadoc
+  javadocWarnings=`$GREP -c '\[javadoc\] [0-9]* warning' $baseDir/builds/$BUILD_ID/log`
+  echo ""
+  echo ""
+  echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build."
+  if [[ $javadocWarnings != 0 ]] ; then
+    JIRA_COMMENT="$JIRA_COMMENT
+
+    javadoc -1.  The javadoc tool appears to have generated $javadocWarningswarning messages."
+    return 1
+  fi
+  JIRA_COMMENT="$JIRA_COMMENT
+
+    javadoc +1.  The javadoc tool did not generate any warning messages."
+return 0
+}
+
+###############################################################################
+### Check there are no changes in the number of Javac warnings
+checkJavacWarnings () {
+  echo ""
+  echo ""
+  echo "======================================================================"
+  echo "======================================================================"
+  echo "    Determining number of patched javac warnings."
+  echo "======================================================================"
+  echo "======================================================================"
+  echo ""
+  echo ""
+  $ANT_HOME/bin/ant -Dversion=${revisionNum}_${defect}_PATCH-${patchNum} -Djavac.args="-Xlint
-Xmaxwarns 1000" -DHadoopPatchProcess= tar &> $baseDir/current/patchJavacWarnings
+
+  ### Compare current and patch javac warning numbers
+  if [[ -f $baseDir/javacWarnings ]] ; then
+    currentJavacWarnings=`$GREP -o '\[javac\] [0-9]* warning' $baseDir/current/currentJavacWarnings
| awk '{total += $2} END {print total}'`
+    patchJavacWarnings=`$GREP -o '\[javac\] [0-9]* warning' $baseDir/current/patchJavacWarnings
| awk '{total += $2} END {print total}'`
+    echo ""
+    echo ""
+    echo "There appear to be $currentJavacWarnings javac compiler warnings before the patch
and $patchJavacWarnings javac compiler warnings after applying the patch."
+    if [[ $patchJavacWarnings != "" && $currentJavacWarnings != "" ]] ; then
+      if [[ $patchJavacWarnings > $currentJavacWarnings ]] ; then
+        JIRA_COMMENT="$JIRA_COMMENT
+
+    javac -1.  The applied patch generated $patchJavacWarnings javac compiler warnings (more
than the trunk's current $currentJavacWarnings warnings)."
+        return 1
+      fi
+    fi
+  fi
+  JIRA_COMMENT="$JIRA_COMMENT
+
+    javac +1.  The applied patch does not generate any new compiler warnings."
+  return 0
+}
+
+###############################################################################
+### Check there are no changes in the number of Checkstyle warnings
+checkStyle () {
+  echo ""
+  echo ""
+  echo "======================================================================"
+  echo "======================================================================"
+  echo "    Determining number of patched checkstyle warnings."
+  echo "======================================================================"
+  echo "======================================================================"
+  echo ""
+  echo ""
+  echo "THIS IS NOT IMPLEMENTED YET"
+  echo ""
+  echo ""
+  $ANT_HOME/bin/ant -Dversion=${revisionNum}_${defect}_PATCH-${patchNum} -DHadoopPatchProcess=
checkstyle
+  JIRA_COMMENT_FOOTER="Checkstyle results: http://lucene.zones.apache.org:8080/hudson/job/Hadoop-Patch/$BUILD_NUMBER/artifact/trunk/build/test/checkstyle-errors.html
+$JIRA_COMMENT_FOOTER"
+  ### TODO: calculate actual patchStyleErrors
+#  patchStyleErrors=0
+#  if [[ $patchStyleErrors != 0 ]] ; then
+#    JIRA_COMMENT="$JIRA_COMMENT
+#
+#    checkstyle -1.  The patch generated $patchStyleErrors code style errors."
+#    return 1
+#  fi
+#  JIRA_COMMENT="$JIRA_COMMENT
+#
+#    checkstyle +1.  The patch generated 0 code style errors."
+  return 0
+}
+
+###############################################################################
+### Check there are no changes in the number of Findbugs warnings
+checkFindbugsWarnings () {
+  echo ""
+  echo ""
+  echo "======================================================================"
+  echo "======================================================================"
+  echo "    Determining number of patched Findbugs warnings."
+  echo "======================================================================"
+  echo "======================================================================"
+  echo ""
+  echo ""
+  $ANT_HOME/bin/ant -Dversion=${revisionNum}_${defect}_PATCH-${patchNum} -Dfindbugs.home=$FINDBUGS_HOME
-DHadoopPatchProcess= findbugs 
+JIRA_COMMENT_FOOTER="Findbugs warnings: http://lucene.zones.apache.org:8080/hudson/job/Hadoop-Patch/$BUILD_NUMBER/artifact/trunk/build/test/findbugs/newPatchFindbugsWarnings.html
+$JIRA_COMMENT_FOOTER"
+  cp $baseDir/workspace/trunk/build/test/findbugs/*.xml $baseDir/current/patchFindbugsWarnings.xml
+  $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/1999" \
+    $baseDir/current/currentFindbugsWarnings.xml \
+    $baseDir/current/currentFindbugsWarnings.xml
+  $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \
+    $baseDir/current/patchFindbugsWarnings.xml \
+    $baseDir/current/patchFindbugsWarnings.xml
+  $FINDBUGS_HOME/bin/computeBugHistory -output $baseDir/current/findbugsMerge.xml \
+    $baseDir/current/currentFindbugsWarnings.xml \
+    $baseDir/current/patchFindbugsWarnings.xml
+  findbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $baseDir/current/findbugsMerge.xml
\
+    $baseDir/workspace/trunk/build/test/findbugs/newPatchFindbugsWarnings.xml | /usr/bin/nawk
'{print $1}'`
+  $FINDBUGS_HOME/bin/convertXmlToText -html \
+    $baseDir/workspace/trunk/build/test/findbugs/newPatchFindbugsWarnings.xml \
+    $baseDir/workspace/trunk/build/test/findbugs/newPatchFindbugsWarnings.html
+  if [[ $findbugsWarnings != 0 ]] ; then
+    JIRA_COMMENT="$JIRA_COMMENT
+
+    findbugs -1.  The patch appears to introduce $findbugsWarnings new Findbugs warnings."
+    return 1
+  fi
+  JIRA_COMMENT="$JIRA_COMMENT
+
+    findbugs +1.  The patch does not introduce any new Findbugs warnings."
+  return 0
+}
+
+###############################################################################
+### Run the test-core target
+runCoreTests () {
+  echo ""
+  echo ""
+  echo "======================================================================"
+  echo "======================================================================"
+  echo "    Running core tests."
+  echo "======================================================================"
+  echo "======================================================================"
+  echo ""
+  echo ""
+  ### Kill any rogue build processes from the last attempt
+  $PS -auxwww | $GREP HadoopPatchProcess | /usr/bin/nawk '{print $2}' | /usr/bin/xargs -t
-I {} /usr/bin/kill -9 {} > /dev/null
+
+  $ANT_HOME/bin/ant -Dversion=${revisionNum}_${defect}_PATCH-${patchNum} -DHadoopPatchProcess=
-Dtest.junit.output.format=xml -Dtest.output=yes tar test-core
+  if [[ $? != 0 ]] ; then
+    JIRA_COMMENT="$JIRA_COMMENT
+
+    core tests -1.  The patch failed core unit tests."
+    return 1
+  fi
+  JIRA_COMMENT="$JIRA_COMMENT
+
+    core tests +1.  The patch passed core unit tests."
+  return 0
+}
+
+###############################################################################
+### Run the test-contrib target
+runContribTests () {
+  echo ""
+  echo ""
+  echo "======================================================================"
+  echo "======================================================================"
+  echo "    Running contrib tests."
+  echo "======================================================================"
+  echo "======================================================================"
+  echo ""
+  echo ""
+  ### Kill any rogue build processes from the last attempt
+  $PS -auxwww | $GREP HadoopPatchProcess | /usr/bin/nawk '{print $2}' | /usr/bin/xargs -t
-I {} /usr/bin/kill -9 {} > /dev/null
+
+  $ANT_HOME/bin/ant -Dversion=${revisionNum}_${defect}_PATCH-${patchNum} -DHadoopPatchProcess=
-Dtest.junit.output.format=xml -Dtest.output=yes test-contrib
+  if [[ $? != 0 ]] ; then
+    JIRA_COMMENT="$JIRA_COMMENT
+
+    contrib tests -1.  The patch failed contrib unit tests."
+    return 1
+  fi
+  JIRA_COMMENT="$JIRA_COMMENT
+
+    contrib tests +1.  The patch passed contrib unit tests."
+  return 0
+}
+
+###############################################################################
+### Submit a comment to the defect's Jira
+submitJiraComment () {
+  local result=$1
+  if [[ $result == 0 ]] ; then
+    comment="+1 overall.  $JIRA_COMMENT
+
+$JIRA_COMMENT_FOOTER"
+  else
+    comment="-1 overall.  $JIRA_COMMENT
+
+$JIRA_COMMENT_FOOTER"
+  fi
+  echo ""
+  echo ""
+  echo "======================================================================"
+  echo "======================================================================"
+  echo "    Adding comment to Jira."
+  echo "======================================================================"
+  echo "======================================================================"
+  echo ""
+  echo ""
+  echo "$comment"
+
+  ### Update Jira with a comment
+  export USER=hudson
+#  $JIRA -s issues.apache.org/jira login hadoopqa `cat ~/private/hadoopqaJiraPassword`
+#  $JIRA -s issues.apache.org/jira comment $defect "$comment"
+}
+
+###############################################################################
+### Cleanup files
+cleanupAndExit () {
+  local result=$1
+  if [ ! -z "$defect" ] ; then
+    if [ -e $baseDir/$defect ] ; then
+      rm -rf $baseDir/$defect
+    fi
+  fi
+  if [ -e $baseDir/current ] ; then
+    rm -f $baseDir/current
+  fi
+  echo ""
+  echo ""
+  echo "======================================================================"
+  echo "======================================================================"
+  echo "    Finished build."
+  echo "======================================================================"
+  echo "======================================================================"
+  echo ""
+  echo ""
+  exit $result
+}
+
+###############################################################################
+###############################################################################
+###############################################################################
+
+setup
+checkAuthor
+RESULT=$?
+applyPatch
+if [[ $? != 0 ]] ; then
+  submitJiraComment 1
+  cleanupAndExit $?
+fi
+checkJavadocWarnings
+(( RESULT = RESULT + $? ))
+checkJavacWarnings
+(( RESULT = RESULT + $? ))
+checkStyle
+(( RESULT = RESULT + $? ))
+checkFindbugsWarnings
+(( RESULT = RESULT + $? ))
+runCoreTests
+(( RESULT = RESULT + $? ))
+runContribTests
+(( RESULT = RESULT + $? ))
+JIRA_COMMENT_FOOTER="Test results: http://lucene.zones.apache.org:8080/hudson/job/Hadoop-Patch/$BUILD_NUMBER/testReport/
+$JIRA_COMMENT_FOOTER"
+
+submitJiraComment $RESULT
+cleanupAndExit $RESULT
+

Propchange: lucene/hadoop/nightly/hudsonBuildHadoopPatch.sh
------------------------------------------------------------------------------
    svn:executable = *

Added: lucene/hadoop/nightly/processHadoopPatchEmail.sh
URL: http://svn.apache.org/viewvc/lucene/hadoop/nightly/processHadoopPatchEmail.sh?rev=575463&view=auto
==============================================================================
--- lucene/hadoop/nightly/processHadoopPatchEmail.sh (added)
+++ lucene/hadoop/nightly/processHadoopPatchEmail.sh Thu Sep 13 14:27:34 2007
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+#set -x
+
+baseDir=/export/home/hudson/hudson/jobs/Hadoop-Patch
+LOG=$baseDir/log.txt
+
+### Scan email
+while read line
+do
+  ### Look for first occurance of issue number
+  if [[ `echo $line | /usr/local/bin/grep -c "HADOOP-"` == 1 ]] ; then
+    if [[ -z $defect ]] ; then
+      defect=`expr "$line" : '.*\(HADOOP-[0-9]*\)'`
+    fi
+  fi
+  ### Check to see if this issue was just made "Patch Available"
+  if [[ `echo $line | /usr/local/bin/grep -c "Status: Patch Available"` == 1 ]] ; then
+    patch=true
+    break
+  fi
+done
+
+### If this email indicates a new patch, start a build
+if [[ -n $patch && ! -d $baseDir/$defect ]] ; then
+  echo "$defect is being processed at `date`" >> $LOG
+  mkdir $baseDir/$defect
+
+  ### Write the defect number to a file so buildTest.sh 
+  ### knows which patch to test.
+  echo $defect > $baseDir/$defect/defectNum
+
+  ### Write the process number of this script to a file so
+  ### it can be killed by a human if necessary.
+  echo $$ > $baseDir/$defect/emailProcessingProcessNumber
+
+  ### Since this script is run by the 'daemon' user by sendmail,
+  ### make sure everything it creates is group writable
+  chmod -R g+w $baseDir/$defect
+
+  ### Wait here until the current build is complete.
+  ### The 'current' link provides a locking mechanism 
+  ### between this script and the buildTest.sh script.
+  while [[ -L $baseDir/current ]] ; do
+    sleep 59
+  done
+
+  ### check to make sure the defect dir still exists; this allows
+  ### an admin to delete the directory and bypass the testing
+  if [[ -d $baseDir/$defect ]] ; then
+    ### Create the 'current' link to this build
+    ln -s $baseDir/$defect $baseDir/current
+
+    ### Start build.  The 'token' is a secret.
+    echo "$defect patch submitted for testing at `date`" >> $LOG
+    token=`cat ~/private/hudsonBuildToken`
+    /usr/sfw/bin/wget -q -O $baseDir/$defect/build http://lucene.zones.apache.org:8080/hudson/job/Hadoop-Patch/build?token=$token
+    chmod -R g+w $baseDir/$defect
+  fi
+fi
+exit 0

Propchange: lucene/hadoop/nightly/processHadoopPatchEmail.sh
------------------------------------------------------------------------------
    svn:executable = *



Mime
View raw message