hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ni...@apache.org
Subject svn commit: r1026225 - /hadoop/nightly/hudsonPatchQueueAdmin.sh
Date Fri, 22 Oct 2010 06:49:58 GMT
Author: nigel
Date: Fri Oct 22 06:49:58 2010
New Revision: 1026225

URL: http://svn.apache.org/viewvc?rev=1026225&view=rev
Log:
HADOOP-7003: New script for managing queue of pre-commit patches that need testing.  Contributed
by nigel.

Modified:
    hadoop/nightly/hudsonPatchQueueAdmin.sh

Modified: hadoop/nightly/hudsonPatchQueueAdmin.sh
URL: http://svn.apache.org/viewvc/hadoop/nightly/hudsonPatchQueueAdmin.sh?rev=1026225&r1=1026224&r2=1026225&view=diff
==============================================================================
--- hadoop/nightly/hudsonPatchQueueAdmin.sh (original)
+++ hadoop/nightly/hudsonPatchQueueAdmin.sh Fri Oct 22 06:49:58 2010
@@ -1,6 +1,6 @@
-#!/bin/bash
+#!/bin/bash -e
 
-usage()
+usage() 
 {
 cat << EOF
 usage: $0 options
@@ -9,55 +9,38 @@ This script initates patch build's on Hu
 
 OPTIONS:
    -h      Show this message
-   -p      Project Name like HADOOP, PIG or ZOOPKEEPER.
-   -s      Build Server Names
-   -c      Called by values like USER OR Build Server name
-   -t      Path to curl binary
-   -S      Path to sed binary
-   -u      Build url token
-   -q      Path to the Queue Directory
-		### The directory to accumulate the patch queue.  Must be
-		### writable by this process.
+   -f      Url to Jira XML filter of patch available issues.
+   -c      Path to curl binary
+   -s      Path to sed binary
+   -t      Build token
    -v      Verbose
 EOF
 }
 
-PROJECT=
-QUEUE_DIR=
-BUILD_SERVERS=
-CALLED_BY=
+FILTER=
 CURL=
-BUILD_URL_TOKEN=
+BUILD_TOKEN=
 VERBOSE=
 SED= 
 
-while getopts "hp:s:c:t:u:q:S:v" OPTION
+while getopts "hf:t:s:u:v" OPTION
 do
      case $OPTION in
          h)
              usage
              exit 1
              ;;
-         p)
-             PROJECT=$OPTARG 
-             ;;
-         s)
-             BUILD_SERVERS=$OPTARG
+         f)
+             FILTER=$OPTARG 
              ;;
          c)
-             CALLED_BY=$OPTARG
-             ;;
-         t)
              CURL=$OPTARG
              ;;
-         S)
+         s)
              SED=$OPTARG
              ;;
-         u)
-             BUILD_URL_TOKEN=$OPTARG
-             ;;
-         q)
-             QUEUE_DIR=$OPTARG
+         t)
+             BUILD_TOKEN=$OPTARG
              ;;
          v)
              VERBOSE=1
@@ -69,72 +52,44 @@ do
      esac
 done
 
-if [[ -z $PROJECT ]] || [[ -z $BUILD_SERVERS ]] || [[ -z $CALLED_BY ]] || [[ -z $SED ]] ||
[[ -z $CURL ]] || [[ -z $BUILD_URL_TOKEN ]] || [[ -z $QUEUE_DIR ]]
-then
-     usage
-     exit 1
+if [[ -z $FILTER ]] || [[ -z $SED ]] || [[ -z $CURL ]] || [[ -z $BUILD_TOKEN ]] ; then
+  usage
+  exit 1
 fi
 
-if [[ $VERBOSE == 1 ]]; then
-set -x
+if [[ $VERBOSE == 1 ]] ; then
+  set -x
 fi
 
-PATCH_QUEUE=${QUEUE_DIR}/patch_queue.txt
-
-### To check if this patch_admin job called by test-patch.sh or hudson
-### if called by test-patch.sh then freee up the slae machine.
-if [[ $CALLED_BY != "" && $CALLED_BY != "USER" ]]; then
-  rm -rf ${QUEUE_DIR}/${CALLED_BY}
-fi
+### Grab the latest patch available query from Jira
+curl --fail --location --retry 3 --output patch_available.xml $FILTER
 
-QUEUE_HTML_FILE=`pwd`/${PROJECT}_PatchQueue.html
+### Grab all the key (issue numbers) and largest attachment id for each item in the XML
+xpath -q -e "//item/key/text() | //item/attachments/attachment[not(../attachment/@id >
@id)]/@id" patch_available.xml |tee patch_available2.elements
 
-echo "<html>" > $QUEUE_HTML_FILE
-echo "<title>Patch Queue for $PROJECT</title>" >> $QUEUE_HTML_FILE
-echo "<h3 align='left'><img src="http://hadoop.apache.org/core/images/hadoop-logo.jpg"
height="50"></img>Patch Queue for ${PROJECT}</h3>" >> $QUEUE_HTML_FILE
-echo "<hr style='height:2px;border-width:0;color:red;background-color:blue'>" >>
$QUEUE_HTML_FILE
-echo "<h4>Currently Running (or Waiting To Run)</h4>" >> $QUEUE_HTML_FILE
-echo "<table cellspacing=10><tr align=left><th>Issue</th><th>Submitted
to</th><th>Date Submitted to Run</th></tr>" >> $QUEUE_HTML_FILE
-
-for SLAVE in $BUILD_SERVERS 
-do 
-  TRIGGER_BUILD_URL=${HUDSON_URL}'job/'${PROJECT}'-Patch-'${SLAVE}${BUILD_URL_TOKEN}
-  CURRENT_PATCH=${QUEUE_DIR}/${SLAVE}
-  defect=`head -n 1 $PATCH_QUEUE | awk '{print $1}'`
-if [[ ! -f $CURRENT_PATCH ]]; then
-  if [[ $defect ]]; then
-    echo $defect > ${CURRENT_PATCH}
-      $SED -i '1d' $PATCH_QUEUE
-      echo "$defect patch submitted for testing at `date +"%b %a %H:%M"`"
-      $CURL "$TRIGGER_BUILD_URL&DEFECTNUM=$defect"
-      echo "<tr><td><a href=\"http://issues.apache.org/jira/browse/${defect}\">$defect</a></td><td>${SLAVE}</td><td>`date
+"%b %a %H:%M"` EST</td></tr>" >> $QUEUE_HTML_FILE	
-  else
-    CURRENT_PATCH="none"
-    CURRENT_PATCH_TIME="-"
-    echo "<tr><td>$CURRENT_PATCH</td><td>$CURRENT_PATCH_TIME</td></tr>"
>> $QUEUE_HTML_FILE
+### Replace newlines with nothing, then replace id=" with =, then replace " with newline
+### to yield lines with pairs (issueNumber,largestAttachmentId). Example: HADOOP-123,456984
+awk '{ printf "%s", $0 }' patch_available2.elements | sed -e "s/\W*id=\"/,/g" | sed -e "s/\"/\n/g"
|tee patch_available3.txt
+
+### Grab the latest patch_tested.txt file from this job on Hudson
+curl --fail --location --retry 3 --output patch_tested.txt ${JOB_URL}/lastSuccessfulBuild/artifact/patch_tested.txt
+
+### Iterate through issue list and find the (issueNumber,largestAttachmentId) pairs that
have 
+### not been tested (ie don't already exist in the patch_tested.txt file
+touch patch_tested.txt
+cat patch_available3.txt | while read PAIR ; do
+  set +e
+  COUNT=`grep -c "$PAIR" patch_tested.txt`
+  set -e
+  if [ "$COUNT" -lt "1" ] ; then
+    ### Parse $PAIR into project, issue number, and attachment id
+    PROJECT=`echo $PAIR | sed -e "s/-.*//g"`
+    ISSUE=`echo $PAIR | sed -e "s/.*-//g" | sed -e "s/,.*//g"`
+    ATTACHMENT=`echo $PAIR | sed -e "s/.*,//g"`
+    ### Kick off job
+    echo "Starting job $PROJECT with issue $ISSUE and attachment $ATTACHMENT"
+    curl --fail --location --retry 3 "${HUDSON_URL}/job/PreCommit-${PROJECT}-Build/buildWithParameters?token=${BUILD_TOKEN}&ISSUE_NUM=${ISSUE}&ATTACHMENT_ID=${ATTACHMENT}"
+    ### Mark this pair as tested by appending to file
+    echo "$PAIR" >> patch_tested.txt
   fi
-else
-  CURRENT_PATCH_TIME=`/bin/ls -dtl ${CURRENT_PATCH} | awk '{print $6" "$7" "$8" EST"}'`
-  EXEC_PATCH=`cat ${CURRENT_PATCH}`
-  echo "<tr><td><a href=\"http://issues.apache.org/jira/browse/$EXEC_PATCH\">$EXEC_PATCH</a></td><td>${SLAVE}</td><td>$CURRENT_PATCH_TIME</td></tr>"
>> $QUEUE_HTML_FILE
-fi
 done
-echo "</table>" >> $QUEUE_HTML_FILE
-echo "<hr style='height:2px;border-width:0;color:red;background-color:blue'>" >>
$QUEUE_HTML_FILE
-echo "<h4>Waiting in the Queue</h4>" >> $QUEUE_HTML_FILE
-echo "<table cellspacing=10><tr align=left><th>Issue</th><th>Date
Submitted</th></tr>" >> $QUEUE_HTML_FILE
-
-if [ -s $PATCH_QUEUE ]; then
-  while read PATCH_DTLS
-  do 
-  TEMP_QUEUE_DTLS=`echo $PATCH_DTLS | awk '{print "<tr><td><a href=\"http://issues.apache.org/jira/browse/"$1"\">"$1"</a></td><td>"$2"
"$3" "$4" "$5" EST</td></tr>"}'` 
-  echo $TEMP_QUEUE_DTLS >> $QUEUE_HTML_FILE  
-  done < "$PATCH_QUEUE"
-else 
-  echo "<tr><td>empty</td><td>-</td></tr>" >> $QUEUE_HTML_FILE
-fi
-
-echo "</table>" >> $QUEUE_HTML_FILE
-echo "<hr style='height:2px;border-width:0;color:red;background-color:blue'>" >>
$QUEUE_HTML_FILE
-echo "This file last updated at: `date`" >>$QUEUE_HTML_FILE
-echo "</html>" >> $QUEUE_HTML_FILE



Mime
View raw message