hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ni...@apache.org
Subject svn commit: r748037 - in /hadoop/core/nightly: hudsonPatchQueueAdmin.sh processHadoopPatchEmail.sh
Date Thu, 26 Feb 2009 06:34:09 GMT
Author: nigel
Date: Thu Feb 26 06:34:09 2009
New Revision: 748037

URL: http://svn.apache.org/viewvc?rev=748037&view=rev
Log:
HADOOP-5232. Enable patch testing to occur on more than one host. Contributed by Giri Kesavan.

Modified:
    hadoop/core/nightly/hudsonPatchQueueAdmin.sh
    hadoop/core/nightly/processHadoopPatchEmail.sh

Modified: hadoop/core/nightly/hudsonPatchQueueAdmin.sh
URL: http://svn.apache.org/viewvc/hadoop/core/nightly/hudsonPatchQueueAdmin.sh?rev=748037&r1=748036&r2=748037&view=diff
==============================================================================
--- hadoop/core/nightly/hudsonPatchQueueAdmin.sh (original)
+++ hadoop/core/nightly/hudsonPatchQueueAdmin.sh Thu Feb 26 06:34:09 2009
@@ -1,67 +1,136 @@
 #!/bin/bash
 
-#set -x
+usage()
+{
+cat << EOF
+usage: $0 options
+
+This script initates patch build's on Hudson slaves.
+
+OPTIONS:
+   -h      Show this message
+   -p      Project Name like HADOOP, PIG or ZOOPKEEPER.
+   -s      Build Server Names
+   -c      Called by values like USER OR Build Server name
+   -t      Path to curl binary
+   -u      Build url token
+   -q      Path to the Queue Directory
+		### The directory to accumulate the patch queue.  Must be
+		### writable by this process.
+   -v      Verbose
+EOF
+}
+
+PROJECT=
+QUEUE_DIR=
+BUILD_SERVERS=
+CALLED_BY=
+CURL=
+BUILD_URL_TOKEN=
+VERBOSE=
+
+while getopts "hp:s:c:t:u:q:v" OPTION
+do
+     case $OPTION in
+         h)
+             usage
+             exit 1
+             ;;
+         p)
+             PROJECT=$OPTARG 
+             ;;
+         s)
+             BUILD_SERVERS=$OPTARG
+             ;;
+         c)
+             CALLED_BY=$OPTARG
+             ;;
+         t)
+             CURL=$OPTARG
+             ;;
+         u)
+             BUILD_URL_TOKEN=$OPTARG
+             ;;
+         q)
+             QUEUE_DIR=$OPTARG
+             ;;
+         v)
+             VERBOSE=1
+             ;;
+         ?)
+             usage
+             exit
+             ;;
+     esac
+done
+
+if [[ -z $PROJECT ]] || [[ -z $BUILD_SERVERS ]] || [[ -z $CALLED_BY ]] || [[ -z $CURL ]]
|| [[ -z $BUILD_URL_TOKEN ]] || [[ -z $QUEUE_DIR ]]
+then
+     usage
+     exit 1
+fi
 
-### The Jira project name.  Examples: HADOOP or RIVER or LUCENE
-PROJECT=$1
-### The directory to accumulate the patch queue.  Must be 
-### writable by this process.
-QUEUE_DIR=$2
-### The complete URL to trigger a build.
-TRIGGER_BUILD_URL=$3
+if [[ $VERBOSE == 1 ]]; then
+set -x
+fi
 
-WGET=/usr/sfw/bin/wget
+PATCH_QUEUE=${QUEUE_DIR}/patch_queue.txt
+
+### To check if this patch_admin job called by test-patch.sh or hudson
+### if called by test-patch.sh then freee up the slae machine.
+if [[ $CALLEDBY != "" && $CALLEDBY != "USER" ]]; then
+  rm -rf ${QUEUE_DIR}/${CALLEDBY}
+fi
 
 QUEUE_HTML_FILE=`pwd`/${PROJECT}_PatchQueue.html
-CURRENT_DIR=${QUEUE_DIR}/current
 
-CURRENT_PATCH=`cat $CURRENT_DIR/defectNum`
-### If there is no current patch being tested, 
-### look for the next one in the queue.
-if [[ $CURRENT_PATCH == "" ]] ; then
-  cd $QUEUE_DIR
-  CURRENT_PATCH=`/bin/ls -1dtr ${PROJECT}* | head -1`
-  ### If there is another patch in the queue, start testing it.
-  if [[ $CURRENT_PATCH != "" ]] ; then
-    rm -rf $CURRENT_DIR
-    mv $CURRENT_PATCH $CURRENT_DIR
-    ### Start build.
-    echo "$CURRENT_PATCH patch submitted for testing at `date`"
-    $WGET -q -O $CURRENT_DIR/build $TRIGGER_BUILD_URL
-    chmod -R g+w $CURRENT_DIR
+echo "<html>" > $QUEUE_HTML_FILE
+echo "<title>Patch Queue for $PROJECT</title>" >> $QUEUE_HTML_FILE
+echo "<h3 align='left'><img src="http://hadoop.apache.org/core/images/hadoop-logo.jpg"
height="50"></img>Patch Queue for HADOOP</h3>" >> $QUEUE_HTML_FILE
+echo "<hr style='height:2px;border-width:0;color:red;background-color:blue'>" >>
$QUEUE_HTML_FILE
+echo "<h4>Currently Running (or Waiting To Run)</h4>" >> $QUEUE_HTML_FILE
+echo "<table cellspacing=10><tr align=left><th>Issue</th><th>Submitted
to</th><th>Date Submitted to Run</th></tr>" >> $QUEUE_HTML_FILE
+
+for SLAVE in $BUILD_SERVERS 
+do 
+  TRIGGER_BUILD_URL=${HUDSON_URL}'job/Hadoop-Patch-'${SLAVE}${BUILD_URL_TOKEN}
+  CURRENT_PATCH=${QUEUE_DIR}/${SLAVE}
+  defect=`head -n 1 $PATCH_QUEUE | awk '{print $1}'`
+if [[ ! -f $CURRENT_PATCH ]]; then
+  if [[ $defect ]]; then
+    echo $defect > ${CURRENT_PATCH}
+      sed -i '1d' $PATCH_QUEUE
+      echo "$defect patch submitted for testing at `date +"%b %a %H:%M"`"
+      echo "$CURL $TRIGGER_BUILD_URL&DEFECTNUM=$defect"
+      $CURL "$TRIGGER_BUILD_URL&DEFECTNUM=$defect"
+      echo "<tr><td><a href=\"http://issues.apache.org/jira/browse/${defect}\">$defect</a></td><td>${SLAVE}</td><td>`date
+"%b %a %H:%M"` UTC</td></tr>" >> $QUEUE_HTML_FILE	
   else
     CURRENT_PATCH="none"
     CURRENT_PATCH_TIME="-"
+    echo "<tr><td>$CURRENT_PATCH</td><td>$CURRENT_PATCH_TIME</td></tr>"
>> $QUEUE_HTML_FILE
   fi
-fi
-if [[ -z $CURRENT_PATCH_TIME ]] ; then
-  CURRENT_PATCH_TIME=`/bin/ls -dtl ${CURRENT_DIR}* | awk '{print $6" "$7" "$8" GMT"}'`
-fi
-
-cd $QUEUE_DIR
-QUEUE=`/bin/ls -1dtrl ${PROJECT}* | awk '{print "<tr><td><a href=\"http://issues.apache.org/jira/browse/"$9"\">"$9"</a></td><td>"$6"
"$7" "$8" GMT</td></tr>"}'`
-if [[ $QUEUE == "" ]] ; then
-  QUEUE="<tr><td>empty</td><td>-</td></tr>"
-fi
-
-echo "<html>" > $QUEUE_HTML_FILE
-echo "<title>Patch Queue for $PROJECT</title>" >> $QUEUE_HTML_FILE
-echo "<h1>Patch Queue for $PROJECT</h1>" >> $QUEUE_HTML_FILE
-echo "<hr>" >> $QUEUE_HTML_FILE
-echo "<h2>Currently Running (or Waiting To Run)</h2>" >> $QUEUE_HTML_FILE
-echo "<table cellspacing=10><tr align=left><th>Issue</th><th>Date
Submitted to Run</th></tr>" >> $QUEUE_HTML_FILE
-if [[ $CURRENT_PATCH == "none" ]] ; then
-  echo "<tr><td>$CURRENT_PATCH</td><td>$CURRENT_PATCH_TIME</td></tr>"
>> $QUEUE_HTML_FILE
 else
-  echo "<tr><td><a href=\"http://issues.apache.org/jira/browse/${CURRENT_PATCH}\">$CURRENT_PATCH</a></td><td>$CURRENT_PATCH_TIME</td></tr>"
>> $QUEUE_HTML_FILE
+  CURRENT_PATCH_TIME=`/bin/ls -dtl ${CURRENT_PATCH} | awk '{print $6" "$7" "$8" UTC"}'`
+  EXEC_PATCH=`cat ${CURRENT_PATCH}`
+  echo "<tr><td><a href=\"http://issues.apache.org/jira/browse/$EXEC_PATCH\">$EXEC_PATCH</a></td><td>${SLAVE}</td><td>$CURRENT_PATCH_TIME</td></tr>"
>> $QUEUE_HTML_FILE
 fi
+done
 echo "</table>" >> $QUEUE_HTML_FILE
-echo "<hr>" >> $QUEUE_HTML_FILE
-echo "<h2>Waiting in the Queue</h2>" >> $QUEUE_HTML_FILE
+echo "<hr style='height:2px;border-width:0;color:red;background-color:blue'>" >>
$QUEUE_HTML_FILE
+echo "<h4>Waiting in the Queue</h4>" >> $QUEUE_HTML_FILE
 echo "<table cellspacing=10><tr align=left><th>Issue</th><th>Date
Submitted</th></tr>" >> $QUEUE_HTML_FILE
-echo "$QUEUE" >> $QUEUE_HTML_FILE
+
+if [ -s $PATCH_QUEUE ]; then
+  while read PATCH_DTLS
+  do 
+  TEMP_QUEUE_DTLS=`echo $PATCH_DTLS | awk '{print "<tr><td><a href=\"http://issues.apache.org/jira/browse/"$1"\">"$1"</a></td><td>"$2"
"$3" "$4" "$5" UTC</td></tr>"}'` 
+  echo $TEMP_QUEUE_DTLS >> $QUEUE_HTML_FILE  
+  done < "$PATCH_QUEUE"
+else 
+  echo "<tr><td>empty</td><td>-</td></tr>" >> $QUEUE_HTML_FILE
+fi
+
 echo "</table>" >> $QUEUE_HTML_FILE
-echo "<hr>" >> $QUEUE_HTML_FILE
+echo "<hr style='height:2px;border-width:0;color:red;background-color:blue'>" >>
$QUEUE_HTML_FILE
 echo "This file last updated at: `date`" >>$QUEUE_HTML_FILE
 echo "</html>" >> $QUEUE_HTML_FILE
-

Modified: hadoop/core/nightly/processHadoopPatchEmail.sh
URL: http://svn.apache.org/viewvc/hadoop/core/nightly/processHadoopPatchEmail.sh?rev=748037&r1=748036&r2=748037&view=diff
==============================================================================
--- hadoop/core/nightly/processHadoopPatchEmail.sh (original)
+++ hadoop/core/nightly/processHadoopPatchEmail.sh Thu Feb 26 06:34:09 2009
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -x
+#set -x
 
 ### The Jira project name.  Examples: HADOOP or RIVER or LUCENE
 PROJECT=$1
@@ -8,8 +8,9 @@
 ### writable by this process.
 QUEUE_DIR=$2
 
-GREP=/usr/sfw/bin/ggrep
+GREP=/bin/grep
 LOG=$QUEUE_DIR/log.txt
+PATCH_QUEUE=$QUEUE_DIR/patch_queue.txt
 
 ### Scan email
 while read line
@@ -25,17 +26,20 @@
   fi
 done
 
-### If this email indicates a new patch, start a build
-if [[ -n $patch && ! -d $QUEUE_DIR/$defect ]] ; then
-  echo "$defect is being processed at `date`" >> $LOG
-  mkdir $QUEUE_DIR/$defect
-
-  ### Write the defect number to a file so buildTest.sh 
-  ### knows which patch to test.
-  echo $defect > $QUEUE_DIR/$defect/defectNum
-
-  ### Since this script is run by the 'daemon' user by sendmail,
-  ### make sure everything it creates is group writable
-  chmod -R a+w $QUEUE_DIR/$defect
+if [[ -n $patch && -n $defect ]] ; then
+  while read line
+  do
+    #### To check if defect already in patch queue
+    if [[ `echo $line | grep -c $defect ` == 1 ]] ; then
+      duplicate=true
+      break;
+    fi
+  done < $PATCH_QUEUE
+  ### Append the defect # to the patch queue if defect # not in patch queue already
+  if [[ -z $duplicate ]]; then
+    echo "$defect is being processed at `date`"
+    echo $defect `date` >> $PATCH_QUEUE
+    chmod -R a+w $QUEUE_DIR
+  fi
 fi
 exit 0



Mime
View raw message