hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1546120 - in /hadoop/nightly: hudsonPatchQueueAdmin.sh jenkinsPrecommitAdmin.py
Date Wed, 27 Nov 2013 17:14:39 GMT
Author: todd
Date: Wed Nov 27 17:14:39 2013
New Revision: 1546120

URL: http://svn.apache.org/r1546120
Log:
HADOOP-9765. Precommit Admin job chokes on issues without an attachment. Contributed by Brock
Noland.

Added:
    hadoop/nightly/jenkinsPrecommitAdmin.py   (with props)
Modified:
    hadoop/nightly/hudsonPatchQueueAdmin.sh

Modified: hadoop/nightly/hudsonPatchQueueAdmin.sh
URL: http://svn.apache.org/viewvc/hadoop/nightly/hudsonPatchQueueAdmin.sh?rev=1546120&r1=1546119&r2=1546120&view=diff
==============================================================================
--- hadoop/nightly/hudsonPatchQueueAdmin.sh (original)
+++ hadoop/nightly/hudsonPatchQueueAdmin.sh Wed Nov 27 17:14:39 2013
@@ -1,102 +0,0 @@
-#!/bin/bash -e
-
-usage() 
-{
-cat << EOF
-usage: $0 options
-
-This script initates patch build's on Hudson slaves.
-
-OPTIONS:
-   -h      Show this message
-   -f      Url to Jira XML filter of patch available issues.
-   -c      Path to curl binary
-   -s      Path to sed binary
-   -t      Build token
-   -v      Verbose
-EOF
-}
-
-FILTER=
-CURL=
-BUILD_TOKEN=
-VERBOSE=
-SED= 
-
-while getopts "hf:t:s:c:v" OPTION
-do
-     case $OPTION in
-         h)
-             usage
-             exit 1
-             ;;
-         f)
-             FILTER=$OPTARG 
-             ;;
-         c)
-             CURL=$OPTARG
-             ;;
-         s)
-             SED=$OPTARG
-             ;;
-         t)
-             BUILD_TOKEN=$OPTARG
-             ;;
-         v)
-             VERBOSE=1
-             ;;
-         ?)
-             usage
-             exit
-             ;;
-     esac
-done
-
-if [[ -z $FILTER ]] || [[ -z $SED ]] || [[ -z $CURL ]] || [[ -z $BUILD_TOKEN ]] ; then
-  usage
-  exit 1
-fi
-
-if [[ $VERBOSE == 1 ]] ; then
-  set -x
-fi
-
-### Grab the latest patch_tested.txt file from this job on Hudson
-curl --fail --location --retry 3 --output patch_tested.txt ${JOB_URL}/lastSuccessfulBuild/artifact/patch_tested.txt
-
-### Fail fast if downloaded file may be corrupted
-FIRSTLINE=`head -1 patch_tested.txt`
-if [ "$FIRSTLINE" != "TESTED ISSUES" ] ; then
-  echo "Downloaded patch_tested.txt control file may be corrupted. Failing."
-  exit 1
-fi
-
-### Grab the latest patch available query from Jira
-curl --fail --location --retry 3 --output patch_available.xml $FILTER
-
-### Grab all the key (issue numbers) and largest attachment id for each item in the XML
-xpath -q -e "//item/key/text() | //item/attachments/attachment[not(../attachment/@id >
@id)]/@id" patch_available.xml |tee patch_available2.elements
-
-### Replace newlines with nothing, then replace id=" with =, then replace " with newline
-### to yield lines with pairs (issueNumber,largestAttachmentId). Example: HADOOP-123,456984
-awk '{ printf "%s", $0 }' patch_available2.elements | sed -e "s/\W*id=\"/,/g" | sed -e "s/\"/\n/g"
|tee patch_available3.txt
-
-### Iterate through issue list and find the (issueNumber,largestAttachmentId) pairs that
have 
-### not been tested (ie don't already exist in the patch_tested.txt file
-touch patch_tested.txt
-cat patch_available3.txt | while read PAIR ; do
-  set +e
-  COUNT=`grep -c "$PAIR" patch_tested.txt`
-  set -e
-  if [ "$COUNT" -lt "1" ] ; then
-    ### Parse $PAIR into project, issue number, and attachment id
-    PROJECT=`echo $PAIR | sed -e "s/-.*//g"`
-    ISSUE=`echo $PAIR | sed -e "s/.*-//g" | sed -e "s/,.*//g"`
-    ATTACHMENT=`echo $PAIR | sed -e "s/.*,//g"`
-    ### Kick off job
-    echo "Starting job $PROJECT with issue $ISSUE and attachment $ATTACHMENT"
-    curl --fail --location --retry 3 "${HUDSON_URL}/job/PreCommit-${PROJECT}-Build/buildWithParameters?token=${BUILD_TOKEN}&ISSUE_NUM=${ISSUE}&ATTACHMENT_ID=${ATTACHMENT}"
-    ### Mark this pair as tested by appending to file
-    echo "$PAIR" >> patch_tested.txt
-  fi
-done

Added: hadoop/nightly/jenkinsPrecommitAdmin.py
URL: http://svn.apache.org/viewvc/hadoop/nightly/jenkinsPrecommitAdmin.py?rev=1546120&view=auto
==============================================================================
--- hadoop/nightly/jenkinsPrecommitAdmin.py (added)
+++ hadoop/nightly/jenkinsPrecommitAdmin.py Wed Nov 27 17:14:39 2013
@@ -0,0 +1,138 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Script uses a JIRA search filter to submit test Jenkins
+# jobs for patch avilable issues. For more information see:
+#  http://wiki.apache.org/general/PreCommitBuilds
+#  https://builds.apache.org/job/PreCommit-Admin/
+
+from optparse import OptionParser
+from tempfile import NamedTemporaryFile
+from xml.etree import ElementTree
+import os, re, sys, urllib2
+
+# max number of entries to keep in the patch_tested.txt file
+MAX_HISTORY = 5000
+
+def httpGet(resource, ignoreError=False):
+  if ignoreError:
+    try:
+      return urllib2.urlopen(resource).read()
+    except urllib2.URLError, e:
+      print "ERROR retrieving resource %s: %s" % (resource, e)
+      return ""
+  return urllib2.urlopen(resource).read()
+
+
+# returns a map of (project, issue) => attachment id
+def parseJiraData(fileName):
+  tree = ElementTree.parse(fileName)
+  root = tree.getroot()
+  jiraPattern = re.compile('([A-Z]+)\-([0-9]+)')
+  result = {}
+  for item in root.findall("./channel/item"):
+    key = item.find('key')
+    if key == None: continue
+    issue = key.text
+    matcher = jiraPattern.match(issue)
+    if not matcher: continue
+    issue = matcher.group(1), matcher.group(2)
+    attachmentIds = []
+    for attachment in item.findall('./attachments/attachment'):
+      attachmentId = attachment.get('id')
+      try:
+        attachmentIds.append(int(attachmentId))
+      except ValueError:
+        pass
+    if len(attachmentIds) > 0:
+      attachmentIds.sort()
+      result[issue] = attachmentIds[-1]
+  return result
+
+if __name__ == '__main__':
+  parser = OptionParser()
+  parser.add_option("--jenkins-url", dest="jenkinsUrl",
+                    help="Jenkins base URL", metavar="URL")
+  parser.add_option("--jenkins-token", dest="jenkinsToken",
+                    help="Jenkins Token", metavar="TOKEN")
+  parser.add_option("--jira-filter", dest="jiraFilter",
+                    help="JIRA filter URL", metavar="URL")
+  parser.add_option("--jenkins-url-override", dest="jenkinsUrlOverrides", action="append",
+                    help="Project specific Jenkins base URL", metavar="PROJECT=URL")
+  parser.add_option("--live", dest="live", action="store_true",
+                    help="Submit Job to jenkins")
+  (options, args) = parser.parse_args()
+  if not options.jiraFilter:
+    parser.error("JIRA Filter is a required argument")
+  if not options.jenkinsUrl:
+    parser.error("Jenkins URL is a required argument")
+  if options.live and not options.jenkinsToken:
+    parser.error("Jenkins Token is required when in live mode")
+  jenkinsUrlOverrides = {}
+  if options.jenkinsUrlOverrides:
+    for override in options.jenkinsUrlOverrides:
+      if "=" not in override:
+        parser.error("Invalid Jenkins Url Override: " + override)
+      (project, url) = override.split("=", 1)
+      jenkinsUrlOverrides[project.upper()] = url
+  tempFile = NamedTemporaryFile(delete=False)
+  try:
+    jobLogHistory = httpGet(options.jenkinsUrl + \
+      "/job/PreCommit-Admin/lastSuccessfulBuild/artifact/patch_tested.txt").strip().split("\n")
+    if "TESTED ISSUES" not in jobLogHistory[0]:
+      print "Downloaded patch_tested.txt control file may be corrupted. Failing."
+      sys.exit(1)
+    jobLog = open('patch_tested.txt', 'w+')
+    if len(jobLogHistory)  > MAX_HISTORY:
+      jobLogHistory = [ jobLogHistory[0] ] + jobLogHistory[len(jobLogHistory) - MAX_HISTORY:]
+    for jobHistoryRecord in jobLogHistory:
+      jobLog.write(jobHistoryRecord + "\n")
+    jobLog.flush()
+    rssData = httpGet(options.jiraFilter)
+    tempFile.write(rssData)
+    tempFile.flush()
+    for key, attachment in parseJiraData(tempFile.name).items():
+      (project, issue) = key
+      if jenkinsUrlOverrides.has_key(project):
+        url = jenkinsUrlOverrides[project]
+      else:
+        url = options.jenkinsUrl
+      jenkinsUrlTemplate = url + "/job/PreCommit-{project}-Build/buildWithParameters" + \
+        "?token={token}&ISSUE_NUM={issue}&ATTACHMENT_ID={attachment}"
+      urlArgs = {'token': options.jenkinsToken, 'project': project, 'issue': issue, 'attachment':
attachment }
+      jenkinsUrl = jenkinsUrlTemplate.format(**urlArgs)
+      # submit job
+      jobName = "%s-%s,%s" % (project, issue, attachment)
+      if jobName not in jobLogHistory:
+        print jobName + " has not been processed, submitting"
+        jobLog.write(jobName + "\n")
+        jobLog.flush()
+        if options.live:
+          httpGet(jenkinsUrl, True)
+        else:
+          print "GET " + jenkinsUrl
+      else:
+        print jobName + " has been processed, ignoring"
+    jobLog.close()
+  finally:
+    if options.live:
+      os.remove(tempFile.name)
+    else:
+      print "JIRA Data is located: " + tempFile.name

Propchange: hadoop/nightly/jenkinsPrecommitAdmin.py
------------------------------------------------------------------------------
    svn:executable = *



Mime
View raw message