openwhisk-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chet...@apache.org
Subject [incubator-openwhisk] branch master updated: Adjust citool to work with Travis build matrix. (#3848)
Date Tue, 10 Jul 2018 08:45:28 GMT
This is an automated email from the ASF dual-hosted git repository.

chetanm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git


The following commit(s) were added to refs/heads/master by this push:
     new 238ad96  Adjust citool to work with Travis build matrix. (#3848)
238ad96 is described below

commit 238ad965e1b4143837e64302d98febd55a96780e
Author: rodric rabbah <rodric@gmail.com>
AuthorDate: Tue Jul 10 04:45:23 2018 -0400

    Adjust citool to work with Travis build matrix. (#3848)
    
    Enables fetching of logs for various jobs in a matrix by adding sub job index as suffix
    
    ./citool monitor -p 402088870.1
---
 tools/build/README.md |  1 +
 tools/build/citool    | 72 +++++++++++++++++++++++++++++++++++++--------------
 2 files changed, 54 insertions(+), 19 deletions(-)

diff --git a/tools/build/README.md b/tools/build/README.md
index f6166a5..77d77dd 100644
--- a/tools/build/README.md
+++ b/tools/build/README.md
@@ -63,6 +63,7 @@ To change the Travis (or Jenkins) host URL, use `-u`.
 - monitor a Travis CI build with job number `N`: `citool monitor N`
 - monitor same job `N` until completion: `citool monitor -p N`
 - save job output to a file: `citool -o monitor N`
+- for Travis CI matrix builds, use the matrix index after the job number as in `citool monitor
N.i` where 1 <= i <= matrix buidls.
 
 To monitor a Jenkins build `B` with job number `N` on host `https://jenkins.host:port`:
 ```
diff --git a/tools/build/citool b/tools/build/citool
index 371b646..2af9342 100755
--- a/tools/build/citool
+++ b/tools/build/citool
@@ -37,13 +37,17 @@ import sys
 import time
 import traceback
 import argparse
-import httplib
+if sys.version_info.major >= 3:
+    from http.client import HTTPConnection, HTTPSConnection, IncompleteRead, TEMPORARY_REDIRECT,
OK
+    from urllib.parse import urlparse
+else:
+    from httplib import HTTPConnection, HTTPSConnection, IncompleteRead, TEMPORARY_REDIRECT,
OK
+    from urlparse import urlparse
 import re
 import threading
 import json
 from xml.dom import minidom
 from subprocess import Popen, PIPE, STDOUT
-from urlparse import urlparse
 
 def main():
     exitCode = 0
@@ -64,7 +68,8 @@ def parseArgs():
     parser = argparse.ArgumentParser(description='tool for analyzing logs from CI')
     subparsers = parser.add_subparsers(title='available commands', dest='cmd')
 
-    parser.add_argument('job', help='job number')
+    parser.add_argument('job', help='job number; for matrix jobs add the matrix index after
a period (e.g., 401881768.2)')
+
     parser.add_argument('-b', '--build', help='build name', default='travis')
     parser.add_argument('-v', '--verbose', help='verbose output', action='store_true')
     parser.add_argument('-i', '--input-file', help='read logs from file rather than CI',
action='store_true', dest='ifile')
@@ -87,9 +92,9 @@ def parseArgs():
 def request(method, urlString, body = "", headers = {}, auth = None, verbose = False):
     url = urlparse(urlString)
     if url.scheme == 'http':
-        conn = httplib.HTTPConnection(url.netloc)
+        conn = HTTPConnection(url.netloc)
     else:
-        conn = httplib.HTTPSConnection(url.netloc)
+        conn = HTTPSConnection(url.netloc)
 
     if verbose:
         print("%s %s" % (method, urlString))
@@ -124,22 +129,48 @@ def shell(cmd, data = None, verbose = False):
 
 def getTravisHeaders():
     return {'User-Agent': 'wsk citool/0.0.1',
+            'Travis-API-Version': 3,
             'Accept': 'application/vnd.travis-ci.2+json'
     }
 
+def getTravisMatrixId(parts, values):
+    N = len(values)
+    if len(parts) == 1:
+        return -1
+    else:
+        try:
+            matrix = int(parts[1]) -1
+            if matrix < 0:
+                print('Matrix id must be positive. Valid values are [1..%s].' % N)
+                exit(-1)
+            if matrix >= N:
+                print('Matrix id is out of bounds. Valid values are [1..%s].' % N)
+                exit(-1)
+            return matrix
+        except Exception:
+            print('Matrix id is not an integer as expected. Valid values are [1..%s].' %
N)
+            exit(-1)
+
 def getJobUrl(args):
     if args.build.lower() == 'travis':
         # Get build information
-        buildUrl = '%s/builds/%s' % (args.url, args.job)
+        parts = args.job.split('.')
+        jobid = parts[0]
+        if len(parts) > 2:
+            print('Job is malformed')
+            exit(-1)
+
+        buildUrl = '%s/build/%s' % (args.url, jobid)
         buildRes = request('get', buildUrl, headers = getTravisHeaders(), verbose = args.verbose)
         body = validateResponse(buildRes)
         try:
             body = json.loads(body)
-            job = body['build']['job_ids'][-1]
+            index = getTravisMatrixId(parts, body['jobs'])
+            job = body['jobs'][index]['id']
         except Exception:
-            print('expected response to contain build and job-ids properties in %s' % body)
+            print('Expected response to contain build and job-ids properties in %s' % body)
             exit(-1)
-        url = '%s/jobs/%s' % (args.url, job)
+        url = '%s/job/%s' % (args.url, job)
     else: # assume jenkins
         url = '%s/job/%s/%s' % (args.url, args.build, args.job)
     return url
@@ -164,8 +195,8 @@ def monitorOnce(args):
     else:
         if args.build.lower() == 'travis':
             url = '%s/log.txt' % getJobUrl(args)
-            res = request('get', url, verbose = args.verbose)
-            if res.status == httplib.TEMPORARY_REDIRECT:
+            res = request('get', url, headers = getTravisHeaders(), verbose = args.verbose)
+            if res.status == TEMPORARY_REDIRECT:
                 url = res.getheader('location')
                 res = request('get', url, headers = getTravisHeaders(), verbose = args.verbose)
         else: # assume jenkins
@@ -178,7 +209,7 @@ def monitorOnce(args):
         file = open('%s-console.log' % args.job, 'w')
         file.write(body)
         file.close()
-    if args.ifile or res.status == httplib.OK:
+    if args.ifile or res.status == OK:
         grepForFailingTests(args, body)
         return reportBuildStatus(args, body)
     elif args.ofile is False:
@@ -188,7 +219,7 @@ def monitorOnce(args):
 def validateResponse(res):
     body = res.read()
 
-    if res.status != httplib.OK:
+    if res.status != OK:
         if body.startswith('<'):
             dom = minidom.parseString(body)
             print(dom.toprettyxml()),
@@ -196,7 +227,7 @@ def validateResponse(res):
             print(body)
         exit(res.status)
     elif not body:
-        print('build log is empty')
+        print('Build log is empty.')
         exit(-1)
     else:
         return body
@@ -206,21 +237,24 @@ def grepForFailingTests(args, body):
     # check that tests ran
     (time, output, error) = shell(cmd, body, args.verbose)
     if output == '':
-        print('no tests detected')
+        print('No tests detected.')
     # no tests: either build failure or task not yet reached, skip further check
     else:
         cmd = 'grep -E "^\w+\.*.*[&gt;|>] \w*.* FAILED%s"' % ("|PASSED" if args.all
else "")
         (time, output, error) = shell(cmd, body, args.verbose)
         if output == '':
-            print('all tests passing')
+            print('All tests passing.')
         else:
             print(output.replace('&gt;', '>')),
 
 def reportBuildStatus(args, body):
-    lines = body.rstrip('\n').rsplit('\n', 1)
+    lines = body.decode('utf8').rstrip('\n').rsplit('\n', 1)
+
     if len(lines) == 2:
         output = lines[1]
         output = re.sub('<[^<]+?>', '', output).strip()
+    else:
+        output = None
 
     if output and ('Finished: ' in output or output.startswith('Done.') or ('exceeded' in
output and 'terminated' in output)):
         print(output)
@@ -246,7 +280,7 @@ def cat(args):
         url = '%s/artifact/%s/%s/%s_logs.log' % (getJobUrl(args), args.artifactPath, component,
component)
         res = request('get', url, verbose = args.verbose)
         body = res.read()
-        if res.status == httplib.OK:
+        if res.status == OK:
             return body
         else:
             return ''
@@ -267,7 +301,7 @@ def cat(args):
         joined = file.read()
         file.close()
     elif args.build.lower == 'travis':
-        print('feature not yet supported for Travis builds')
+        print('Feature not yet supported for Travis builds.')
         return 2
     else:
         components = {


Mime
View raw message