couchdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From woh...@apache.org
Subject [couchdb] 01/01: Add auto log uploader for CI workflow
Date Wed, 10 May 2017 23:08:00 GMT
This is an automated email from the ASF dual-hosted git repository.

wohali pushed a commit to branch upload-failed-logfiles
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 4cfdb51257941d139a0e4f9e25273208324e60fa
Author: Joan Touzet <joant@atypical.net>
AuthorDate: Wed May 10 18:45:19 2017 -0400

    Add auto log uploader for CI workflow
---
 .gitignore                    |  12 +++++
 .travis.yml                   |  21 ++++++--
 build-aux/logfile-uploader.py | 111 ++++++++++++++++++++++++++++++++++++++++++
 test/javascript/run           |   6 ++-
 4 files changed, 145 insertions(+), 5 deletions(-)

diff --git a/.gitignore b/.gitignore
index e3601f7..f6e0c37 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,8 @@ log
 apache-couchdb-*/
 bin/
 config.erl
+*.tar.gz
+*.tar.bz2
 dev/boot_node.beam
 dev/lib/
 dev/logs/
@@ -106,3 +108,13 @@ src/mango/ebin/
 src/mango/test/*.pyc
 src/mango/venv/
 src/mango/.eunit
+
+src/chttpd/.eunit/
+src/couch_event/.eunit/
+src/couch_mrview/.eunit/
+src/couch_plugins/.eunit/
+src/couch_stats/.eunit/
+src/ddoc_cache/.eunit/
+src/fabric/.eunit/
+src/mem3/.eunit/
+src/rexi/.eunit/
diff --git a/.travis.yml b/.travis.yml
index 580ac6c..156856c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,19 +17,29 @@ addons:
     - libicu-dev
     - libmozjs185-dev
     - pkg-config
+    - python-requests
+    - python-sphinx
     - help2man
     - shunit2
 
+python:
+  - "2.7"
+
 git:
   depth: 10
 
+# logfile uploader uses requests
+cache:
+  - pip
+
+# logfile uploader credentials
+env:
+  global:
+    - secure: "UdA/gKIlyuXaW+hUgRx40t1TYjLCGxMqHvM5Uw7UbUH2dqEkgJiLfhZGchS1JVzl8M01VKZUUzS7v2nvRLiHZN1kvaw5kfq31VRoafUah8jfmvqNWZVdLovHl3aw5UX/HRt0RkbWbhdbdknTfh6+YinSZ+Nb54jCErMg9nabXtM="
+
 # Enable this block if you want to build docs & fauxton too
-#cache:
-#  - pip
 #node_js:
 #  - 6
-#install:
-#  - pip install sphinx
 #before_script:
 #  - ./configure -c
 
@@ -40,6 +50,9 @@ before_script:
 script:
    - make check
 
+after_failure:
+  - build-aux/logfile-uploader.py
+
 # Re-enable once test suite is reliable
 #notifications:
 #  email: false
diff --git a/build-aux/logfile-uploader.py b/build-aux/logfile-uploader.py
new file mode 100755
index 0000000..b656670
--- /dev/null
+++ b/build-aux/logfile-uploader.py
@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+from __future__ import print_function
+
+import datetime
+import glob
+import json
+import os
+import tarfile
+import time
+
+import requests
+
+COUCH_URL = "http://couchdb-vm.apache.org:5984/ci_errorlogs"
+TARFILE = "couchlog.tar.gz"
+
+def _tojson(req):
+    """Support requests v0.x as well as 1.x+"""
+    if requests.__version__[0] == '0':
+        return json.loads(req.content)
+    return req.json()
+
+def collect_logfiles():
+    """ Find and tarball all logfiles """
+    tb = tarfile.open(name=TARFILE, mode='w:gz')
+    # EUnit
+    for log in glob.glob('src/*/.eunit/couch.log'):
+        tb.add(log)
+    # JS harness
+    for log in glob.glob('dev/logs/node1.log'):
+        tb.add(log)
+    tb.close()
+
+def build_ci_doc():
+    """ Build a metadata document with relevant detail from CI env """
+    doc = {}
+    if 'TRAVIS' in os.environ:
+        doc['builder'] = 'travis'
+        doc['build_id'] = os.environ['TRAVIS_JOB_ID']
+        doc['erlang'] = os.environ['TRAVIS_OTP_RELEASE']
+        doc['url'] = 'https://travis-ci.org/apache/couchdb/jobs/' + \
+            os.environ['TRAVIS_JOB_ID']
+        doc['branch'] = os.environ['TRAVIS_BRANCH']
+        doc['commit'] = os.environ['TRAVIS_COMMIT']
+        doc['repo'] = 'https://github.com/' + os.environ['TRAVIS_REPO_SLUG']
+    elif 'JENKINS_URL' in os.environ:
+        doc['builder'] = 'jenkins'
+        doc['build_id'] = os.environ['BUILD_NUMBER']
+        doc['url'] = os.environ['BUILD_URL']
+        doc['branch'] = os.environ['GIT_BRANCH']
+        doc['commit'] = os.environ['GIT_COMMIT']
+        doc['repo'] = os.environ['GIT_URL']
+    else:
+        doc['builder'] = 'manual'
+        # TODO: shell out to get correct repo, commit, branch info?
+        doc['repo'] = 'https://github.com/apache/couchdb'
+        doc['build_id'] = str(time.time())
+
+    # shorten doc id
+    repo = doc['repo'].split('/')[-1]
+    repo = repo.replate('.git', '')
+
+    doc['_id'] = doc['builder'] + '-' + repo + '-' + \
+        doc['build_id'] + \
+        '-' + datetime.datetime.utcnow().isoformat()
+
+    return doc
+
+def upload_logs():
+    lp = os.environ['COUCHAUTH'].split(':')
+    creds = (lp[0], lp[1])
+    doc = build_ci_doc()
+    req = requests.post(COUCH_URL,
+        data=json.dumps(doc),
+        auth=creds,
+        headers={'Content-type': 'application/json'})
+    req.raise_for_status()
+    req = _tojson(req)
+    with open(TARFILE, 'rb') as f:
+        req2 = requests.put(COUCH_URL + '/' + doc['_id'] + '/' + TARFILE,
+            headers={'Content-type': 'application/x-gtar'},
+            auth=creds,
+            params={'rev': req['rev']},
+            data=f)
+    req2.raise_for_status()
+    return req2
+
+
+def main():
+    """ Find latest logfile and upload to Couch logfile db. """
+    print ("Uploading logfiles...")
+    collect_logfiles()
+    req = upload_logs()
+    print (req.url)
+    print (req.text)
+    print ("Upload complete!")
+
+if __name__ == '__main__':
+    main()
diff --git a/test/javascript/run b/test/javascript/run
index 10d00d8..7f366eb 100755
--- a/test/javascript/run
+++ b/test/javascript/run
@@ -99,7 +99,9 @@ def options():
     return [
         op.make_option("-s", "--start", metavar="FILENAME", default=None,
                        help="Start from the given filename if multiple files "
-                            "are passed")
+                            "are passed"),
+        op.make_option("-a", "--all", action="store_true", dest="all",
+                       help="Run all tests, even if one or more fail")
     ]
 
 
@@ -141,6 +143,8 @@ def main():
             passed += 1
         else:
             failed += 1
+            if not opts.all:
+                break
 
     sys.stderr.write("======================================================="
         + os.linesep)

-- 
To stop receiving notification emails like this one, please contact
"commits@couchdb.apache.org" <commits@couchdb.apache.org>.

Mime
View raw message