cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aweisb...@apache.org
Subject cassandra-dtest git commit: Add plumbing to cassandra dtests to allow collecting test names for multiple builds as well as merging test output
Date Wed, 15 Nov 2017 21:36:21 GMT
Repository: cassandra-dtest
Updated Branches:
  refs/heads/master 8e65211b9 -> ac0ce6044


Add plumbing to cassandra dtests to allow collecting test names for multiple builds as well
as merging test output

Patch by Ariel Weisberg; Reviewd by Michael Kjellman for CASSANDRA-14017


Project: http://git-wip-us.apache.org/repos/asf/cassandra-dtest/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra-dtest/commit/ac0ce604
Tree: http://git-wip-us.apache.org/repos/asf/cassandra-dtest/tree/ac0ce604
Diff: http://git-wip-us.apache.org/repos/asf/cassandra-dtest/diff/ac0ce604

Branch: refs/heads/master
Commit: ac0ce60443ee59a8ad333f5d5a08bc6efa574a53
Parents: 8e65211
Author: Ariel Weisberg <aweisberg@apple.com>
Authored: Wed Nov 15 16:17:39 2017 -0500
Committer: Ariel Weisberg <aweisberg@apple.com>
Committed: Wed Nov 15 16:32:47 2017 -0500

----------------------------------------------------------------------
 plugins/dtestcollect.py |  93 ++++++++++++
 plugins/dtesttag.py     |  47 ++++++
 plugins/dtestxunit.py   | 346 +++++++++++++++++++++++++++++++++++++++++++
 run_dtests.py           |  20 ++-
 4 files changed, 500 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra-dtest/blob/ac0ce604/plugins/dtestcollect.py
----------------------------------------------------------------------
diff --git a/plugins/dtestcollect.py b/plugins/dtestcollect.py
new file mode 100644
index 0000000..e8a3c8a
--- /dev/null
+++ b/plugins/dtestcollect.py
@@ -0,0 +1,93 @@
+from collections import namedtuple
+
+from pprint import pprint
+import os
+import inspect
+from nose.plugins.base import Plugin
+from nose.case import Test
+import logging
+import unittest
+
+log = logging.getLogger(__name__)
+
+class DTestCollect(Plugin):
+    """
+    Collect and output test names only, don't run any tests.
+    """
+    name = 'dtest_collect'
+    enableOpt = 'dtest_collect_only'
+
+    def options(self, parser, env):
+        """Register commandline options.
+        """
+        parser.add_option('--dtest-collect-only',
+                          action='store_true',
+                          dest=self.enableOpt,
+                          default=env.get('DTEST_NOSE_COLLECT_ONLY'),
+                          help="Enable collect-only: %s [COLLECT_ONLY]" %
+                          (self.help()))
+
+    def prepareTestLoader(self, loader):
+        """Install collect-only suite class in TestLoader.
+        """
+        # Disable context awareness
+        log.debug("Preparing test loader")
+        loader.suiteClass = TestSuiteFactory(self.conf)
+
+    def prepareTestCase(self, test):
+        """Replace actual test with dummy that always passes.
+        """
+        # Return something that always passes
+        log.debug("Preparing test case %s", test)
+        if not isinstance(test, Test):
+            return
+        def run(result):
+            # We need to make these plugin calls because there won't be
+            # a result proxy, due to using a stripped-down test suite
+            self.conf.plugins.startTest(test)
+            result.startTest(test)
+            self.conf.plugins.addSuccess(test)
+            result.addSuccess(test)
+            self.conf.plugins.stopTest(test)
+            result.stopTest(test)
+        return run
+
+    def describeTest(self, test):
+        tag = os.getenv('TEST_TAG', '')
+        if tag == '':
+            tag = test.test._testMethodName
+        else:
+            tag = test.test._testMethodName + "-" + tag
+        retval = "%s:%s.%s" % (test.test.__module__, test.test.__class__.__name__, tag)
+        return retval;
+
+class TestSuiteFactory:
+    """
+    Factory for producing configured test suites.
+    """
+    def __init__(self, conf):
+        self.conf = conf
+
+    def __call__(self, tests=(), **kw):
+        return TestSuite(tests, conf=self.conf)
+
+
+class TestSuite(unittest.TestSuite):
+    """
+    Basic test suite that bypasses most proxy and plugin calls, but does
+    wrap tests in a nose.case.Test so prepareTestCase will be called.
+    """
+    def __init__(self, tests=(), conf=None):
+        self.conf = conf
+        # Exec lazy suites: makes discovery depth-first
+        if callable(tests):
+            tests = tests()
+        log.debug("TestSuite(%r)", tests)
+        unittest.TestSuite.__init__(self, tests)
+
+    def addTest(self, test):
+        log.debug("Add test %s", test)
+        if isinstance(test, unittest.TestSuite):
+            self._tests.append(test)
+        else:
+            self._tests.append(Test(test, config=self.conf))

http://git-wip-us.apache.org/repos/asf/cassandra-dtest/blob/ac0ce604/plugins/dtesttag.py
----------------------------------------------------------------------
diff --git a/plugins/dtesttag.py b/plugins/dtesttag.py
new file mode 100644
index 0000000..94effcb
--- /dev/null
+++ b/plugins/dtesttag.py
@@ -0,0 +1,47 @@
+from collections import namedtuple
+
+from nose import plugins
+from pprint import pprint
+import os
+import inspect
+
+class DTestTag(plugins.Plugin):
+    enabled = True  # if this plugin is loaded at all, we're using it
+    name = 'dtest_tag'
+
+    def __init__(self):
+        pass
+
+    def configure(self, options, conf):
+        pass
+
+    def nice_classname(self, obj):
+        """Returns a nice name for class object or class instance.
+    
+            >>> nice_classname(Exception()) # doctest: +ELLIPSIS
+            '...Exception'
+            >>> nice_classname(Exception) # doctest: +ELLIPSIS
+            '...Exception'
+    
+        """
+        if inspect.isclass(obj):
+            cls_name = obj.__name__
+        else:
+            cls_name = obj.__class__.__name__
+        mod = inspect.getmodule(obj)
+        if mod:
+            name = mod.__name__
+            # jython
+            if name.startswith('org.python.core.'):
+                name = name[len('org.python.core.'):]
+            return "%s.%s" % (name, cls_name)
+        else:
+            return cls_name
+
+    def describeTest(self, test):
+        tag = os.getenv('TEST_TAG', '')
+        if tag == '':
+            tag = test.test._testMethodName
+        else:
+            tag = test.test._testMethodName + "-" + tag
+        return "%s (%s)" % (tag, self.nice_classname(test.test))

http://git-wip-us.apache.org/repos/asf/cassandra-dtest/blob/ac0ce604/plugins/dtestxunit.py
----------------------------------------------------------------------
diff --git a/plugins/dtestxunit.py b/plugins/dtestxunit.py
new file mode 100644
index 0000000..c4a041f
--- /dev/null
+++ b/plugins/dtestxunit.py
@@ -0,0 +1,346 @@
+"""This plugin provides test results in the standard XUnit XML format.
+
+It's designed for the `Jenkins`_ (previously Hudson) continuous build
+system, but will probably work for anything else that understands an
+XUnit-formatted XML representation of test results.
+
+Add this shell command to your builder ::
+
+    nosetests --with-dtestxunit
+
+And by default a file named nosetests.xml will be written to the
+working directory.
+
+In a Jenkins builder, tick the box named "Publish JUnit test result report"
+under the Post-build Actions and enter this value for Test report XMLs::
+
+    **/nosetests.xml
+
+If you need to change the name or location of the file, you can set the
+``--dtestxunit-file`` option.
+
+If you need to change the name of the test suite, you can set the
+``--dtestxunit-testsuite-name`` option.
+
+Here is an abbreviated version of what an XML test report might look like::
+
+    <?xml version="1.0" encoding="UTF-8"?>
+    <testsuite name="nosetests" tests="1" errors="1" failures="0" skip="0">
+        <testcase classname="path_to_test_suite.TestSomething"
+                  name="test_it" time="0">
+            <error type="exceptions.TypeError" message="oops, wrong type">
+            Traceback (most recent call last):
+            ...
+            TypeError: oops, wrong type
+            </error>
+        </testcase>
+    </testsuite>
+
+.. _Jenkins: http://jenkins-ci.org/
+
+"""
+import codecs
+import doctest
+import os
+import sys
+import traceback
+import re
+import inspect
+from StringIO import StringIO
+from time import time
+from xml.sax import saxutils
+
+from nose.plugins.base import Plugin
+from nose.exc import SkipTest
+from nose.pyversion import force_unicode, format_exception
+
+# Invalid XML characters, control characters 0-31 sans \t, \n and \r
+CONTROL_CHARACTERS = re.compile(r"[\000-\010\013\014\016-\037]")
+
+TEST_ID = re.compile(r'^(.*?)(\(.*\))$')
+
+def xml_safe(value):
+    """Replaces invalid XML characters with '?'."""
+    return CONTROL_CHARACTERS.sub('?', value)
+
+def escape_cdata(cdata):
+    """Escape a string for an XML CDATA section."""
+    return xml_safe(cdata).replace(']]>', ']]>]]&gt;<![CDATA[')
+
+def id_split(idval):
+    m = TEST_ID.match(idval)
+    retval = []
+    if m:
+        name, fargs = m.groups()
+        head, tail = name.rsplit(".", 1)
+        retval = [head, tail+fargs]
+    else:
+        retval = idval.rsplit(".", 1)
+    tag = os.getenv('TEST_TAG', '')
+    if tag != '':
+        retval[-1] = retval[-1] + "-" + tag
+    return retval
+
+def nice_classname(obj):
+    """Returns a nice name for class object or class instance.
+
+        >>> nice_classname(Exception()) # doctest: +ELLIPSIS
+        '...Exception'
+        >>> nice_classname(Exception) # doctest: +ELLIPSIS
+        '...Exception'
+
+    """
+    if inspect.isclass(obj):
+        cls_name = obj.__name__
+    else:
+        cls_name = obj.__class__.__name__
+    mod = inspect.getmodule(obj)
+    if mod:
+        name = mod.__name__
+        # jython
+        if name.startswith('org.python.core.'):
+            name = name[len('org.python.core.'):]
+        return "%s.%s" % (name, cls_name)
+    else:
+        return cls_name
+
+def exc_message(exc_info):
+    """Return the exception's message."""
+    exc = exc_info[1]
+    if exc is None:
+        # str exception
+        result = exc_info[0]
+    else:
+        try:
+            result = str(exc)
+        except UnicodeEncodeError:
+            try:
+                result = unicode(exc)
+            except UnicodeError:
+                # Fallback to args as neither str nor
+                # unicode(Exception(u'\xe6')) work in Python < 2.6
+                result = exc.args[0]
+    result = force_unicode(result, 'UTF-8')
+    return xml_safe(result)
+
+class Tee(object):
+    def __init__(self, encoding, *args):
+        self._encoding = encoding
+        self._streams = args
+
+    def write(self, data):
+        data = force_unicode(data, self._encoding)
+        for s in self._streams:
+            s.write(data)
+
+    def writelines(self, lines):
+        for line in lines:
+            self.write(line)
+
+    def flush(self):
+        for s in self._streams:
+            s.flush()
+
+    def isatty(self):
+        return False
+
+
+class DTestXunit(Plugin):
+    """This plugin provides test results in the standard XUnit XML format."""
+    name = 'dtestxunit'
+    score = 1500
+    encoding = 'UTF-8'
+    error_report_file = None
+
+    def __init__(self):
+        super(DTestXunit, self).__init__()
+        self._capture_stack = []
+        self._currentStdout = None
+        self._currentStderr = None
+
+    def _timeTaken(self):
+        if hasattr(self, '_timer'):
+            taken = time() - self._timer
+        else:
+            # test died before it ran (probably error in setup())
+            # or success/failure added before test started probably
+            # due to custom TestResult munging
+            taken = 0.0
+        return taken
+
+    def _quoteattr(self, attr):
+        """Escape an XML attribute. Value can be unicode."""
+        attr = xml_safe(attr)
+        return saxutils.quoteattr(attr)
+
+    def options(self, parser, env):
+        """Sets additional command line options."""
+        Plugin.options(self, parser, env)
+        parser.add_option(
+            '--dtestxunit-file', action='store',
+            dest='dtestxunit_file', metavar="FILE",
+            default=env.get('NOSE_XUNIT_FILE', 'nosetests.xml'),
+            help=("Path to xml file to store the xunit report in. "
+                  "Default is nosetests.xml in the working directory "
+                  "[NOSE_XUNIT_FILE]"))
+
+        parser.add_option(
+            '--dtestxunit-testsuite-name', action='store',
+            dest='dtestxunit_testsuite_name', metavar="PACKAGE",
+            default=env.get('NOSE_XUNIT_TESTSUITE_NAME', 'nosetests'),
+            help=("Name of the testsuite in the xunit xml, generated by plugin. "
+                  "Default test suite name is nosetests."))
+
+    def configure(self, options, config):
+        """Configures the xunit plugin."""
+        Plugin.configure(self, options, config)
+        self.config = config
+        if self.enabled:
+            self.stats = {'errors': 0,
+                          'failures': 0,
+                          'passes': 0,
+                          'skipped': 0
+                          }
+            self.errorlist = []
+            self.error_report_file_name = os.path.realpath(options.dtestxunit_file)
+            self.xunit_testsuite_name = options.dtestxunit_testsuite_name
+
+    def report(self, stream):
+        """Writes an Xunit-formatted XML file
+
+        The file includes a report of test errors and failures.
+
+        """
+        self.error_report_file = codecs.open(self.error_report_file_name, 'w',
+                                             self.encoding, 'replace')
+        self.stats['encoding'] = self.encoding
+        self.stats['testsuite_name'] = self.xunit_testsuite_name
+        self.stats['total'] = (self.stats['errors'] + self.stats['failures']
+                               + self.stats['passes'] + self.stats['skipped'])
+        self.error_report_file.write(
+            u'<?xml version="1.0" encoding="%(encoding)s"?>'
+            u'<testsuite name="%(testsuite_name)s" tests="%(total)d" '
+            u'errors="%(errors)d" failures="%(failures)d" '
+            u'skip="%(skipped)d">' % self.stats)
+        self.error_report_file.write(u''.join([force_unicode(e, self.encoding)
+                                               for e in self.errorlist]))
+        self.error_report_file.write(u'</testsuite>')
+        self.error_report_file.close()
+        if self.config.verbosity > 1:
+            stream.writeln("-" * 70)
+            stream.writeln("XML: %s" % self.error_report_file.name)
+
+    def _startCapture(self):
+        self._capture_stack.append((sys.stdout, sys.stderr))
+        self._currentStdout = StringIO()
+        self._currentStderr = StringIO()
+        sys.stdout = Tee(self.encoding, self._currentStdout, sys.stdout)
+        sys.stderr = Tee(self.encoding, self._currentStderr, sys.stderr)
+
+    def startContext(self, context):
+        self._startCapture()
+
+    def stopContext(self, context):
+        self._endCapture()
+
+    def beforeTest(self, test):
+        """Initializes a timer before starting a test."""
+        self._timer = time()
+        self._startCapture()
+
+    def _endCapture(self):
+        if self._capture_stack:
+            sys.stdout, sys.stderr = self._capture_stack.pop()
+
+    def afterTest(self, test):
+        self._endCapture()
+        self._currentStdout = None
+        self._currentStderr = None
+
+    def finalize(self, test):
+        while self._capture_stack:
+            self._endCapture()
+
+    def _getCapturedStdout(self):
+        if self._currentStdout:
+            value = self._currentStdout.getvalue()
+            if value:
+                return '<system-out><![CDATA[%s]]></system-out>' % escape_cdata(
+                        value)
+        return ''
+
+    def _getCapturedStderr(self):
+        if self._currentStderr:
+            value = self._currentStderr.getvalue()
+            if value:
+                return '<system-err><![CDATA[%s]]></system-err>' % escape_cdata(
+                        value)
+        return ''
+
+    def addError(self, test, err, capt=None):
+        """Add error output to Xunit report.
+        """
+        taken = self._timeTaken()
+
+        if issubclass(err[0], SkipTest):
+            type = 'skipped'
+            self.stats['skipped'] += 1
+        else:
+            type = 'error'
+            self.stats['errors'] += 1
+
+        tb = format_exception(err, self.encoding)
+        id = test.id()
+
+        self.errorlist.append(
+            u'<testcase classname=%(cls)s name=%(name)s time="%(taken).3f">'
+            u'<%(type)s type=%(errtype)s message=%(message)s><![CDATA[%(tb)s]]>'
+            u'</%(type)s>%(systemout)s%(systemerr)s</testcase>' %
+            {'cls': self._quoteattr(id_split(id)[0]),
+             'name': self._quoteattr(id_split(id)[-1]),
+             'taken': taken,
+             'type': type,
+             'errtype': self._quoteattr(nice_classname(err[0])),
+             'message': self._quoteattr(exc_message(err)),
+             'tb': escape_cdata(tb),
+             'systemout': self._getCapturedStdout(),
+             'systemerr': self._getCapturedStderr(),
+             })
+
+    def addFailure(self, test, err, capt=None, tb_info=None):
+        """Add failure output to Xunit report.
+        """
+        taken = self._timeTaken()
+        tb = format_exception(err, self.encoding)
+        self.stats['failures'] += 1
+        id = test.id()
+
+        self.errorlist.append(
+            u'<testcase classname=%(cls)s name=%(name)s time="%(taken).3f">'
+            u'<failure type=%(errtype)s message=%(message)s><![CDATA[%(tb)s]]>'
+            u'</failure>%(systemout)s%(systemerr)s</testcase>' %
+            {'cls': self._quoteattr(id_split(id)[0]),
+             'name': self._quoteattr(id_split(id)[-1]),
+             'taken': taken,
+             'errtype': self._quoteattr(nice_classname(err[0])),
+             'message': self._quoteattr(exc_message(err)),
+             'tb': escape_cdata(tb),
+             'systemout': self._getCapturedStdout(),
+             'systemerr': self._getCapturedStderr(),
+             })
+
+    def addSuccess(self, test, capt=None):
+        """Add success output to Xunit report.
+        """
+        taken = self._timeTaken()
+        self.stats['passes'] += 1
+        id = test.id()
+        self.errorlist.append(
+            '<testcase classname=%(cls)s name=%(name)s '
+            'time="%(taken).3f">%(systemout)s%(systemerr)s</testcase>' %
+            {'cls': self._quoteattr(id_split(id)[0]),
+             'name': self._quoteattr(id_split(id)[-1]),
+             'taken': taken,
+             'systemout': self._getCapturedStdout(),
+             'systemerr': self._getCapturedStderr(),
+             })

http://git-wip-us.apache.org/repos/asf/cassandra-dtest/blob/ac0ce604/run_dtests.py
----------------------------------------------------------------------
diff --git a/run_dtests.py b/run_dtests.py
index cb68827..4f2d187 100755
--- a/run_dtests.py
+++ b/run_dtests.py
@@ -23,9 +23,11 @@ example:
 from __future__ import print_function
 
 import subprocess
+import sys
+import os
 from collections import namedtuple
 from itertools import product
-from os import getcwd
+from os import getcwd, environ
 from tempfile import NamedTemporaryFile
 
 from docopt import docopt
@@ -212,9 +214,15 @@ if __name__ == '__main__':
         # How do we execute code in a new interpreter each time? Generate the
         # code as text, then shell out to a new interpreter.
         to_execute = (
-            'import nose\n'
-            'from plugins.dtestconfig import DtestConfigPlugin, GlobalConfigObject\n'
-            'nose.main(addplugins=[DtestConfigPlugin({config})])\n'
+            "import nose\n" +
+            "from plugins.dtestconfig import DtestConfigPlugin, GlobalConfigObject\n" +
+            "from plugins.dtestxunit import DTestXunit\n" +
+            "from plugins.dtesttag import DTestTag\n"  +
+            "from plugins.dtestcollect import DTestCollect\n" +
+            "import sys\n" +
+            "print sys.getrecursionlimit()\n" +
+            "print sys.setrecursionlimit(8000)\n" +
+            ("nose.main(addplugins=[DtestConfigPlugin({config}), DTestXunit(), DTestCollect(),
DTestTag()])\n" if "TEST_TAG" in environ else "nose.main(addplugins=[DtestConfigPlugin({config}),
DTestCollect(), DTestXunit()])\n")
         ).format(config=repr(config))
         temp = NamedTemporaryFile(dir=getcwd())
         debug('Writing the following to {}:'.format(temp.name))
@@ -228,7 +236,7 @@ if __name__ == '__main__':
         # command line are treated one way, args passed in as
         # nose.main(argv=...) are treated another. Compare with the options
         # -xsv for an example.
-        cmd_list = ['python', temp.name] + nose_argv
+        cmd_list = [sys.executable, temp.name] + nose_argv
         debug('subprocess.call-ing {cmd_list}'.format(cmd_list=cmd_list))
 
         if options['--dry-run']:
@@ -240,7 +248,7 @@ if __name__ == '__main__':
                 contents=contents
             ))
         else:
-            results.append(subprocess.call(cmd_list))
+            results.append(subprocess.call(cmd_list, env=os.environ.copy()))
         # separate the end of the last subprocess.call output from the
         # beginning of the next by printing a newline.
         print()


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


Mime
View raw message