Repository: cassandra-dtest Updated Branches: refs/heads/master 8e65211b9 -> ac0ce6044
Add plumbing to cassandra dtests to allow collecting test names for multiple builds as well as merging test output Patch by Ariel Weisberg; Reviewd by Michael Kjellman for CASSANDRA-14017 Project: http://git-wip-us.apache.org/repos/asf/cassandra-dtest/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra-dtest/commit/ac0ce604 Tree: http://git-wip-us.apache.org/repos/asf/cassandra-dtest/tree/ac0ce604 Diff: http://git-wip-us.apache.org/repos/asf/cassandra-dtest/diff/ac0ce604 Branch: refs/heads/master Commit: ac0ce60443ee59a8ad333f5d5a08bc6efa574a53 Parents: 8e65211 Author: Ariel Weisberg <aweisb...@apple.com> Authored: Wed Nov 15 16:17:39 2017 -0500 Committer: Ariel Weisberg <aweisb...@apple.com> Committed: Wed Nov 15 16:32:47 2017 -0500 ---------------------------------------------------------------------- plugins/dtestcollect.py | 93 ++++++++++++ plugins/dtesttag.py | 47 ++++++ plugins/dtestxunit.py | 346 +++++++++++++++++++++++++++++++++++++++++++ run_dtests.py | 20 ++- 4 files changed, 500 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra-dtest/blob/ac0ce604/plugins/dtestcollect.py ---------------------------------------------------------------------- diff --git a/plugins/dtestcollect.py b/plugins/dtestcollect.py new file mode 100644 index 0000000..e8a3c8a --- /dev/null +++ b/plugins/dtestcollect.py @@ -0,0 +1,93 @@ +from collections import namedtuple + +from pprint import pprint +import os +import inspect +from nose.plugins.base import Plugin +from nose.case import Test +import logging +import unittest + +log = logging.getLogger(__name__) + +class DTestCollect(Plugin): + """ + Collect and output test names only, don't run any tests. + """ + name = 'dtest_collect' + enableOpt = 'dtest_collect_only' + + def options(self, parser, env): + """Register commandline options. + """ + parser.add_option('--dtest-collect-only', + action='store_true', + dest=self.enableOpt, + default=env.get('DTEST_NOSE_COLLECT_ONLY'), + help="Enable collect-only: %s [COLLECT_ONLY]" % + (self.help())) + + def prepareTestLoader(self, loader): + """Install collect-only suite class in TestLoader. + """ + # Disable context awareness + log.debug("Preparing test loader") + loader.suiteClass = TestSuiteFactory(self.conf) + + def prepareTestCase(self, test): + """Replace actual test with dummy that always passes. + """ + # Return something that always passes + log.debug("Preparing test case %s", test) + if not isinstance(test, Test): + return + def run(result): + # We need to make these plugin calls because there won't be + # a result proxy, due to using a stripped-down test suite + self.conf.plugins.startTest(test) + result.startTest(test) + self.conf.plugins.addSuccess(test) + result.addSuccess(test) + self.conf.plugins.stopTest(test) + result.stopTest(test) + return run + + def describeTest(self, test): + tag = os.getenv('TEST_TAG', '') + if tag == '': + tag = test.test._testMethodName + else: + tag = test.test._testMethodName + "-" + tag + retval = "%s:%s.%s" % (test.test.__module__, test.test.__class__.__name__, tag) + return retval; + +class TestSuiteFactory: + """ + Factory for producing configured test suites. + """ + def __init__(self, conf): + self.conf = conf + + def __call__(self, tests=(), **kw): + return TestSuite(tests, conf=self.conf) + + +class TestSuite(unittest.TestSuite): + """ + Basic test suite that bypasses most proxy and plugin calls, but does + wrap tests in a nose.case.Test so prepareTestCase will be called. + """ + def __init__(self, tests=(), conf=None): + self.conf = conf + # Exec lazy suites: makes discovery depth-first + if callable(tests): + tests = tests() + log.debug("TestSuite(%r)", tests) + unittest.TestSuite.__init__(self, tests) + + def addTest(self, test): + log.debug("Add test %s", test) + if isinstance(test, unittest.TestSuite): + self._tests.append(test) + else: + self._tests.append(Test(test, config=self.conf)) http://git-wip-us.apache.org/repos/asf/cassandra-dtest/blob/ac0ce604/plugins/dtesttag.py ---------------------------------------------------------------------- diff --git a/plugins/dtesttag.py b/plugins/dtesttag.py new file mode 100644 index 0000000..94effcb --- /dev/null +++ b/plugins/dtesttag.py @@ -0,0 +1,47 @@ +from collections import namedtuple + +from nose import plugins +from pprint import pprint +import os +import inspect + +class DTestTag(plugins.Plugin): + enabled = True # if this plugin is loaded at all, we're using it + name = 'dtest_tag' + + def __init__(self): + pass + + def configure(self, options, conf): + pass + + def nice_classname(self, obj): + """Returns a nice name for class object or class instance. + + >>> nice_classname(Exception()) # doctest: +ELLIPSIS + '...Exception' + >>> nice_classname(Exception) # doctest: +ELLIPSIS + '...Exception' + + """ + if inspect.isclass(obj): + cls_name = obj.__name__ + else: + cls_name = obj.__class__.__name__ + mod = inspect.getmodule(obj) + if mod: + name = mod.__name__ + # jython + if name.startswith('org.python.core.'): + name = name[len('org.python.core.'):] + return "%s.%s" % (name, cls_name) + else: + return cls_name + + def describeTest(self, test): + tag = os.getenv('TEST_TAG', '') + if tag == '': + tag = test.test._testMethodName + else: + tag = test.test._testMethodName + "-" + tag + return "%s (%s)" % (tag, self.nice_classname(test.test)) http://git-wip-us.apache.org/repos/asf/cassandra-dtest/blob/ac0ce604/plugins/dtestxunit.py ---------------------------------------------------------------------- diff --git a/plugins/dtestxunit.py b/plugins/dtestxunit.py new file mode 100644 index 0000000..c4a041f --- /dev/null +++ b/plugins/dtestxunit.py @@ -0,0 +1,346 @@ +"""This plugin provides test results in the standard XUnit XML format. + +It's designed for the `Jenkins`_ (previously Hudson) continuous build +system, but will probably work for anything else that understands an +XUnit-formatted XML representation of test results. + +Add this shell command to your builder :: + + nosetests --with-dtestxunit + +And by default a file named nosetests.xml will be written to the +working directory. + +In a Jenkins builder, tick the box named "Publish JUnit test result report" +under the Post-build Actions and enter this value for Test report XMLs:: + + **/nosetests.xml + +If you need to change the name or location of the file, you can set the +``--dtestxunit-file`` option. + +If you need to change the name of the test suite, you can set the +``--dtestxunit-testsuite-name`` option. + +Here is an abbreviated version of what an XML test report might look like:: + + <?xml version="1.0" encoding="UTF-8"?> + <testsuite name="nosetests" tests="1" errors="1" failures="0" skip="0"> + <testcase classname="path_to_test_suite.TestSomething" + name="test_it" time="0"> + <error type="exceptions.TypeError" message="oops, wrong type"> + Traceback (most recent call last): + ... + TypeError: oops, wrong type + </error> + </testcase> + </testsuite> + +.. _Jenkins: http://jenkins-ci.org/ + +""" +import codecs +import doctest +import os +import sys +import traceback +import re +import inspect +from StringIO import StringIO +from time import time +from xml.sax import saxutils + +from nose.plugins.base import Plugin +from nose.exc import SkipTest +from nose.pyversion import force_unicode, format_exception + +# Invalid XML characters, control characters 0-31 sans \t, \n and \r +CONTROL_CHARACTERS = re.compile(r"[\000-\010\013\014\016-\037]") + +TEST_ID = re.compile(r'^(.*?)(\(.*\))$') + +def xml_safe(value): + """Replaces invalid XML characters with '?'.""" + return CONTROL_CHARACTERS.sub('?', value) + +def escape_cdata(cdata): + """Escape a string for an XML CDATA section.""" + return xml_safe(cdata).replace(']]>', ']]>]]><![CDATA[') + +def id_split(idval): + m = TEST_ID.match(idval) + retval = [] + if m: + name, fargs = m.groups() + head, tail = name.rsplit(".", 1) + retval = [head, tail+fargs] + else: + retval = idval.rsplit(".", 1) + tag = os.getenv('TEST_TAG', '') + if tag != '': + retval[-1] = retval[-1] + "-" + tag + return retval + +def nice_classname(obj): + """Returns a nice name for class object or class instance. + + >>> nice_classname(Exception()) # doctest: +ELLIPSIS + '...Exception' + >>> nice_classname(Exception) # doctest: +ELLIPSIS + '...Exception' + + """ + if inspect.isclass(obj): + cls_name = obj.__name__ + else: + cls_name = obj.__class__.__name__ + mod = inspect.getmodule(obj) + if mod: + name = mod.__name__ + # jython + if name.startswith('org.python.core.'): + name = name[len('org.python.core.'):] + return "%s.%s" % (name, cls_name) + else: + return cls_name + +def exc_message(exc_info): + """Return the exception's message.""" + exc = exc_info[1] + if exc is None: + # str exception + result = exc_info[0] + else: + try: + result = str(exc) + except UnicodeEncodeError: + try: + result = unicode(exc) + except UnicodeError: + # Fallback to args as neither str nor + # unicode(Exception(u'\xe6')) work in Python < 2.6 + result = exc.args[0] + result = force_unicode(result, 'UTF-8') + return xml_safe(result) + +class Tee(object): + def __init__(self, encoding, *args): + self._encoding = encoding + self._streams = args + + def write(self, data): + data = force_unicode(data, self._encoding) + for s in self._streams: + s.write(data) + + def writelines(self, lines): + for line in lines: + self.write(line) + + def flush(self): + for s in self._streams: + s.flush() + + def isatty(self): + return False + + +class DTestXunit(Plugin): + """This plugin provides test results in the standard XUnit XML format.""" + name = 'dtestxunit' + score = 1500 + encoding = 'UTF-8' + error_report_file = None + + def __init__(self): + super(DTestXunit, self).__init__() + self._capture_stack = [] + self._currentStdout = None + self._currentStderr = None + + def _timeTaken(self): + if hasattr(self, '_timer'): + taken = time() - self._timer + else: + # test died before it ran (probably error in setup()) + # or success/failure added before test started probably + # due to custom TestResult munging + taken = 0.0 + return taken + + def _quoteattr(self, attr): + """Escape an XML attribute. Value can be unicode.""" + attr = xml_safe(attr) + return saxutils.quoteattr(attr) + + def options(self, parser, env): + """Sets additional command line options.""" + Plugin.options(self, parser, env) + parser.add_option( + '--dtestxunit-file', action='store', + dest='dtestxunit_file', metavar="FILE", + default=env.get('NOSE_XUNIT_FILE', 'nosetests.xml'), + help=("Path to xml file to store the xunit report in. " + "Default is nosetests.xml in the working directory " + "[NOSE_XUNIT_FILE]")) + + parser.add_option( + '--dtestxunit-testsuite-name', action='store', + dest='dtestxunit_testsuite_name', metavar="PACKAGE", + default=env.get('NOSE_XUNIT_TESTSUITE_NAME', 'nosetests'), + help=("Name of the testsuite in the xunit xml, generated by plugin. " + "Default test suite name is nosetests.")) + + def configure(self, options, config): + """Configures the xunit plugin.""" + Plugin.configure(self, options, config) + self.config = config + if self.enabled: + self.stats = {'errors': 0, + 'failures': 0, + 'passes': 0, + 'skipped': 0 + } + self.errorlist = [] + self.error_report_file_name = os.path.realpath(options.dtestxunit_file) + self.xunit_testsuite_name = options.dtestxunit_testsuite_name + + def report(self, stream): + """Writes an Xunit-formatted XML file + + The file includes a report of test errors and failures. + + """ + self.error_report_file = codecs.open(self.error_report_file_name, 'w', + self.encoding, 'replace') + self.stats['encoding'] = self.encoding + self.stats['testsuite_name'] = self.xunit_testsuite_name + self.stats['total'] = (self.stats['errors'] + self.stats['failures'] + + self.stats['passes'] + self.stats['skipped']) + self.error_report_file.write( + u'<?xml version="1.0" encoding="%(encoding)s"?>' + u'<testsuite name="%(testsuite_name)s" tests="%(total)d" ' + u'errors="%(errors)d" failures="%(failures)d" ' + u'skip="%(skipped)d">' % self.stats) + self.error_report_file.write(u''.join([force_unicode(e, self.encoding) + for e in self.errorlist])) + self.error_report_file.write(u'</testsuite>') + self.error_report_file.close() + if self.config.verbosity > 1: + stream.writeln("-" * 70) + stream.writeln("XML: %s" % self.error_report_file.name) + + def _startCapture(self): + self._capture_stack.append((sys.stdout, sys.stderr)) + self._currentStdout = StringIO() + self._currentStderr = StringIO() + sys.stdout = Tee(self.encoding, self._currentStdout, sys.stdout) + sys.stderr = Tee(self.encoding, self._currentStderr, sys.stderr) + + def startContext(self, context): + self._startCapture() + + def stopContext(self, context): + self._endCapture() + + def beforeTest(self, test): + """Initializes a timer before starting a test.""" + self._timer = time() + self._startCapture() + + def _endCapture(self): + if self._capture_stack: + sys.stdout, sys.stderr = self._capture_stack.pop() + + def afterTest(self, test): + self._endCapture() + self._currentStdout = None + self._currentStderr = None + + def finalize(self, test): + while self._capture_stack: + self._endCapture() + + def _getCapturedStdout(self): + if self._currentStdout: + value = self._currentStdout.getvalue() + if value: + return '<system-out><![CDATA[%s]]></system-out>' % escape_cdata( + value) + return '' + + def _getCapturedStderr(self): + if self._currentStderr: + value = self._currentStderr.getvalue() + if value: + return '<system-err><![CDATA[%s]]></system-err>' % escape_cdata( + value) + return '' + + def addError(self, test, err, capt=None): + """Add error output to Xunit report. + """ + taken = self._timeTaken() + + if issubclass(err[0], SkipTest): + type = 'skipped' + self.stats['skipped'] += 1 + else: + type = 'error' + self.stats['errors'] += 1 + + tb = format_exception(err, self.encoding) + id = test.id() + + self.errorlist.append( + u'<testcase classname=%(cls)s name=%(name)s time="%(taken).3f">' + u'<%(type)s type=%(errtype)s message=%(message)s><![CDATA[%(tb)s]]>' + u'</%(type)s>%(systemout)s%(systemerr)s</testcase>' % + {'cls': self._quoteattr(id_split(id)[0]), + 'name': self._quoteattr(id_split(id)[-1]), + 'taken': taken, + 'type': type, + 'errtype': self._quoteattr(nice_classname(err[0])), + 'message': self._quoteattr(exc_message(err)), + 'tb': escape_cdata(tb), + 'systemout': self._getCapturedStdout(), + 'systemerr': self._getCapturedStderr(), + }) + + def addFailure(self, test, err, capt=None, tb_info=None): + """Add failure output to Xunit report. + """ + taken = self._timeTaken() + tb = format_exception(err, self.encoding) + self.stats['failures'] += 1 + id = test.id() + + self.errorlist.append( + u'<testcase classname=%(cls)s name=%(name)s time="%(taken).3f">' + u'<failure type=%(errtype)s message=%(message)s><![CDATA[%(tb)s]]>' + u'</failure>%(systemout)s%(systemerr)s</testcase>' % + {'cls': self._quoteattr(id_split(id)[0]), + 'name': self._quoteattr(id_split(id)[-1]), + 'taken': taken, + 'errtype': self._quoteattr(nice_classname(err[0])), + 'message': self._quoteattr(exc_message(err)), + 'tb': escape_cdata(tb), + 'systemout': self._getCapturedStdout(), + 'systemerr': self._getCapturedStderr(), + }) + + def addSuccess(self, test, capt=None): + """Add success output to Xunit report. + """ + taken = self._timeTaken() + self.stats['passes'] += 1 + id = test.id() + self.errorlist.append( + '<testcase classname=%(cls)s name=%(name)s ' + 'time="%(taken).3f">%(systemout)s%(systemerr)s</testcase>' % + {'cls': self._quoteattr(id_split(id)[0]), + 'name': self._quoteattr(id_split(id)[-1]), + 'taken': taken, + 'systemout': self._getCapturedStdout(), + 'systemerr': self._getCapturedStderr(), + }) http://git-wip-us.apache.org/repos/asf/cassandra-dtest/blob/ac0ce604/run_dtests.py ---------------------------------------------------------------------- diff --git a/run_dtests.py b/run_dtests.py index cb68827..4f2d187 100755 --- a/run_dtests.py +++ b/run_dtests.py @@ -23,9 +23,11 @@ example: from __future__ import print_function import subprocess +import sys +import os from collections import namedtuple from itertools import product -from os import getcwd +from os import getcwd, environ from tempfile import NamedTemporaryFile from docopt import docopt @@ -212,9 +214,15 @@ if __name__ == '__main__': # How do we execute code in a new interpreter each time? Generate the # code as text, then shell out to a new interpreter. to_execute = ( - 'import nose\n' - 'from plugins.dtestconfig import DtestConfigPlugin, GlobalConfigObject\n' - 'nose.main(addplugins=[DtestConfigPlugin({config})])\n' + "import nose\n" + + "from plugins.dtestconfig import DtestConfigPlugin, GlobalConfigObject\n" + + "from plugins.dtestxunit import DTestXunit\n" + + "from plugins.dtesttag import DTestTag\n" + + "from plugins.dtestcollect import DTestCollect\n" + + "import sys\n" + + "print sys.getrecursionlimit()\n" + + "print sys.setrecursionlimit(8000)\n" + + ("nose.main(addplugins=[DtestConfigPlugin({config}), DTestXunit(), DTestCollect(), DTestTag()])\n" if "TEST_TAG" in environ else "nose.main(addplugins=[DtestConfigPlugin({config}), DTestCollect(), DTestXunit()])\n") ).format(config=repr(config)) temp = NamedTemporaryFile(dir=getcwd()) debug('Writing the following to {}:'.format(temp.name)) @@ -228,7 +236,7 @@ if __name__ == '__main__': # command line are treated one way, args passed in as # nose.main(argv=...) are treated another. Compare with the options # -xsv for an example. - cmd_list = ['python', temp.name] + nose_argv + cmd_list = [sys.executable, temp.name] + nose_argv debug('subprocess.call-ing {cmd_list}'.format(cmd_list=cmd_list)) if options['--dry-run']: @@ -240,7 +248,7 @@ if __name__ == '__main__': contents=contents )) else: - results.append(subprocess.call(cmd_list)) + results.append(subprocess.call(cmd_list, env=os.environ.copy())) # separate the end of the last subprocess.call output from the # beginning of the next by printing a newline. print() --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org