Diff
Modified: trunk/Tools/ChangeLog (112297 => 112298)
--- trunk/Tools/ChangeLog 2012-03-27 19:08:02 UTC (rev 112297)
+++ trunk/Tools/ChangeLog 2012-03-27 19:15:53 UTC (rev 112298)
@@ -1,3 +1,53 @@
+2012-03-27 Dirk Pranke <dpra...@chromium.org>
+
+ test-webkitpy: prepare for better test run output
+ https://bugs.webkit.org/show_bug.cgi?id=82290
+
+ Reviewed by Adam Barth.
+
+ This code basically re-implements the output of the TextTestRunner default
+ runner code from unittest, although the implementation is quite
+ different, in preparation for changing the test output to be
+ metered and possibly running in parallel.
+
+ The output is almost identical to before, except that instead of
+ logging "test_regular (webkitpy.main.RunnerTest) passed" we log
+ "webkitpy.main.RunnerTest.test_regular passed". It has always
+ annoyed me that they invert the names to be harder to read and
+ so that you can't copy & paste back to the input for
+ test-webkitpy.
+
+ This patch is provided to add a skeleton for unit tests and for
+ comparison to an upcoming patch that will actually add new
+ functionality.
+
+ * Scripts/webkitpy/test/main.py:
+ (Tester.__init__):
+ (Tester._configure_logging):
+ (Tester._run_tests):
+ * Scripts/webkitpy/test/runner.py: Added.
+ (TestRunner):
+ (TestRunner.__init__):
+ (TestRunner.test_name):
+ (TestRunner.all_test_names):
+ (TestRunner.run):
+ (TestRunner.write_result):
+ (TestRunner.write_summary):
+ * Scripts/webkitpy/test/runner_unittest.py: Added.
+ (FakeModuleSuite):
+ (FakeModuleSuite.__init__):
+ (FakeModuleSuite.__str__):
+ (FakeModuleSuite.run):
+ (FakeTopSuite):
+ (FakeTopSuite.__init__):
+ (FakeLoader):
+ (FakeLoader.__init__):
+ (FakeLoader.top_suite):
+ (FakeLoader.loadTestsFromName):
+ (RunnerTest):
+ (RunnerTest.test_regular):
+ (RunnerTest.test_verbose):
+
2012-03-27 Gustavo Noronha Silva <g...@gnome.org>
[GTK] Build gnutls without p11-kit support
Modified: trunk/Tools/Scripts/webkitpy/test/main.py (112297 => 112298)
--- trunk/Tools/Scripts/webkitpy/test/main.py 2012-03-27 19:08:02 UTC (rev 112297)
+++ trunk/Tools/Scripts/webkitpy/test/main.py 2012-03-27 19:15:53 UTC (rev 112298)
@@ -32,6 +32,7 @@
from webkitpy.common.system.filesystem import FileSystem
from webkitpy.test.test_finder import TestFinder
+from webkitpy.test.runner import TestRunner
_log = logging.getLogger(__name__)
@@ -40,6 +41,7 @@
def __init__(self, filesystem=None):
self._verbosity = 1
self.finder = TestFinder(filesystem or FileSystem())
+ self.stream = sys.stderr
def add_tree(self, top_directory, starting_subdirectory=None):
self.finder.add_tree(top_directory, starting_subdirectory)
@@ -92,7 +94,7 @@
except for messages from the autoinstall module. Also set the
logging level as described below.
"""
- handler = logging.StreamHandler(sys.stderr)
+ handler = logging.StreamHandler(self.stream)
# We constrain the level on the handler rather than on the root
# logger itself. This is probably better because the handler is
# configured and known only to this module, whereas the root logger
@@ -142,7 +144,7 @@
names = self.finder.find_names(args, self._options.skip_integrationtests, self._options.all)
return self._run_tests(names)
- def _run_tests(self, args):
+ def _run_tests(self, names):
if self._options.coverage:
try:
import webkitpy.thirdparty.autoinstalled.coverage as coverage
@@ -159,16 +161,18 @@
loader = unittest.defaultTestLoader
suites = []
- for name in args:
+ for name in names:
if self.finder.is_module(name):
- # import modules explicitly before loading their tests because
- # loadTestsFromName() produces lousy error messages for bad modules.
+ # if we failed to load a name and it looks like a module,
+ # try importing it directly, because loadTestsFromName()
+ # produces lousy error messages for bad modules.
try:
__import__(name)
except ImportError, e:
_log.fatal('Failed to import %s:' % name)
self._log_exception()
return False
+
suites.append(loader.loadTestsFromName(name, None))
test_suite = unittest.TestSuite(suites)
@@ -176,7 +180,7 @@
from webkitpy.thirdparty.autoinstalled.xmlrunner import XMLTestRunner
test_runner = XMLTestRunner(output='test-webkitpy-xml-reports')
else:
- test_runner = unittest.TextTestRunner(verbosity=self._verbosity)
+ test_runner = TestRunner(self.stream, self._options, loader)
_log.debug("Running the tests.")
result = test_runner.run(test_suite)
Added: trunk/Tools/Scripts/webkitpy/test/runner.py (0 => 112298)
--- trunk/Tools/Scripts/webkitpy/test/runner.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/test/runner.py 2012-03-27 19:15:53 UTC (rev 112298)
@@ -0,0 +1,129 @@
+# Copyright (C) 2012 Google, Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""code to actually run a list of python tests."""
+
+import logging
+import re
+import time
+import unittest
+
+
+_log = logging.getLogger(__name__)
+
+
+class TestRunner(object):
+ def __init__(self, stream, options, loader):
+ self.options = options
+ self.stream = stream
+ self.loader = loader
+ self.test_description = re.compile("(\w+) \(([\w.]+)\)")
+
+ def test_name(self, test):
+ m = self.test_description.match(str(test))
+ return "%s.%s" % (m.group(2), m.group(1))
+
+ def all_test_names(self, suite):
+ names = []
+ if hasattr(suite, '_tests'):
+ for t in suite._tests:
+ names.extend(self.all_test_names(t))
+ else:
+ names.append(self.test_name(suite))
+ return names
+
+ def run(self, suite):
+ run_start_time = time.time()
+ all_test_names = self.all_test_names(suite)
+ result = unittest.TestResult()
+ stop = run_start_time
+ for test_name in all_test_names:
+ if self.options.verbose:
+ self.stream.write(test_name)
+ num_failures = len(result.failures)
+ num_errors = len(result.errors)
+
+ start = time.time()
+ # FIXME: it's kinda lame that we re-load the test suites for each
+ # test, and this may slow things down, but this makes implementing
+ # the logging easy and will also allow us to parallelize nicely.
+ self.loader.loadTestsFromName(test_name, None).run(result)
+ stop = time.time()
+
+ err = None
+ failure = None
+ if len(result.failures) > num_failures:
+ failure = result.failures[num_failures][1]
+ elif len(result.errors) > num_errors:
+ err = result.errors[num_errors][1]
+ self.write_result(result, test_name, stop - start, failure, err)
+
+ self.write_summary(result, stop - run_start_time)
+
+ return result
+
+ def write_result(self, result, test_name, test_time, failure=None, err=None):
+ if self.options.verbose:
+ if failure:
+ msg = ' failed'
+ elif err:
+ msg = ' erred'
+ else:
+ msg = ' passed'
+ self.stream.write(msg + '\n')
+ else:
+ if failure:
+ msg = 'F'
+ elif err:
+ msg = 'E'
+ else:
+ msg = '.'
+ self.stream.write(msg)
+
+ def write_summary(self, result, run_time):
+ self.stream.write('\n')
+
+ for (test, err) in result.errors:
+ self.stream.write("=" * 80 + '\n')
+ self.stream.write("ERROR: " + self.test_name(test) + '\n')
+ self.stream.write("-" * 80 + '\n')
+ for line in err.splitlines():
+ self.stream.write(line + '\n')
+ self.stream.write('\n')
+
+ for (test, failure) in result.failures:
+ self.stream.write("=" * 80 + '\n')
+ self.stream.write("FAILURE: " + self.test_name(test) + '\n')
+ self.stream.write("-" * 80 + '\n')
+ for line in failure.splitlines():
+ self.stream.write(line + '\n')
+ self.stream.write('\n')
+
+ self.stream.write('-' * 80 + '\n')
+ self.stream.write('Ran %d test%s in %.3fs\n' %
+ (result.testsRun, result.testsRun != 1 and "s" or "", run_time))
+
+ if result.wasSuccessful():
+ self.stream.write('\nOK\n')
+ else:
+ self.stream.write('FAILED (failures=%d, errors=%d)\n' %
+ (len(result.failures), len(result.errors)))
Added: trunk/Tools/Scripts/webkitpy/test/runner_unittest.py (0 => 112298)
--- trunk/Tools/Scripts/webkitpy/test/runner_unittest.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/test/runner_unittest.py 2012-03-27 19:15:53 UTC (rev 112298)
@@ -0,0 +1,99 @@
+# Copyright (C) 2012 Google, Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import re
+import StringIO
+import unittest
+
+from webkitpy.tool.mocktool import MockOptions
+from webkitpy.test.runner import TestRunner
+
+
+class FakeModuleSuite(object):
+ def __init__(self, name, result, msg):
+ self.name = name
+ self.result = result
+ self.msg = msg
+
+ def __str__(self):
+ return self.name
+
+ def run(self, result):
+ result.testsRun += 1
+ if self.result == 'F':
+ result.failures.append((self.name, self.msg))
+ elif self.result == 'E':
+ result.errors.append((self.name, self.msg))
+
+
+class FakeTopSuite(object):
+ def __init__(self, tests):
+ self._tests = tests
+
+
+class FakeLoader(object):
+ def __init__(self, *test_triples):
+ self.triples = test_triples
+ self._tests = []
+ self._results = {}
+ for test_name, result, msg in self.triples:
+ self._tests.append(test_name)
+ m = re.match("(\w+) \(([\w.]+)\)", test_name)
+ self._results['%s.%s' % (m.group(2), m.group(1))] = tuple([test_name, result, msg])
+
+ def top_suite(self):
+ return FakeTopSuite(self._tests)
+
+ def loadTestsFromName(self, name, dummy):
+ return FakeModuleSuite(*self._results[name])
+
+
+class RunnerTest(unittest.TestCase):
+ def test_regular(self):
+ options = MockOptions(verbose=False)
+ stream = StringIO.StringIO()
+ loader = FakeLoader(('test1 (Foo)', '.', ''),
+ ('test2 (Foo)', 'F', 'test2\nfailed'),
+ ('test3 (Foo)', 'E', 'test3\nerred'))
+ result = TestRunner(stream, options, loader).run(loader.top_suite())
+ self.assertFalse(result.wasSuccessful())
+ self.assertEquals(result.testsRun, 3)
+ self.assertEquals(len(result.failures), 1)
+ self.assertEquals(len(result.errors), 1)
+ # FIXME: check the output from the test
+
+ def test_verbose(self):
+ options = MockOptions(verbose=True)
+ stream = StringIO.StringIO()
+ loader = FakeLoader(('test1 (Foo)', '.', ''),
+ ('test2 (Foo)', 'F', 'test2\nfailed'),
+ ('test3 (Foo)', 'E', 'test3\nerred'))
+ result = TestRunner(stream, options, loader).run(loader.top_suite())
+ self.assertFalse(result.wasSuccessful())
+ self.assertEquals(result.testsRun, 3)
+ self.assertEquals(len(result.failures), 1)
+ self.assertEquals(len(result.errors), 1)
+ # FIXME: check the output from the test
+
+
+if __name__ == '__main__':
+ unittest.main()
