This is okay, but I was planning on breaking that out into multiple files. Right now the test_results module also has the EventBuilder in it, which is not a results formatter. But we can break that out separately.
On Mon, Dec 7, 2015 at 1:23 PM, Zachary Turner via lldb-commits < lldb-commits@lists.llvm.org> wrote: > Author: zturner > Date: Mon Dec 7 15:23:41 2015 > New Revision: 254946 > > URL: http://llvm.org/viewvc/llvm-project?rev=254946&view=rev > Log: > Rename test_results.py to result_formatter.py. > > There is already a class called LLDBTestResults which I would like > to move into a separate file, but the most appropriate filename > was taken. > > Added: > lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py > - copied, changed from r254944, > lldb/trunk/packages/Python/lldbsuite/test/test_results.py > Removed: > lldb/trunk/packages/Python/lldbsuite/test/test_results.py > Modified: > lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py > lldb/trunk/packages/Python/lldbsuite/test/curses_results.py > lldb/trunk/packages/Python/lldbsuite/test/dosep.py > lldb/trunk/packages/Python/lldbsuite/test/dotest.py > > Modified: > lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py > URL: > http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py?rev=254946&r1=254945&r2=254946&view=diff > > ============================================================================== > --- lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py > (original) > +++ lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py > Mon Dec 7 15:23:41 2015 > @@ -13,10 +13,11 @@ from __future__ import print_function > import os > > # Our imports > -from . import test_results > +from . import result_formatter > import lldbsuite > > -class BasicResultsFormatter(test_results.ResultsFormatter): > + > +class BasicResultsFormatter(result_formatter.ResultsFormatter): > """Provides basic test result output.""" > @classmethod > def arg_parser(cls): > @@ -240,16 +241,16 @@ class BasicResultsFormatter(test_results > # Output each of the test result entries. > categories = [ > # result id, printed name, print matching tests?, detail label > - [test_results.EventBuilder.STATUS_SUCCESS, > + [result_formatter.EventBuilder.STATUS_SUCCESS, > "Success", False, None], > - [test_results.EventBuilder.STATUS_EXPECTED_FAILURE, > + [result_formatter.EventBuilder.STATUS_EXPECTED_FAILURE, > "Expected Failure", False, None], > - [test_results.EventBuilder.STATUS_FAILURE, > + [result_formatter.EventBuilder.STATUS_FAILURE, > "Failure", True, "FAIL"], > - [test_results.EventBuilder.STATUS_ERROR, "Error", True, > "ERROR"], > - [test_results.EventBuilder.STATUS_UNEXPECTED_SUCCESS, > + [result_formatter.EventBuilder.STATUS_ERROR, "Error", True, > "ERROR"], > + [result_formatter.EventBuilder.STATUS_UNEXPECTED_SUCCESS, > "Unexpected Success", True, "UNEXPECTED SUCCESS"], > - [test_results.EventBuilder.STATUS_SKIP, "Skip", False, None]] > + [result_formatter.EventBuilder.STATUS_SKIP, "Skip", False, > None]] > > # Partition all the events by test result status > result_events_by_status = self._partition_results_by_status( > > Modified: lldb/trunk/packages/Python/lldbsuite/test/curses_results.py > URL: > http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/curses_results.py?rev=254946&r1=254945&r2=254946&view=diff > > ============================================================================== > --- lldb/trunk/packages/Python/lldbsuite/test/curses_results.py (original) > +++ lldb/trunk/packages/Python/lldbsuite/test/curses_results.py Mon Dec 7 > 15:23:41 2015 > @@ -23,11 +23,11 @@ import time > > # LLDB modules > from . import lldbcurses > -from . import test_results > -from .test_results import EventBuilder > +from . import result_formatter > +from .result_formatter import EventBuilder > > > -class Curses(test_results.ResultsFormatter): > +class Curses(result_formatter.ResultsFormatter): > """Receives live results from tests that are running and reports them > to the terminal in a curses GUI""" > > def __init__(self, out_file, options): > > Modified: lldb/trunk/packages/Python/lldbsuite/test/dosep.py > URL: > http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/dosep.py?rev=254946&r1=254945&r2=254946&view=diff > > ============================================================================== > --- lldb/trunk/packages/Python/lldbsuite/test/dosep.py (original) > +++ lldb/trunk/packages/Python/lldbsuite/test/dosep.py Mon Dec 7 15:23:41 > 2015 > @@ -53,7 +53,7 @@ import lldbsuite.support.seven as seven > > from . import dotest_channels > from . import dotest_args > -from . import test_results > +from . import result_formatter > > # Todo: Convert this folder layout to be relative-import friendly and > don't hack up > # sys.path like this > @@ -1429,9 +1429,9 @@ def main(print_details_on_success, num_t > # Figure out exit code by count of test result types. > issue_count = ( > results_formatter.counts_by_test_result_status( > - test_results.EventBuilder.STATUS_ERROR) + > + result_formatter.EventBuilder.STATUS_ERROR) + > results_formatter.counts_by_test_result_status( > - test_results.EventBuilder.STATUS_FAILURE) + > + result_formatter.EventBuilder.STATUS_FAILURE) + > timeout_count) > # Return with appropriate result code > if issue_count > 0: > > Modified: lldb/trunk/packages/Python/lldbsuite/test/dotest.py > URL: > http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/dotest.py?rev=254946&r1=254945&r2=254946&view=diff > > ============================================================================== > --- lldb/trunk/packages/Python/lldbsuite/test/dotest.py (original) > +++ lldb/trunk/packages/Python/lldbsuite/test/dotest.py Mon Dec 7 > 15:23:41 2015 > @@ -43,8 +43,8 @@ import lldbsuite > from . import dotest_args > from . import lldbtest_config > from . import test_categories > -from . import test_results > -from .test_results import EventBuilder > +from . import result_formatter > +from .result_formatter import EventBuilder > from ..support import seven > > def is_exe(fpath): > @@ -795,7 +795,7 @@ def parseOptionsAndInitTestdirs(): > # Tell the event builder to create all events with these > # key/val pairs in them. > if len(entries) > 0: > - test_results.EventBuilder.add_entries_to_all_events(entries) > + > result_formatter.EventBuilder.add_entries_to_all_events(entries) > > # Gather all the dirs passed on the command line. > if len(args.args) > 0: > @@ -930,13 +930,13 @@ def setupTestResults(): > else: > results_file_object = open(results_filename, "w") > cleanup_func = results_file_object.close > - default_formatter_name = > "lldbsuite.test.test_results.XunitFormatter" > + default_formatter_name = > "lldbsuite.test.result_formatter.XunitFormatter" > elif results_port: > # Connect to the specified localhost port. > results_file_object, cleanup_func = createSocketToLocalPort( > results_port) > default_formatter_name = ( > - "lldbsuite.test.test_results.RawPickledFormatter") > + "lldbsuite.test.result_formatter.RawPickledFormatter") > > # If we have a results formatter name specified and we didn't specify > # a results file, we should use stdout. > > Copied: lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py > (from r254944, lldb/trunk/packages/Python/lldbsuite/test/test_results.py) > URL: > http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py?p2=lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py&p1=lldb/trunk/packages/Python/lldbsuite/test/test_results.py&r1=254944&r2=254946&rev=254946&view=diff > > ============================================================================== > (empty) > > Removed: lldb/trunk/packages/Python/lldbsuite/test/test_results.py > URL: > http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/test_results.py?rev=254945&view=auto > > ============================================================================== > --- lldb/trunk/packages/Python/lldbsuite/test/test_results.py (original) > +++ lldb/trunk/packages/Python/lldbsuite/test/test_results.py (removed) > @@ -1,1042 +0,0 @@ > -""" > - The LLVM Compiler Infrastructure > - > -This file is distributed under the University of Illinois Open Source > -License. See LICENSE.TXT for details. > - > -Provides classes used by the test results reporting infrastructure > -within the LLDB test suite. > -""" > - > -from __future__ import print_function > -from __future__ import absolute_import > - > -# System modules > -import argparse > -import inspect > -import os > -import pprint > -import re > -import sys > -import threading > -import time > -import traceback > -import xml.sax.saxutils > - > -# Third-party modules > -import six > -from six.moves import cPickle > - > -# LLDB modules > - > - > -class EventBuilder(object): > - """Helper class to build test result event dictionaries.""" > - > - BASE_DICTIONARY = None > - > - # Test Status Tags > - STATUS_SUCCESS = "success" > - STATUS_FAILURE = "failure" > - STATUS_EXPECTED_FAILURE = "expected_failure" > - STATUS_UNEXPECTED_SUCCESS = "unexpected_success" > - STATUS_SKIP = "skip" > - STATUS_ERROR = "error" > - > - @staticmethod > - def _get_test_name_info(test): > - """Returns (test-class-name, test-method-name) from a test case > instance. > - > - @param test a unittest.TestCase instance. > - > - @return tuple containing (test class name, test method name) > - """ > - test_class_components = test.id().split(".") > - test_class_name = ".".join(test_class_components[:-1]) > - test_name = test_class_components[-1] > - return (test_class_name, test_name) > - > - @staticmethod > - def bare_event(event_type): > - """Creates an event with default additions, event type and > timestamp. > - > - @param event_type the value set for the "event" key, used > - to distinguish events. > - > - @returns an event dictionary with all default additions, the > "event" > - key set to the passed in event_type, and the event_time value set > to > - time.time(). > - """ > - if EventBuilder.BASE_DICTIONARY is not None: > - # Start with a copy of the "always include" entries. > - event = dict(EventBuilder.BASE_DICTIONARY) > - else: > - event = {} > - > - event.update({ > - "event": event_type, > - "event_time": time.time() > - }) > - return event > - > - @staticmethod > - def _event_dictionary_common(test, event_type): > - """Returns an event dictionary setup with values for the given > event type. > - > - @param test the unittest.TestCase instance > - > - @param event_type the name of the event type (string). > - > - @return event dictionary with common event fields set. > - """ > - test_class_name, test_name = > EventBuilder._get_test_name_info(test) > - > - event = EventBuilder.bare_event(event_type) > - event.update({ > - "test_class": test_class_name, > - "test_name": test_name, > - "test_filename": inspect.getfile(test.__class__) > - }) > - return event > - > - @staticmethod > - def _error_tuple_class(error_tuple): > - """Returns the unittest error tuple's error class as a string. > - > - @param error_tuple the error tuple provided by the test framework. > - > - @return the error type (typically an exception) raised by the > - test framework. > - """ > - type_var = error_tuple[0] > - module = inspect.getmodule(type_var) > - if module: > - return "{}.{}".format(module.__name__, type_var.__name__) > - else: > - return type_var.__name__ > - > - @staticmethod > - def _error_tuple_message(error_tuple): > - """Returns the unittest error tuple's error message. > - > - @param error_tuple the error tuple provided by the test framework. > - > - @return the error message provided by the test framework. > - """ > - return str(error_tuple[1]) > - > - @staticmethod > - def _error_tuple_traceback(error_tuple): > - """Returns the unittest error tuple's error message. > - > - @param error_tuple the error tuple provided by the test framework. > - > - @return the error message provided by the test framework. > - """ > - return error_tuple[2] > - > - @staticmethod > - def _event_dictionary_test_result(test, status): > - """Returns an event dictionary with common test result fields set. > - > - @param test a unittest.TestCase instance. > - > - @param status the status/result of the test > - (e.g. "success", "failure", etc.) > - > - @return the event dictionary > - """ > - event = EventBuilder._event_dictionary_common(test, "test_result") > - event["status"] = status > - return event > - > - @staticmethod > - def _event_dictionary_issue(test, status, error_tuple): > - """Returns an event dictionary with common issue-containing test > result > - fields set. > - > - @param test a unittest.TestCase instance. > - > - @param status the status/result of the test > - (e.g. "success", "failure", etc.) > - > - @param error_tuple the error tuple as reported by the test runner. > - This is of the form (type<error>, error). > - > - @return the event dictionary > - """ > - event = EventBuilder._event_dictionary_test_result(test, status) > - event["issue_class"] = > EventBuilder._error_tuple_class(error_tuple) > - event["issue_message"] = > EventBuilder._error_tuple_message(error_tuple) > - backtrace = EventBuilder._error_tuple_traceback(error_tuple) > - if backtrace is not None: > - event["issue_backtrace"] = traceback.format_tb(backtrace) > - return event > - > - @staticmethod > - def event_for_start(test): > - """Returns an event dictionary for the test start event. > - > - @param test a unittest.TestCase instance. > - > - @return the event dictionary > - """ > - return EventBuilder._event_dictionary_common(test, "test_start") > - > - @staticmethod > - def event_for_success(test): > - """Returns an event dictionary for a successful test. > - > - @param test a unittest.TestCase instance. > - > - @return the event dictionary > - """ > - return EventBuilder._event_dictionary_test_result( > - test, EventBuilder.STATUS_SUCCESS) > - > - @staticmethod > - def event_for_unexpected_success(test, bugnumber): > - """Returns an event dictionary for a test that succeeded but was > - expected to fail. > - > - @param test a unittest.TestCase instance. > - > - @param bugnumber the issue identifier for the bug tracking the > - fix request for the test expected to fail (but is in fact > - passing here). > - > - @return the event dictionary > - > - """ > - event = EventBuilder._event_dictionary_test_result( > - test, EventBuilder.STATUS_UNEXPECTED_SUCCESS) > - if bugnumber: > - event["bugnumber"] = str(bugnumber) > - return event > - > - @staticmethod > - def event_for_failure(test, error_tuple): > - """Returns an event dictionary for a test that failed. > - > - @param test a unittest.TestCase instance. > - > - @param error_tuple the error tuple as reported by the test runner. > - This is of the form (type<error>, error). > - > - @return the event dictionary > - """ > - return EventBuilder._event_dictionary_issue( > - test, EventBuilder.STATUS_FAILURE, error_tuple) > - > - @staticmethod > - def event_for_expected_failure(test, error_tuple, bugnumber): > - """Returns an event dictionary for a test that failed as expected. > - > - @param test a unittest.TestCase instance. > - > - @param error_tuple the error tuple as reported by the test runner. > - This is of the form (type<error>, error). > - > - @param bugnumber the issue identifier for the bug tracking the > - fix request for the test expected to fail. > - > - @return the event dictionary > - > - """ > - event = EventBuilder._event_dictionary_issue( > - test, EventBuilder.STATUS_EXPECTED_FAILURE, error_tuple) > - if bugnumber: > - event["bugnumber"] = str(bugnumber) > - return event > - > - @staticmethod > - def event_for_skip(test, reason): > - """Returns an event dictionary for a test that was skipped. > - > - @param test a unittest.TestCase instance. > - > - @param reason the reason why the test is being skipped. > - > - @return the event dictionary > - """ > - event = EventBuilder._event_dictionary_test_result( > - test, EventBuilder.STATUS_SKIP) > - event["skip_reason"] = reason > - return event > - > - @staticmethod > - def event_for_error(test, error_tuple): > - """Returns an event dictionary for a test that hit a test > execution error. > - > - @param test a unittest.TestCase instance. > - > - @param error_tuple the error tuple as reported by the test runner. > - This is of the form (type<error>, error). > - > - @return the event dictionary > - """ > - return EventBuilder._event_dictionary_issue( > - test, EventBuilder.STATUS_ERROR, error_tuple) > - > - @staticmethod > - def event_for_cleanup_error(test, error_tuple): > - """Returns an event dictionary for a test that hit a test > execution error > - during the test cleanup phase. > - > - @param test a unittest.TestCase instance. > - > - @param error_tuple the error tuple as reported by the test runner. > - This is of the form (type<error>, error). > - > - @return the event dictionary > - """ > - event = EventBuilder._event_dictionary_issue( > - test, EventBuilder.STATUS_ERROR, error_tuple) > - event["issue_phase"] = "cleanup" > - return event > - > - @staticmethod > - def add_entries_to_all_events(entries_dict): > - """Specifies a dictionary of entries to add to all test events. > - > - This provides a mechanism for, say, a parallel test runner to > - indicate to each inferior dotest.py that it should add a > - worker index to each. > - > - Calling this method replaces all previous entries added > - by a prior call to this. > - > - Event build methods will overwrite any entries that collide. > - Thus, the passed in dictionary is the base, which gets merged > - over by event building when keys collide. > - > - @param entries_dict a dictionary containing key and value > - pairs that should be merged into all events created by the > - event generator. May be None to clear out any extra entries. > - """ > - EventBuilder.BASE_DICTIONARY = dict(entries_dict) > - > - > -class ResultsFormatter(object): > - > - """Provides interface to formatting test results out to a file-like > object. > - > - This class allows the LLDB test framework's raw test-realted > - events to be processed and formatted in any manner desired. > - Test events are represented by python dictionaries, formatted > - as in the EventBuilder class above. > - > - ResultFormatter instances are given a file-like object in which > - to write their results. > - > - ResultFormatter lifetime looks like the following: > - > - # The result formatter is created. > - # The argparse options dictionary is generated from calling > - # the SomeResultFormatter.arg_parser() with the options data > - # passed to dotest.py via the "--results-formatter-options" > - # argument. See the help on that for syntactic requirements > - # on getting that parsed correctly. > - formatter = SomeResultFormatter(file_like_object, > argpared_options_dict) > - > - # Single call to session start, before parsing any events. > - formatter.begin_session() > - > - formatter.handle_event({"event":"initialize",...}) > - > - # Zero or more calls specified for events recorded during the test > session. > - # The parallel test runner manages getting results from all the > inferior > - # dotest processes, so from a new format perspective, don't worry > about > - # that. The formatter will be presented with a single stream of > events > - # sandwiched between a single begin_session()/end_session() pair in > the > - # parallel test runner process/thread. > - for event in zero_or_more_test_events(): > - formatter.handle_event(event) > - > - # Single call to terminate/wrap-up. Formatters that need all the > - # data before they can print a correct result (e.g. xUnit/JUnit), > - # this is where the final report can be generated. > - formatter.handle_event({"event":"terminate",...}) > - > - It is not the formatter's responsibility to close the > file_like_object. > - (i.e. do not close it). > - > - The lldb test framework passes these test events in real time, so they > - arrive as they come in. > - > - In the case of the parallel test runner, the dotest inferiors > - add a 'pid' field to the dictionary that indicates which inferior > - pid generated the event. > - > - Note more events may be added in the future to support richer test > - reporting functionality. One example: creating a true flaky test > - result category so that unexpected successes really mean the test > - is marked incorrectly (either should be marked flaky, or is indeed > - passing consistently now and should have the xfail marker > - removed). In this case, a flaky_success and flaky_fail event > - likely will be added to capture these and support reporting things > - like percentages of flaky test passing so we can see if we're > - making some things worse/better with regards to failure rates. > - > - Another example: announcing all the test methods that are planned > - to be run, so we can better support redo operations of various kinds > - (redo all non-run tests, redo non-run tests except the one that > - was running [perhaps crashed], etc.) > - > - Implementers are expected to override all the public methods > - provided in this class. See each method's docstring to see > - expectations about when the call should be chained. > - > - """ > - @classmethod > - def arg_parser(cls): > - """@return arg parser used to parse formatter-specific options.""" > - parser = argparse.ArgumentParser( > - description='{} options'.format(cls.__name__), > - usage=('dotest.py --results-formatter-options=' > - '"--option1 value1 [--option2 value2 [...]]"')) > - return parser > - > - def __init__(self, out_file, options): > - super(ResultsFormatter, self).__init__() > - self.out_file = out_file > - self.options = options > - self.using_terminal = False > - if not self.out_file: > - raise Exception("ResultsFormatter created with no file > object") > - self.start_time_by_test = {} > - self.terminate_called = False > - > - # Store counts of test_result events by status. > - self.result_status_counts = { > - EventBuilder.STATUS_SUCCESS: 0, > - EventBuilder.STATUS_EXPECTED_FAILURE: 0, > - EventBuilder.STATUS_SKIP: 0, > - EventBuilder.STATUS_UNEXPECTED_SUCCESS: 0, > - EventBuilder.STATUS_FAILURE: 0, > - EventBuilder.STATUS_ERROR: 0 > - } > - > - # Lock that we use while mutating inner state, like the > - # total test count and the elements. We minimize how > - # long we hold the lock just to keep inner state safe, not > - # entirely consistent from the outside. > - self.lock = threading.Lock() > - > - def handle_event(self, test_event): > - """Handles the test event for collection into the formatter > output. > - > - Derived classes may override this but should call down to this > - implementation first. > - > - @param test_event the test event as formatted by one of the > - event_for_* calls. > - """ > - # Keep track of whether terminate was received. We do this so > - # that a process can call the 'terminate' event on its own, to > - # close down a formatter at the appropriate time. Then the > - # atexit() cleanup can call the "terminate if it hasn't been > - # called yet". > - if test_event is not None: > - event_type = test_event.get("event", "") > - if event_type == "terminate": > - self.terminate_called = True > - elif event_type == "test_result": > - # Keep track of event counts per test result status type > - status = test_event["status"] > - self.result_status_counts[status] += 1 > - > - def track_start_time(self, test_class, test_name, start_time): > - """tracks the start time of a test so elapsed time can be > computed. > - > - this alleviates the need for test results to be processed serially > - by test. it will save the start time for the test so that > - elapsed_time_for_test() can compute the elapsed time properly. > - """ > - if test_class is None or test_name is None: > - return > - > - test_key = "{}.{}".format(test_class, test_name) > - with self.lock: > - self.start_time_by_test[test_key] = start_time > - > - def elapsed_time_for_test(self, test_class, test_name, end_time): > - """returns the elapsed time for a test. > - > - this function can only be called once per test and requires that > - the track_start_time() method be called sometime prior to calling > - this method. > - """ > - if test_class is None or test_name is None: > - return -2.0 > - > - test_key = "{}.{}".format(test_class, test_name) > - with self.lock: > - if test_key not in self.start_time_by_test: > - return -1.0 > - else: > - start_time = self.start_time_by_test[test_key] > - del self.start_time_by_test[test_key] > - return end_time - start_time > - > - def is_using_terminal(self): > - """returns true if this results formatter is using the terminal > and > - output should be avoided.""" > - return self.using_terminal > - > - def send_terminate_as_needed(self): > - """sends the terminate event if it hasn't been received yet.""" > - if not self.terminate_called: > - terminate_event = EventBuilder.bare_event("terminate") > - self.handle_event(terminate_event) > - > - # Derived classes may require self access > - # pylint: disable=no-self-use > - def replaces_summary(self): > - """Returns whether the results formatter includes a summary > - suitable to replace the old lldb test run results. > - > - @return True if the lldb test runner can skip its summary > - generation when using this results formatter; False otherwise. > - """ > - return False > - > - def counts_by_test_result_status(self, status): > - """Returns number of test method results for the given status. > - > - @status_result a test result status (e.g. success, fail, skip) > - as defined by the EventBuilder.STATUS_* class members. > - > - @return an integer returning the number of test methods matching > - the given test result status. > - """ > - return self.result_status_counts[status] > - > - > -class XunitFormatter(ResultsFormatter): > - """Provides xUnit-style formatted output. > - """ > - > - # Result mapping arguments > - RM_IGNORE = 'ignore' > - RM_SUCCESS = 'success' > - RM_FAILURE = 'failure' > - RM_PASSTHRU = 'passthru' > - > - @staticmethod > - def _build_illegal_xml_regex(): > - """Contructs a regex to match all illegal xml characters. > - > - Expects to be used against a unicode string.""" > - # Construct the range pairs of invalid unicode chareacters. > - illegal_chars_u = [ > - (0x00, 0x08), (0x0B, 0x0C), (0x0E, 0x1F), (0x7F, 0x84), > - (0x86, 0x9F), (0xFDD0, 0xFDDF), (0xFFFE, 0xFFFF)] > - > - # For wide builds, we have more. > - if sys.maxunicode >= 0x10000: > - illegal_chars_u.extend( > - [(0x1FFFE, 0x1FFFF), (0x2FFFE, 0x2FFFF), (0x3FFFE, > 0x3FFFF), > - (0x4FFFE, 0x4FFFF), (0x5FFFE, 0x5FFFF), (0x6FFFE, > 0x6FFFF), > - (0x7FFFE, 0x7FFFF), (0x8FFFE, 0x8FFFF), (0x9FFFE, > 0x9FFFF), > - (0xAFFFE, 0xAFFFF), (0xBFFFE, 0xBFFFF), (0xCFFFE, > 0xCFFFF), > - (0xDFFFE, 0xDFFFF), (0xEFFFE, 0xEFFFF), (0xFFFFE, > 0xFFFFF), > - (0x10FFFE, 0x10FFFF)]) > - > - # Build up an array of range expressions. > - illegal_ranges = [ > - "%s-%s" % (six.unichr(low), six.unichr(high)) > - for (low, high) in illegal_chars_u] > - > - # Compile the regex > - return re.compile(six.u('[%s]') % six.u('').join(illegal_ranges)) > - > - @staticmethod > - def _quote_attribute(text): > - """Returns the given text in a manner safe for usage in an XML > attribute. > - > - @param text the text that should appear within an XML attribute. > - @return the attribute-escaped version of the input text. > - """ > - return xml.sax.saxutils.quoteattr(text) > - > - def _replace_invalid_xml(self, str_or_unicode): > - """Replaces invalid XML characters with a '?'. > - > - @param str_or_unicode a string to replace invalid XML > - characters within. Can be unicode or not. If not unicode, > - assumes it is a byte string in utf-8 encoding. > - > - @returns a utf-8-encoded byte string with invalid > - XML replaced with '?'. > - """ > - # Get the content into unicode > - if isinstance(str_or_unicode, str): > - unicode_content = str_or_unicode.decode('utf-8') > - else: > - unicode_content = str_or_unicode > - return self.invalid_xml_re.sub( > - six.u('?'), unicode_content).encode('utf-8') > - > - @classmethod > - def arg_parser(cls): > - """@return arg parser used to parse formatter-specific options.""" > - parser = super(XunitFormatter, cls).arg_parser() > - > - # These are valid choices for results mapping. > - results_mapping_choices = [ > - XunitFormatter.RM_IGNORE, > - XunitFormatter.RM_SUCCESS, > - XunitFormatter.RM_FAILURE, > - XunitFormatter.RM_PASSTHRU] > - parser.add_argument( > - "--assert-on-unknown-events", > - action="store_true", > - help=('cause unknown test events to generate ' > - 'a python assert. Default is to ignore.')) > - parser.add_argument( > - "--ignore-skip-name", > - "-n", > - metavar='PATTERN', > - action="append", > - dest='ignore_skip_name_patterns', > - help=('a python regex pattern, where ' > - 'any skipped test with a test method name where regex ' > - 'matches (via search) will be ignored for xUnit test ' > - 'result purposes. Can be specified multiple times.')) > - parser.add_argument( > - "--ignore-skip-reason", > - "-r", > - metavar='PATTERN', > - action="append", > - dest='ignore_skip_reason_patterns', > - help=('a python regex pattern, where ' > - 'any skipped test with a skip reason where the regex ' > - 'matches (via search) will be ignored for xUnit test ' > - 'result purposes. Can be specified multiple times.')) > - parser.add_argument( > - "--xpass", action="store", choices=results_mapping_choices, > - default=XunitFormatter.RM_FAILURE, > - help=('specify mapping from unexpected success to jUnit/xUnit > ' > - 'result type')) > - parser.add_argument( > - "--xfail", action="store", choices=results_mapping_choices, > - default=XunitFormatter.RM_IGNORE, > - help=('specify mapping from expected failure to jUnit/xUnit ' > - 'result type')) > - return parser > - > - @staticmethod > - def _build_regex_list_from_patterns(patterns): > - """Builds a list of compiled regexes from option value. > - > - @param option string containing a comma-separated list of regex > - patterns. Zero-length or None will produce an empty regex list. > - > - @return list of compiled regular expressions, empty if no > - patterns provided. > - """ > - regex_list = [] > - if patterns is not None: > - for pattern in patterns: > - regex_list.append(re.compile(pattern)) > - return regex_list > - > - def __init__(self, out_file, options): > - """Initializes the XunitFormatter instance. > - @param out_file file-like object where formatted output is > written. > - @param options_dict specifies a dictionary of options for the > - formatter. > - """ > - # Initialize the parent > - super(XunitFormatter, self).__init__(out_file, options) > - self.text_encoding = "UTF-8" > - self.invalid_xml_re = XunitFormatter._build_illegal_xml_regex() > - self.total_test_count = 0 > - self.ignore_skip_name_regexes = ( > - XunitFormatter._build_regex_list_from_patterns( > - options.ignore_skip_name_patterns)) > - self.ignore_skip_reason_regexes = ( > - XunitFormatter._build_regex_list_from_patterns( > - options.ignore_skip_reason_patterns)) > - > - self.elements = { > - "successes": [], > - "errors": [], > - "failures": [], > - "skips": [], > - "unexpected_successes": [], > - "expected_failures": [], > - "all": [] > - } > - > - self.status_handlers = { > - EventBuilder.STATUS_SUCCESS: self._handle_success, > - EventBuilder.STATUS_FAILURE: self._handle_failure, > - EventBuilder.STATUS_ERROR: self._handle_error, > - EventBuilder.STATUS_SKIP: self._handle_skip, > - EventBuilder.STATUS_EXPECTED_FAILURE: > - self._handle_expected_failure, > - EventBuilder.STATUS_UNEXPECTED_SUCCESS: > - self._handle_unexpected_success > - } > - > - def handle_event(self, test_event): > - super(XunitFormatter, self).handle_event(test_event) > - > - event_type = test_event["event"] > - if event_type is None: > - return > - > - if event_type == "terminate": > - self._finish_output() > - elif event_type == "test_start": > - self.track_start_time( > - test_event["test_class"], > - test_event["test_name"], > - test_event["event_time"]) > - elif event_type == "test_result": > - self._process_test_result(test_event) > - else: > - # This is an unknown event. > - if self.options.assert_on_unknown_events: > - raise Exception("unknown event type {} from {}\n".format( > - event_type, test_event)) > - > - def _handle_success(self, test_event): > - """Handles a test success. > - @param test_event the test event to handle. > - """ > - result = self._common_add_testcase_entry(test_event) > - with self.lock: > - self.elements["successes"].append(result) > - > - def _handle_failure(self, test_event): > - """Handles a test failure. > - @param test_event the test event to handle. > - """ > - message = self._replace_invalid_xml(test_event["issue_message"]) > - backtrace = self._replace_invalid_xml( > - "".join(test_event.get("issue_backtrace", []))) > - > - result = self._common_add_testcase_entry( > - test_event, > - inner_content=( > - '<failure type={} > message={}><![CDATA[{}]]></failure>'.format( > - > XunitFormatter._quote_attribute(test_event["issue_class"]), > - XunitFormatter._quote_attribute(message), > - backtrace) > - )) > - with self.lock: > - self.elements["failures"].append(result) > - > - def _handle_error(self, test_event): > - """Handles a test error. > - @param test_event the test event to handle. > - """ > - message = self._replace_invalid_xml(test_event["issue_message"]) > - backtrace = self._replace_invalid_xml( > - "".join(test_event.get("issue_backtrace", []))) > - > - result = self._common_add_testcase_entry( > - test_event, > - inner_content=( > - '<error type={} message={}><![CDATA[{}]]></error>'.format( > - > XunitFormatter._quote_attribute(test_event["issue_class"]), > - XunitFormatter._quote_attribute(message), > - backtrace) > - )) > - with self.lock: > - self.elements["errors"].append(result) > - > - @staticmethod > - def _ignore_based_on_regex_list(test_event, test_key, regex_list): > - """Returns whether to ignore a test event based on patterns. > - > - @param test_event the test event dictionary to check. > - @param test_key the key within the dictionary to check. > - @param regex_list a list of zero or more regexes. May contain > - zero or more compiled regexes. > - > - @return True if any o the regex list match based on the > - re.search() method; false otherwise. > - """ > - for regex in regex_list: > - match = regex.search(test_event.get(test_key, '')) > - if match: > - return True > - return False > - > - def _handle_skip(self, test_event): > - """Handles a skipped test. > - @param test_event the test event to handle. > - """ > - > - # Are we ignoring this test based on test name? > - if XunitFormatter._ignore_based_on_regex_list( > - test_event, 'test_name', self.ignore_skip_name_regexes): > - return > - > - # Are we ignoring this test based on skip reason? > - if XunitFormatter._ignore_based_on_regex_list( > - test_event, 'skip_reason', > self.ignore_skip_reason_regexes): > - return > - > - # We're not ignoring this test. Process the skip. > - reason = self._replace_invalid_xml(test_event.get("skip_reason", > "")) > - result = self._common_add_testcase_entry( > - test_event, > - inner_content='<skipped message={} />'.format( > - XunitFormatter._quote_attribute(reason))) > - with self.lock: > - self.elements["skips"].append(result) > - > - def _handle_expected_failure(self, test_event): > - """Handles a test that failed as expected. > - @param test_event the test event to handle. > - """ > - if self.options.xfail == XunitFormatter.RM_PASSTHRU: > - # This is not a natively-supported junit/xunit > - # testcase mode, so it might fail a validating > - # test results viewer. > - if "bugnumber" in test_event: > - bug_id_attribute = 'bug-id={} '.format( > - > XunitFormatter._quote_attribute(test_event["bugnumber"])) > - else: > - bug_id_attribute = '' > - > - result = self._common_add_testcase_entry( > - test_event, > - inner_content=( > - '<expected-failure {}type={} message={} />'.format( > - bug_id_attribute, > - XunitFormatter._quote_attribute( > - test_event["issue_class"]), > - XunitFormatter._quote_attribute( > - test_event["issue_message"])) > - )) > - with self.lock: > - self.elements["expected_failures"].append(result) > - elif self.options.xfail == XunitFormatter.RM_SUCCESS: > - result = self._common_add_testcase_entry(test_event) > - with self.lock: > - self.elements["successes"].append(result) > - elif self.options.xfail == XunitFormatter.RM_FAILURE: > - result = self._common_add_testcase_entry( > - test_event, > - inner_content='<failure type={} message={} />'.format( > - > XunitFormatter._quote_attribute(test_event["issue_class"]), > - XunitFormatter._quote_attribute( > - test_event["issue_message"]))) > - with self.lock: > - self.elements["failures"].append(result) > - elif self.options.xfail == XunitFormatter.RM_IGNORE: > - pass > - else: > - raise Exception( > - "unknown xfail option: {}".format(self.options.xfail)) > - > - def _handle_unexpected_success(self, test_event): > - """Handles a test that passed but was expected to fail. > - @param test_event the test event to handle. > - """ > - if self.options.xpass == XunitFormatter.RM_PASSTHRU: > - # This is not a natively-supported junit/xunit > - # testcase mode, so it might fail a validating > - # test results viewer. > - result = self._common_add_testcase_entry( > - test_event, > - inner_content=("<unexpected-success />")) > - with self.lock: > - self.elements["unexpected_successes"].append(result) > - elif self.options.xpass == XunitFormatter.RM_SUCCESS: > - # Treat the xpass as a success. > - result = self._common_add_testcase_entry(test_event) > - with self.lock: > - self.elements["successes"].append(result) > - elif self.options.xpass == XunitFormatter.RM_FAILURE: > - # Treat the xpass as a failure. > - if "bugnumber" in test_event: > - message = "unexpected success (bug_id:{})".format( > - test_event["bugnumber"]) > - else: > - message = "unexpected success (bug_id:none)" > - result = self._common_add_testcase_entry( > - test_event, > - inner_content='<failure type={} message={} />'.format( > - XunitFormatter._quote_attribute("unexpected_success"), > - XunitFormatter._quote_attribute(message))) > - with self.lock: > - self.elements["failures"].append(result) > - elif self.options.xpass == XunitFormatter.RM_IGNORE: > - # Ignore the xpass result as far as xUnit reporting goes. > - pass > - else: > - raise Exception("unknown xpass option: {}".format( > - self.options.xpass)) > - > - def _process_test_result(self, test_event): > - """Processes the test_event known to be a test result. > - > - This categorizes the event appropriately and stores the data > needed > - to generate the final xUnit report. This method skips events that > - cannot be represented in xUnit output. > - """ > - if "status" not in test_event: > - raise Exception("test event dictionary missing 'status' key") > - > - status = test_event["status"] > - if status not in self.status_handlers: > - raise Exception("test event status '{}' unsupported".format( > - status)) > - > - # Call the status handler for the test result. > - self.status_handlers[status](test_event) > - > - def _common_add_testcase_entry(self, test_event, inner_content=None): > - """Registers a testcase result, and returns the text created. > - > - The caller is expected to manage failure/skip/success counts > - in some kind of appropriate way. This call simply constructs > - the XML and appends the returned result to the self.all_results > - list. > - > - @param test_event the test event dictionary. > - > - @param inner_content if specified, gets included in the <testcase> > - inner section, at the point before stdout and stderr would be > - included. This is where a <failure/>, <skipped/>, <error/>, etc. > - could go. > - > - @return the text of the xml testcase element. > - """ > - > - # Get elapsed time. > - test_class = test_event["test_class"] > - test_name = test_event["test_name"] > - event_time = test_event["event_time"] > - time_taken = self.elapsed_time_for_test( > - test_class, test_name, event_time) > - > - # Plumb in stdout/stderr once we shift over to only test results. > - test_stdout = '' > - test_stderr = '' > - > - # Formulate the output xml. > - if not inner_content: > - inner_content = "" > - result = ( > - '<testcase classname="{}" name="{}" time="{:.3f}">' > - '{}{}{}</testcase>'.format( > - test_class, > - test_name, > - time_taken, > - inner_content, > - test_stdout, > - test_stderr)) > - > - # Save the result, update total test count. > - with self.lock: > - self.total_test_count += 1 > - self.elements["all"].append(result) > - > - return result > - > - def _finish_output_no_lock(self): > - """Flushes out the report of test executions to form valid xml > output. > - > - xUnit output is in XML. The reporting system cannot complete the > - formatting of the output without knowing when there is no more > input. > - This call addresses notifcation of the completed test run and > thus is > - when we can finish off the report output. > - """ > - > - # Figure out the counts line for the testsuite. If we have > - # been counting either unexpected successes or expected > - # failures, we'll output those in the counts, at the risk of > - # being invalidated by a validating test results viewer. > - # These aren't counted by default so they won't show up unless > - # the user specified a formatter option to include them. > - xfail_count = len(self.elements["expected_failures"]) > - xpass_count = len(self.elements["unexpected_successes"]) > - if xfail_count > 0 or xpass_count > 0: > - extra_testsuite_attributes = ( > - ' expected-failures="{}"' > - ' unexpected-successes="{}"'.format(xfail_count, > xpass_count)) > - else: > - extra_testsuite_attributes = "" > - > - # Output the header. > - self.out_file.write( > - '<?xml version="1.0" encoding="{}"?>\n' > - '<testsuites>' > - '<testsuite name="{}" tests="{}" errors="{}" failures="{}" ' > - 'skip="{}"{}>\n'.format( > - self.text_encoding, > - "LLDB test suite", > - self.total_test_count, > - len(self.elements["errors"]), > - len(self.elements["failures"]), > - len(self.elements["skips"]), > - extra_testsuite_attributes)) > - > - # Output each of the test result entries. > - for result in self.elements["all"]: > - self.out_file.write(result + '\n') > - > - # Close off the test suite. > - self.out_file.write('</testsuite></testsuites>\n') > - > - def _finish_output(self): > - """Finish writing output as all incoming events have arrived.""" > - with self.lock: > - self._finish_output_no_lock() > - > - > -class RawPickledFormatter(ResultsFormatter): > - """Formats events as a pickled stream. > - > - The parallel test runner has inferiors pickle their results and send > them > - over a socket back to the parallel test. The parallel test runner > then > - aggregates them into the final results formatter (e.g. xUnit). > - """ > - > - @classmethod > - def arg_parser(cls): > - """@return arg parser used to parse formatter-specific options.""" > - parser = super(RawPickledFormatter, cls).arg_parser() > - return parser > - > - def __init__(self, out_file, options): > - super(RawPickledFormatter, self).__init__(out_file, options) > - self.pid = os.getpid() > - > - def handle_event(self, test_event): > - super(RawPickledFormatter, self).handle_event(test_event) > - > - # Convert initialize/terminate events into job_begin/job_end > events. > - event_type = test_event["event"] > - if event_type is None: > - return > - > - if event_type == "initialize": > - test_event["event"] = "job_begin" > - elif event_type == "terminate": > - test_event["event"] = "job_end" > - > - # Tack on the pid. > - test_event["pid"] = self.pid > - > - # Send it as > {serialized_length_of_serialized_bytes}{serialized_bytes} > - import struct > - msg = cPickle.dumps(test_event) > - packet = struct.pack("!I%ds" % len(msg), len(msg), msg) > - self.out_file.send(packet) > - > - > -class DumpFormatter(ResultsFormatter): > - """Formats events to the file as their raw python dictionary > format.""" > - > - def handle_event(self, test_event): > - super(DumpFormatter, self).handle_event(test_event) > - self.out_file.write("\n" + pprint.pformat(test_event) + "\n") > > > _______________________________________________ > lldb-commits mailing list > lldb-commits@lists.llvm.org > http://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits > -- -Todd
_______________________________________________ lldb-commits mailing list lldb-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits