Author: Tatyana Krasnukha Date: 2020-01-10T17:37:55+03:00 New Revision: 3eea082535e232b35e6b2dab45dd81728b2ae7f4
URL: https://github.com/llvm/llvm-project/commit/3eea082535e232b35e6b2dab45dd81728b2ae7f4 DIFF: https://github.com/llvm/llvm-project/commit/3eea082535e232b35e6b2dab45dd81728b2ae7f4.diff LOG: [lldb][tests] Make it possible to expect failure for a whole category There already are decorators and "--excluded" option to mark test-cases/files as expected to fail. However, when a new test file is added and it which relates to a feature that a target doesn't support, this requires either adding decorators to that file or modifying the file provided as "--excluded" option value. The purpose of this patch is to avoid any modifications in such cases. E.g. if a target doesn't support "watchpoints" and passes "--xfail-category watchpoint" to dotest, a testing job will not fail after a new watchpoint-related test file is added. Differential Revision: https://reviews.llvm.org/D71906 Added: Modified: lldb/packages/Python/lldbsuite/test/configuration.py lldb/packages/Python/lldbsuite/test/dotest.py lldb/packages/Python/lldbsuite/test/dotest_args.py lldb/packages/Python/lldbsuite/test/test_result.py Removed: ################################################################################ diff --git a/lldb/packages/Python/lldbsuite/test/configuration.py b/lldb/packages/Python/lldbsuite/test/configuration.py index 21d527a90478..0fc831d11730 100644 --- a/lldb/packages/Python/lldbsuite/test/configuration.py +++ b/lldb/packages/Python/lldbsuite/test/configuration.py @@ -30,6 +30,8 @@ use_categories = False # Categories we want to skip skip_categories = ["darwin-log"] +# Categories we expect to fail +xfail_categories = [] # use this to track per-category failures failures_per_category = {} diff --git a/lldb/packages/Python/lldbsuite/test/dotest.py b/lldb/packages/Python/lldbsuite/test/dotest.py index e3f204828271..46600559286a 100644 --- a/lldb/packages/Python/lldbsuite/test/dotest.py +++ b/lldb/packages/Python/lldbsuite/test/dotest.py @@ -330,6 +330,10 @@ def parseOptionsAndInitTestdirs(): configuration.skip_categories += test_categories.validate( args.skip_categories, False) + if args.xfail_categories: + configuration.xfail_categories += test_categories.validate( + args.xfail_categories, False) + if args.E: os.environ['CFLAGS_EXTRAS'] = args.E diff --git a/lldb/packages/Python/lldbsuite/test/dotest_args.py b/lldb/packages/Python/lldbsuite/test/dotest_args.py index 9bcebc58cd22..41c96e3979cd 100644 --- a/lldb/packages/Python/lldbsuite/test/dotest_args.py +++ b/lldb/packages/Python/lldbsuite/test/dotest_args.py @@ -83,6 +83,12 @@ def create_parser(): action='append', dest='skip_categories', help=textwrap.dedent('''Specify categories of test cases to skip. Takes precedence over -G. Can be specified more than once.''')) + group.add_argument( + '--xfail-category', + metavar='category', + action='append', + dest='xfail_categories', + help=textwrap.dedent('''Specify categories of test cases that are expected to fail. Can be specified more than once.''')) # Configuration options group = parser.add_argument_group('Configuration options') diff --git a/lldb/packages/Python/lldbsuite/test/test_result.py b/lldb/packages/Python/lldbsuite/test/test_result.py index 193a64c5168a..7e13e09d9bf0 100644 --- a/lldb/packages/Python/lldbsuite/test/test_result.py +++ b/lldb/packages/Python/lldbsuite/test/test_result.py @@ -164,6 +164,10 @@ def checkExclusion(self, exclusion_list, name): return True return False + def checkCategoryExclusion(self, exclusion_list, test): + return not set(exclusion_list).isdisjoint( + self.getCategoriesForTest(test)) + def startTest(self, test): if configuration.shouldSkipBecauseOfCategories( self.getCategoriesForTest(test)): @@ -182,8 +186,10 @@ def startTest(self, test): EventBuilder.event_for_start(test)) def addSuccess(self, test): - if self.checkExclusion( - configuration.xfail_tests, test.id()): + if (self.checkExclusion( + configuration.xfail_tests, test.id()) or + self.checkCategoryExclusion( + configuration.xfail_categories, test)): self.addUnexpectedSuccess(test, None) return @@ -245,8 +251,10 @@ def addCleanupError(self, test, err): test, err)) def addFailure(self, test, err): - if self.checkExclusion( - configuration.xfail_tests, test.id()): + if (self.checkExclusion( + configuration.xfail_tests, test.id()) or + self.checkCategoryExclusion( + configuration.xfail_categories, test)): self.addExpectedFailure(test, err, None) return _______________________________________________ lldb-commits mailing list lldb-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
