Modified: trunk/Tools/CISupport/ews-build/steps.py (274474 => 274475)
--- trunk/Tools/CISupport/ews-build/steps.py 2021-03-16 13:52:20 UTC (rev 274474)
+++ trunk/Tools/CISupport/ews-build/steps.py 2021-03-16 14:03:50 UTC (rev 274475)
@@ -2028,6 +2028,12 @@
shell.Test.__init__(self, logEnviron=False, **kwargs)
self.incorrectLayoutLines = []
+ def _get_patch(self):
+ sourcestamp = self.build.getSourceStamp(self.getProperty('codebase', ''))
+ if not sourcestamp or not sourcestamp.patch:
+ return None
+ return sourcestamp.patch[1]
+
def doStepIf(self, step):
return not ((self.getProperty('buildername', '').lower() == 'commit-queue') and
(self.getProperty('revert') or self.getProperty('passed_mac_wk2')))
@@ -2056,6 +2062,32 @@
if additionalArguments:
self.setCommand(self.command + additionalArguments)
+
+ if self.name == 'run-layout-tests-without-patch':
+ # In order to speed up testing, on the step that retries running the layout tests without patch
+ # only run the subset of tests that failed on the previous steps.
+ # But only do that if the previous steps didn't exceed the test failure limit and the patch doesn't
+ # modify the TestExpectations files (there are corner cases where we can't guarantee the correctnes
+ # of this optimization if the patch modifies the TestExpectations files, for example, if the patch
+ # removes skipped tests but those tests still fail).
+ first_results_did_exceed_test_failure_limit = self.getProperty('first_results_exceed_failure_limit', False)
+ second_results_did_exceed_test_failure_limit = self.getProperty('second_results_exceed_failure_limit', False)
+ if not first_results_did_exceed_test_failure_limit and not second_results_did_exceed_test_failure_limit:
+ patch_modifies_expectation_files = False
+ patch = self._get_patch()
+ if patch:
+ for line in patch.splitlines():
+ line = line.strip()
+ # patch is stored by buildbot as bytes: https://github.com/buildbot/buildbot/issues/5812#issuecomment-790175979
+ if (b'LayoutTests/' in line and b'TestExpectations' in line) and (line.startswith(b'---') or line.startswith(b'+++')):
+ patch_modifies_expectation_files = True
+ break
+ if not patch_modifies_expectation_files:
+ first_results_failing_tests = set(self.getProperty('first_run_failures', set()))
+ second_results_failing_tests = set(self.getProperty('second_run_failures', set()))
+ list_retry_tests = sorted(first_results_failing_tests.union(second_results_failing_tests))
+ self.setCommand(self.command + list_retry_tests)
+
return shell.Test.start(self)
# FIXME: This will break if run-webkit-tests changes its default log formatter.
Modified: trunk/Tools/CISupport/ews-build/steps_unittest.py (274474 => 274475)
--- trunk/Tools/CISupport/ews-build/steps_unittest.py 2021-03-16 13:52:20 UTC (rev 274474)
+++ trunk/Tools/CISupport/ews-build/steps_unittest.py 2021-03-16 14:03:50 UTC (rev 274475)
@@ -1927,6 +1927,119 @@
self.expectOutcome(result=SUCCESS, state_string='layout-tests')
return self.runStep()
+ def test_success_retry_only_subset(self):
+ self.configureStep()
+ self.setProperty('fullPlatform', 'ios-simulator')
+ self.setProperty('configuration', 'release')
+ self.setProperty('first_run_failures', ['test1', 'test2', 'test3'])
+ self.setProperty('second_run_failures', ['test1', 'test3', 'test4'])
+ self.expectRemoteCommands(
+ ExpectShell(workdir='wkdir',
+ logfiles={'json': self.jsonFileName},
+ logEnviron=False,
+ command=['python',
+ 'Tools/Scripts/run-webkit-tests',
+ '--no-build',
+ '--no-show-results',
+ '--no-new-test-results',
+ '--clobber-old-results',
+ '--release',
+ '--results-directory', 'layout-test-results',
+ '--debug-rwt-logging',
+ '--exit-after-n-failures', '30',
+ '--skip-failing-tests',
+ 'test1', 'test2', 'test3', 'test4'],
+ )
+ + 0,
+ )
+ self.expectOutcome(result=SUCCESS, state_string='layout-tests')
+ return self.runStep()
+
+ def test_success_retry_only_subset_limit_exceeded(self):
+ self.configureStep()
+ self.setProperty('fullPlatform', 'ios-simulator')
+ self.setProperty('configuration', 'release')
+ self.setProperty('first_run_failures', ['test1', 'test2', 'test3'])
+ self.setProperty('second_results_exceed_failure_limit', True)
+ self.setProperty('second_run_failures', ['test{}'.format(i) for i in range(0, 30)])
+ self.expectRemoteCommands(
+ ExpectShell(workdir='wkdir',
+ logfiles={'json': self.jsonFileName},
+ logEnviron=False,
+ command=['python',
+ 'Tools/Scripts/run-webkit-tests',
+ '--no-build',
+ '--no-show-results',
+ '--no-new-test-results',
+ '--clobber-old-results',
+ '--release',
+ '--results-directory', 'layout-test-results',
+ '--debug-rwt-logging',
+ '--exit-after-n-failures', '30',
+ '--skip-failing-tests'],
+ )
+ + 0,
+ )
+ self.expectOutcome(result=SUCCESS, state_string='layout-tests')
+ return self.runStep()
+
+ def test_success_retry_only_subset_patch_no_modifies_expectations(self):
+ self.configureStep()
+ self.setProperty('fullPlatform', 'ios-simulator')
+ self.setProperty('configuration', 'release')
+ self.setProperty('first_run_failures', ['test1', 'test2', 'test3'])
+ self.setProperty('second_run_failures', ['test1', 'test3', 'test4'])
+ RunWebKitTests._get_patch = lambda x: b'+++ Tools/ChangeLog\n+++ Tools/WebKitTestRunner/Options.cpp\n'
+ self.expectRemoteCommands(
+ ExpectShell(workdir='wkdir',
+ logfiles={'json': self.jsonFileName},
+ logEnviron=False,
+ command=['python',
+ 'Tools/Scripts/run-webkit-tests',
+ '--no-build',
+ '--no-show-results',
+ '--no-new-test-results',
+ '--clobber-old-results',
+ '--release',
+ '--results-directory', 'layout-test-results',
+ '--debug-rwt-logging',
+ '--exit-after-n-failures', '30',
+ '--skip-failing-tests',
+ 'test1', 'test2', 'test3', 'test4'],
+ )
+ + 0,
+ )
+ self.expectOutcome(result=SUCCESS, state_string='layout-tests')
+ return self.runStep()
+
+ def test_success_retry_only_subset_patch_modifies_expectations(self):
+ self.configureStep()
+ self.setProperty('fullPlatform', 'ios-simulator')
+ self.setProperty('configuration', 'release')
+ self.setProperty('first_run_failures', ['test1', 'test2', 'test3'])
+ self.setProperty('second_run_failures', ['test1', 'test3', 'test4'])
+ RunWebKitTests._get_patch = lambda x: b'+++ LayoutTests/Changelog\n+++ LayoutTests/platform/gtk/TestExpectations\n'
+ self.expectRemoteCommands(
+ ExpectShell(workdir='wkdir',
+ logfiles={'json': self.jsonFileName},
+ logEnviron=False,
+ command=['python',
+ 'Tools/Scripts/run-webkit-tests',
+ '--no-build',
+ '--no-show-results',
+ '--no-new-test-results',
+ '--clobber-old-results',
+ '--release',
+ '--results-directory', 'layout-test-results',
+ '--debug-rwt-logging',
+ '--exit-after-n-failures', '30',
+ '--skip-failing-tests'],
+ )
+ + 0,
+ )
+ self.expectOutcome(result=SUCCESS, state_string='layout-tests')
+ return self.runStep()
+
def test_failure(self):
self.configureStep()
self.setProperty('fullPlatform', 'ios-simulator')
