Modified: trunk/Tools/CISupport/ews-build/steps.py (273144 => 273145)
--- trunk/Tools/CISupport/ews-build/steps.py 2021-02-19 17:55:36 UTC (rev 273144)
+++ trunk/Tools/CISupport/ews-build/steps.py 2021-02-19 18:21:18 UTC (rev 273145)
@@ -1779,7 +1779,7 @@
first_run_failures = set(self.getProperty('jsc_stress_test_failures', []) + self.getProperty('jsc_binary_failures', []))
second_run_failures = set(self.getProperty('jsc_rerun_stress_test_failures', []) + self.getProperty('jsc_rerun_binary_failures', []))
flaky_failures = first_run_failures.union(second_run_failures) - first_run_failures.intersection(second_run_failures)
- flaky_failures_string = ', '.join(flaky_failures)
+ flaky_failures_string = ', '.join(sorted(flaky_failures))
if rc == SUCCESS or rc == WARNINGS:
pluralSuffix = 's' if len(flaky_failures) > 1 else ''
@@ -1828,13 +1828,13 @@
flaky_stress_failures = first_run_stress_failures.union(second_run_stress_failures) - first_run_stress_failures.intersection(second_run_stress_failures)
flaky_binary_failures = first_run_binary_failures.union(second_run_binary_failures) - first_run_binary_failures.intersection(second_run_binary_failures)
- flaky_failures = (list(flaky_binary_failures) + list(flaky_stress_failures))[:self.NUM_FAILURES_TO_DISPLAY]
+ flaky_failures = sorted(list(flaky_binary_failures) + list(flaky_stress_failures))[:self.NUM_FAILURES_TO_DISPLAY]
flaky_failures_string = ', '.join(flaky_failures)
new_stress_failures = stress_failures_with_patch - clean_tree_stress_failures
new_binary_failures = binary_failures_with_patch - clean_tree_binary_failures
- new_stress_failures_to_display = ', '.join(list(new_stress_failures)[:self.NUM_FAILURES_TO_DISPLAY])
- new_binary_failures_to_display = ', '.join(list(new_binary_failures)[:self.NUM_FAILURES_TO_DISPLAY])
+ new_stress_failures_to_display = ', '.join(sorted(list(new_stress_failures))[:self.NUM_FAILURES_TO_DISPLAY])
+ new_binary_failures_to_display = ', '.join(sorted(list(new_binary_failures))[:self.NUM_FAILURES_TO_DISPLAY])
self._addToLog('stderr', '\nFailures in first run: {}'.format((list(first_run_binary_failures) + list(first_run_stress_failures))[:self.NUM_FAILURES_TO_DISPLAY]))
self._addToLog('stderr', '\nFailures in second run: {}'.format((list(second_run_binary_failures) + list(second_run_stress_failures))[:self.NUM_FAILURES_TO_DISPLAY]))
@@ -2075,7 +2075,7 @@
if first_results:
self.setProperty('first_results_exceed_failure_limit', first_results.did_exceed_test_failure_limit)
- self.setProperty('first_run_failures', first_results.failing_tests)
+ self.setProperty('first_run_failures', sorted(first_results.failing_tests))
if first_results.failing_tests:
self._addToLog(self.test_failures_log_name, '\n'.join(first_results.failing_tests))
self._parseRunWebKitTestsOutput(logText)
@@ -2142,7 +2142,7 @@
second_results_failing_tests = set(self.getProperty('second_run_failures', []))
tests_that_consistently_failed = first_results_failing_tests.intersection(second_results_failing_tests)
flaky_failures = first_results_failing_tests.union(second_results_failing_tests) - first_results_failing_tests.intersection(second_results_failing_tests)
- flaky_failures = list(flaky_failures)[:self.NUM_FAILURES_TO_DISPLAY]
+ flaky_failures = sorted(list(flaky_failures))[:self.NUM_FAILURES_TO_DISPLAY]
flaky_failures_string = ', '.join(flaky_failures)
if rc == SUCCESS or rc == WARNINGS:
@@ -2177,7 +2177,7 @@
if second_results:
self.setProperty('second_results_exceed_failure_limit', second_results.did_exceed_test_failure_limit)
- self.setProperty('second_run_failures', second_results.failing_tests)
+ self.setProperty('second_run_failures', sorted(second_results.failing_tests))
if second_results.failing_tests:
self._addToLog(self.test_failures_log_name, '\n'.join(second_results.failing_tests))
self._parseRunWebKitTestsOutput(logText)
Modified: trunk/Tools/CISupport/ews-build/steps_unittest.py (273144 => 273145)
--- trunk/Tools/CISupport/ews-build/steps_unittest.py 2021-02-19 17:55:36 UTC (rev 273144)
+++ trunk/Tools/CISupport/ews-build/steps_unittest.py 2021-02-19 18:21:18 UTC (rev 273145)
@@ -1506,7 +1506,7 @@
self.configureStep()
self.setProperty('jsc_stress_test_failures', ['test{}'.format(i) for i in range(0, 30)])
self.setProperty('jsc_rerun_stress_test_failures', ['test{}'.format(i) for i in range(0, 30)])
- self.expectOutcome(result=FAILURE, state_string='Found 30 new JSC stress test failures: test1, test0, test3, test2, test5, test4, test7, test6, test9, test8 ... (failure)')
+ self.expectOutcome(result=FAILURE, state_string='Found 30 new JSC stress test failures: test0, test1, test10, test11, test12, test13, test14, test15, test16, test17 ... (failure)')
return self.runStep()
def test_multiple_new_binary_failure(self):
@@ -1513,7 +1513,7 @@
self.configureStep()
self.setProperty('jsc_binary_failures', ['testmasm', 'testair', 'testb3', 'testdfg', 'testapi'])
self.setProperty('jsc_rerun_binary_failures', ['testmasm', 'testair', 'testb3', 'testdfg', 'testapi'])
- self.expectOutcome(result=FAILURE, state_string='Found 5 new JSC binary failures: testb3, testmasm, testapi, testdfg, testair (failure)')
+ self.expectOutcome(result=FAILURE, state_string='Found 5 new JSC binary failures: testair, testapi, testb3, testdfg, testmasm (failure)')
return self.runStep()
def test_new_stress_and_binary_failure(self):
@@ -1673,16 +1673,16 @@
rc = self.runStep()
self.assertEqual(self.getProperty(self.property_exceed_failure_limit), True)
self.assertEqual(self.getProperty(self.property_failures),
- ["imported/w3c/web-platform-tests/IndexedDB/interleaved-cursors-large.html",
- "imported/w3c/web-platform-tests/wasm/jsapi/interface.any.html",
- "imported/w3c/web-platform-tests/wasm/jsapi/instance/constructor-bad-imports.any.html",
- "imported/w3c/web-platform-tests/wasm/jsapi/global/constructor.any.html",
- "imported/w3c/web-platform-tests/wasm/jsapi/global/constructor.any.worker.html",
- "imported/w3c/web-platform-tests/wasm/jsapi/global/toString.any.html",
- "imported/w3c/web-platform-tests/wasm/jsapi/interface.any.worker.html",
- "imported/w3c/web-platform-tests/wasm/jsapi/constructor/instantiate-bad-imports.any.html",
- "imported/w3c/web-platform-tests/wasm/jsapi/constructor/instantiate-bad-imports.any.worker.html",
- "imported/blink/storage/indexeddb/blob-valid-before-commit.html"])
+ ['imported/blink/storage/indexeddb/blob-valid-before-commit.html',
+ 'imported/w3c/web-platform-tests/IndexedDB/interleaved-cursors-large.html',
+ 'imported/w3c/web-platform-tests/wasm/jsapi/constructor/instantiate-bad-imports.any.html',
+ 'imported/w3c/web-platform-tests/wasm/jsapi/constructor/instantiate-bad-imports.any.worker.html',
+ 'imported/w3c/web-platform-tests/wasm/jsapi/global/constructor.any.html',
+ 'imported/w3c/web-platform-tests/wasm/jsapi/global/constructor.any.worker.html',
+ 'imported/w3c/web-platform-tests/wasm/jsapi/global/toString.any.html',
+ 'imported/w3c/web-platform-tests/wasm/jsapi/instance/constructor-bad-imports.any.html',
+ 'imported/w3c/web-platform-tests/wasm/jsapi/interface.any.html',
+ 'imported/w3c/web-platform-tests/wasm/jsapi/interface.any.worker.html'])
return rc
def test_parse_results_json_flakes(self):