[webkit-changes] [273145] trunk/Tools

Fri, 19 Feb 2021 10:22:00 -0800

Title: [273145] trunk/Tools
Revision
273145
Author
aakash_j...@apple.com
Date
2021-02-19 10:21:18 -0800 (Fri, 19 Feb 2021)

Log Message

[ews] Ensure that the test failures are listed in sorted order
https://bugs.webkit.org/show_bug.cgi?id=222176

Reviewed by Jonathan Bedard.

* CISupport/ews-build/steps.py:
(ReRunJavaScriptCoreTests.evaluateCommand):
(AnalyzeJSCTestsResults.start):
(RunWebKitTests.commandComplete):
(ReRunWebKitTests.evaluateCommand):
(ReRunWebKitTests.commandComplete):
* CISupport/ews-build/steps_unittest.py:
(TestAnalyzeJSCTestsResults.test_multiple_new_stress_failure):
(TestAnalyzeJSCTestsResults.test_multiple_new_binary_failure):

Modified Paths

Diff

Modified: trunk/Tools/CISupport/ews-build/steps.py (273144 => 273145)


--- trunk/Tools/CISupport/ews-build/steps.py	2021-02-19 17:55:36 UTC (rev 273144)
+++ trunk/Tools/CISupport/ews-build/steps.py	2021-02-19 18:21:18 UTC (rev 273145)
@@ -1779,7 +1779,7 @@
         first_run_failures = set(self.getProperty('jsc_stress_test_failures', []) + self.getProperty('jsc_binary_failures', []))
         second_run_failures = set(self.getProperty('jsc_rerun_stress_test_failures', []) + self.getProperty('jsc_rerun_binary_failures', []))
         flaky_failures = first_run_failures.union(second_run_failures) - first_run_failures.intersection(second_run_failures)
-        flaky_failures_string = ', '.join(flaky_failures)
+        flaky_failures_string = ', '.join(sorted(flaky_failures))
 
         if rc == SUCCESS or rc == WARNINGS:
             pluralSuffix = 's' if len(flaky_failures) > 1 else ''
@@ -1828,13 +1828,13 @@
 
         flaky_stress_failures = first_run_stress_failures.union(second_run_stress_failures) - first_run_stress_failures.intersection(second_run_stress_failures)
         flaky_binary_failures = first_run_binary_failures.union(second_run_binary_failures) - first_run_binary_failures.intersection(second_run_binary_failures)
-        flaky_failures = (list(flaky_binary_failures) + list(flaky_stress_failures))[:self.NUM_FAILURES_TO_DISPLAY]
+        flaky_failures = sorted(list(flaky_binary_failures) + list(flaky_stress_failures))[:self.NUM_FAILURES_TO_DISPLAY]
         flaky_failures_string = ', '.join(flaky_failures)
 
         new_stress_failures = stress_failures_with_patch - clean_tree_stress_failures
         new_binary_failures = binary_failures_with_patch - clean_tree_binary_failures
-        new_stress_failures_to_display = ', '.join(list(new_stress_failures)[:self.NUM_FAILURES_TO_DISPLAY])
-        new_binary_failures_to_display = ', '.join(list(new_binary_failures)[:self.NUM_FAILURES_TO_DISPLAY])
+        new_stress_failures_to_display = ', '.join(sorted(list(new_stress_failures))[:self.NUM_FAILURES_TO_DISPLAY])
+        new_binary_failures_to_display = ', '.join(sorted(list(new_binary_failures))[:self.NUM_FAILURES_TO_DISPLAY])
 
         self._addToLog('stderr', '\nFailures in first run: {}'.format((list(first_run_binary_failures) + list(first_run_stress_failures))[:self.NUM_FAILURES_TO_DISPLAY]))
         self._addToLog('stderr', '\nFailures in second run: {}'.format((list(second_run_binary_failures) + list(second_run_stress_failures))[:self.NUM_FAILURES_TO_DISPLAY]))
@@ -2075,7 +2075,7 @@
 
         if first_results:
             self.setProperty('first_results_exceed_failure_limit', first_results.did_exceed_test_failure_limit)
-            self.setProperty('first_run_failures', first_results.failing_tests)
+            self.setProperty('first_run_failures', sorted(first_results.failing_tests))
             if first_results.failing_tests:
                 self._addToLog(self.test_failures_log_name, '\n'.join(first_results.failing_tests))
         self._parseRunWebKitTestsOutput(logText)
@@ -2142,7 +2142,7 @@
         second_results_failing_tests = set(self.getProperty('second_run_failures', []))
         tests_that_consistently_failed = first_results_failing_tests.intersection(second_results_failing_tests)
         flaky_failures = first_results_failing_tests.union(second_results_failing_tests) - first_results_failing_tests.intersection(second_results_failing_tests)
-        flaky_failures = list(flaky_failures)[:self.NUM_FAILURES_TO_DISPLAY]
+        flaky_failures = sorted(list(flaky_failures))[:self.NUM_FAILURES_TO_DISPLAY]
         flaky_failures_string = ', '.join(flaky_failures)
 
         if rc == SUCCESS or rc == WARNINGS:
@@ -2177,7 +2177,7 @@
 
         if second_results:
             self.setProperty('second_results_exceed_failure_limit', second_results.did_exceed_test_failure_limit)
-            self.setProperty('second_run_failures', second_results.failing_tests)
+            self.setProperty('second_run_failures', sorted(second_results.failing_tests))
             if second_results.failing_tests:
                 self._addToLog(self.test_failures_log_name, '\n'.join(second_results.failing_tests))
         self._parseRunWebKitTestsOutput(logText)

Modified: trunk/Tools/CISupport/ews-build/steps_unittest.py (273144 => 273145)


--- trunk/Tools/CISupport/ews-build/steps_unittest.py	2021-02-19 17:55:36 UTC (rev 273144)
+++ trunk/Tools/CISupport/ews-build/steps_unittest.py	2021-02-19 18:21:18 UTC (rev 273145)
@@ -1506,7 +1506,7 @@
         self.configureStep()
         self.setProperty('jsc_stress_test_failures', ['test{}'.format(i) for i in range(0, 30)])
         self.setProperty('jsc_rerun_stress_test_failures', ['test{}'.format(i) for i in range(0, 30)])
-        self.expectOutcome(result=FAILURE, state_string='Found 30 new JSC stress test failures: test1, test0, test3, test2, test5, test4, test7, test6, test9, test8 ... (failure)')
+        self.expectOutcome(result=FAILURE, state_string='Found 30 new JSC stress test failures: test0, test1, test10, test11, test12, test13, test14, test15, test16, test17 ... (failure)')
         return self.runStep()
 
     def test_multiple_new_binary_failure(self):
@@ -1513,7 +1513,7 @@
         self.configureStep()
         self.setProperty('jsc_binary_failures', ['testmasm', 'testair', 'testb3', 'testdfg', 'testapi'])
         self.setProperty('jsc_rerun_binary_failures', ['testmasm', 'testair', 'testb3', 'testdfg', 'testapi'])
-        self.expectOutcome(result=FAILURE, state_string='Found 5 new JSC binary failures: testb3, testmasm, testapi, testdfg, testair (failure)')
+        self.expectOutcome(result=FAILURE, state_string='Found 5 new JSC binary failures: testair, testapi, testb3, testdfg, testmasm (failure)')
         return self.runStep()
 
     def test_new_stress_and_binary_failure(self):
@@ -1673,16 +1673,16 @@
         rc = self.runStep()
         self.assertEqual(self.getProperty(self.property_exceed_failure_limit), True)
         self.assertEqual(self.getProperty(self.property_failures),
-                            ["imported/w3c/web-platform-tests/IndexedDB/interleaved-cursors-large.html",
-                             "imported/w3c/web-platform-tests/wasm/jsapi/interface.any.html",
-                             "imported/w3c/web-platform-tests/wasm/jsapi/instance/constructor-bad-imports.any.html",
-                             "imported/w3c/web-platform-tests/wasm/jsapi/global/constructor.any.html",
-                             "imported/w3c/web-platform-tests/wasm/jsapi/global/constructor.any.worker.html",
-                             "imported/w3c/web-platform-tests/wasm/jsapi/global/toString.any.html",
-                             "imported/w3c/web-platform-tests/wasm/jsapi/interface.any.worker.html",
-                             "imported/w3c/web-platform-tests/wasm/jsapi/constructor/instantiate-bad-imports.any.html",
-                             "imported/w3c/web-platform-tests/wasm/jsapi/constructor/instantiate-bad-imports.any.worker.html",
-                             "imported/blink/storage/indexeddb/blob-valid-before-commit.html"])
+                            ['imported/blink/storage/indexeddb/blob-valid-before-commit.html',
+                             'imported/w3c/web-platform-tests/IndexedDB/interleaved-cursors-large.html',
+                             'imported/w3c/web-platform-tests/wasm/jsapi/constructor/instantiate-bad-imports.any.html',
+                             'imported/w3c/web-platform-tests/wasm/jsapi/constructor/instantiate-bad-imports.any.worker.html',
+                             'imported/w3c/web-platform-tests/wasm/jsapi/global/constructor.any.html',
+                             'imported/w3c/web-platform-tests/wasm/jsapi/global/constructor.any.worker.html',
+                             'imported/w3c/web-platform-tests/wasm/jsapi/global/toString.any.html',
+                             'imported/w3c/web-platform-tests/wasm/jsapi/instance/constructor-bad-imports.any.html',
+                             'imported/w3c/web-platform-tests/wasm/jsapi/interface.any.html',
+                             'imported/w3c/web-platform-tests/wasm/jsapi/interface.any.worker.html'])
         return rc
 
     def test_parse_results_json_flakes(self):

Modified: trunk/Tools/ChangeLog (273144 => 273145)


--- trunk/Tools/ChangeLog	2021-02-19 17:55:36 UTC (rev 273144)
+++ trunk/Tools/ChangeLog	2021-02-19 18:21:18 UTC (rev 273145)
@@ -1,3 +1,20 @@
+2021-02-19  Aakash Jain  <aakash_j...@apple.com>
+
+        [ews] Ensure that the test failures are listed in sorted order
+        https://bugs.webkit.org/show_bug.cgi?id=222176
+
+        Reviewed by Jonathan Bedard.
+
+        * CISupport/ews-build/steps.py:
+        (ReRunJavaScriptCoreTests.evaluateCommand):
+        (AnalyzeJSCTestsResults.start):
+        (RunWebKitTests.commandComplete):
+        (ReRunWebKitTests.evaluateCommand):
+        (ReRunWebKitTests.commandComplete):
+        * CISupport/ews-build/steps_unittest.py:
+        (TestAnalyzeJSCTestsResults.test_multiple_new_stress_failure):
+        (TestAnalyzeJSCTestsResults.test_multiple_new_binary_failure):
+
 2021-02-19  Jonathan Bedard  <jbed...@apple.com>
 
         [webkitscmpy] Add clean command

_______________________________________________
webkit-changes mailing list
webkit-changes@lists.webkit.org
https://lists.webkit.org/mailman/listinfo/webkit-changes

Reply via email to