Log Message
[ews] Limit layout tests failures to display in the status-bubble tooltip and buildbot summary https://bugs.webkit.org/show_bug.cgi?id=214085
Reviewed by Alexey Proskuryakov. * BuildSlaveSupport/ews-build/steps.py: Limit NUM_FAILURES_TO_DISPLAY to 10. (AnalyzeLayoutTestsResults.report_failure): (AnalyzeLayoutTestsResults.report_pre_existing_failures): * BuildSlaveSupport/ews-build/steps_unittest.py: (TestAnalyzeLayoutTestsResults.test_clean_tree_has_some_failures): Updated as per new behavior. (TestAnalyzeLayoutTestsResults.test_failure_on_clean_tree): Updated to also test for build_summary. (TestAnalyzeLayoutTestsResults.test_flaky_and_inconsistent_failures_with_clean_tree_failures): Ditto. (TestAnalyzeLayoutTestsResults.test_mildly_flaky_patch_with_some_tree_redness_and_flakiness): Ditto. (TestAnalyzeLayoutTestsResults.test_flaky_failures_in_first_run): Added unit-test. (TestAnalyzeLayoutTestsResults.test_clean_tree_has_lot_of_failures_and_no_new_failure): Added unit-test. (TestAnalyzeLayoutTestsResults.test_patch_introduces_lot_of_failures): Added unit-test.
Modified Paths
Diff
Modified: trunk/Tools/BuildSlaveSupport/ews-build/steps.py (264126 => 264127)
--- trunk/Tools/BuildSlaveSupport/ews-build/steps.py 2020-07-08 19:29:29 UTC (rev 264126)
+++ trunk/Tools/BuildSlaveSupport/ews-build/steps.py 2020-07-08 19:34:18 UTC (rev 264127)
@@ -1920,13 +1920,16 @@
name = 'analyze-layout-tests-results'
description = ['analyze-layout-test-results']
descriptionDone = ['analyze-layout-tests-results']
+ NUM_FAILURES_TO_DISPLAY = 10
def report_failure(self, new_failures):
self.finished(FAILURE)
self.build.results = FAILURE
pluralSuffix = 's' if len(new_failures) > 1 else ''
- new_failures_string = ', '.join([failure_name for failure_name in new_failures])
+ new_failures_string = ', '.join(sorted(new_failures)[:self.NUM_FAILURES_TO_DISPLAY])
message = 'Found {} new test failure{}: {}'.format(len(new_failures), pluralSuffix, new_failures_string)
+ if len(new_failures) > self.NUM_FAILURES_TO_DISPLAY:
+ message += ' ...'
self.descriptionDone = message
if self.getProperty('buildername', '').lower() == 'commit-queue':
@@ -1943,13 +1946,17 @@
self.descriptionDone = 'Passed layout tests'
message = ''
if clean_tree_failures:
- clean_tree_failures_string = ', '.join([failure_name for failure_name in clean_tree_failures])
+ clean_tree_failures_string = ', '.join(sorted(clean_tree_failures)[:self.NUM_FAILURES_TO_DISPLAY])
pluralSuffix = 's' if len(clean_tree_failures) > 1 else ''
message = 'Found {} pre-existing test failure{}: {}'.format(len(clean_tree_failures), pluralSuffix, clean_tree_failures_string)
+ if len(clean_tree_failures) > self.NUM_FAILURES_TO_DISPLAY:
+ message += ' ...'
if flaky_failures:
- flaky_failures_string = ', '.join(flaky_failures)
+ flaky_failures_string = ', '.join(sorted(flaky_failures)[:self.NUM_FAILURES_TO_DISPLAY])
pluralSuffix = 's' if len(flaky_failures) > 1 else ''
message += ' Found flaky test{}: {}'.format(pluralSuffix, flaky_failures_string)
+ if len(flaky_failures) > self.NUM_FAILURES_TO_DISPLAY:
+ message += ' ...'
self.setProperty('build_summary', message)
return defer.succeed(None)
Modified: trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py (264126 => 264127)
--- trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py 2020-07-08 19:29:29 UTC (rev 264126)
+++ trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py 2020-07-08 19:34:18 UTC (rev 264127)
@@ -1886,7 +1886,9 @@
self.setProperty('second_run_failures', ["jquery/offset.html"])
self.setProperty('clean_tree_run_failures', ["jquery/offset.html"])
self.expectOutcome(result=SUCCESS, state_string='Passed layout tests')
- return self.runStep()
+ rc = self.runStep()
+ self.assertEqual(self.getProperty('build_summary'), 'Found 1 pre-existing test failure: jquery/offset.html')
+ return rc
def test_flaky_and_consistent_failures_without_clean_tree_failures(self):
self.configureStep()
@@ -1917,6 +1919,15 @@
self.expectOutcome(result=SUCCESS, state_string='Passed layout tests')
return self.runStep()
+ def test_flaky_failures_in_first_run(self):
+ self.configureStep()
+ self.setProperty('first_run_failures', ['test1', 'test2'])
+ self.setProperty('second_run_failures', [])
+ self.expectOutcome(result=SUCCESS, state_string='Passed layout tests')
+ rc = self.runStep()
+ self.assertEqual(self.getProperty('build_summary'), ' Found flaky tests: test1, test2')
+ return rc
+
def test_flaky_and_inconsistent_failures_with_clean_tree_failures(self):
self.configureStep()
self.setProperty('first_run_failures', ['test1', 'test2'])
@@ -1923,7 +1934,9 @@
self.setProperty('second_run_failures', ['test3'])
self.setProperty('clean_tree_run_failures', ['test1', 'test2', 'test3'])
self.expectOutcome(result=SUCCESS, state_string='Passed layout tests')
- return self.runStep()
+ rc = self.runStep()
+ self.assertEqual(self.getProperty('build_summary'), 'Found 3 pre-existing test failures: test1, test2, test3 Found flaky tests: test1, test2, test3')
+ return rc
def test_flaky_and_consistent_failures_with_clean_tree_failures(self):
self.configureStep()
@@ -1935,11 +1948,13 @@
def test_mildly_flaky_patch_with_some_tree_redness_and_flakiness(self):
self.configureStep()
- self.setProperty('first_run_failures', ['PreExistingFail1', 'PreExistingFail2', 'Fail1'])
- self.setProperty('second_run_failures', ['PreExistingFail1', 'PreExistingFail2'])
- self.setProperty('clean_tree_run_failures', ['PreExistingFail1', 'PreExistingFail2', 'Fail2'])
+ self.setProperty('first_run_failures', ['test1', 'test2', 'test3'])
+ self.setProperty('second_run_failures', ['test1', 'test2'])
+ self.setProperty('clean_tree_run_failures', ['test1', 'test2', 'test4'])
self.expectOutcome(result=SUCCESS, state_string='Passed layout tests')
- return self.runStep()
+ rc = self.runStep()
+ self.assertEqual(self.getProperty('build_summary'), 'Found 3 pre-existing test failures: test1, test2, test4 Found flaky test: test3')
+ return rc
def test_first_run_exceed_failure_limit(self):
self.configureStep()
@@ -1983,9 +1998,33 @@
self.setProperty('second_results_exceed_failure_limit', True)
self.setProperty('second_run_failures', ['test{}'.format(i) for i in range(0, 30)])
self.setProperty('clean_tree_run_failures', ['test{}'.format(i) for i in range(0, 10)])
- self.expectOutcome(result=FAILURE, state_string='Found 30 new test failures: test1, test0, test3, test2, test5, test4, test7, test6, test9, test8, test24, test25, test26, test27, test20, test21, test22, test23, test28, test29, test19, test18, test11, test10, test13, test12, test15, test14, test17, test16 (failure)')
+ self.expectOutcome(result=FAILURE, state_string='Found 30 new test failures: test0, test1, test10, test11, test12, test13, test14, test15, test16, test17 ... (failure)')
return self.runStep()
+ def test_clean_tree_has_lot_of_failures_and_no_new_failure(self):
+ self.configureStep()
+ self.setProperty('first_run_failures', ['test1'])
+ self.setProperty('second_run_failures', ['test1'])
+ self.setProperty('clean_tree_run_failures', ['test{}'.format(i) for i in range(0, 20)])
+ self.expectOutcome(result=SUCCESS, state_string='Passed layout tests')
+ rc = self.runStep()
+ self.assertEqual(self.getProperty('build_summary'), 'Found 20 pre-existing test failures: test0, test1, test10, test11, test12, test13, test14, test15, test16, test17 ...')
+ return rc
+
+ def test_patch_introduces_lot_of_failures(self):
+ self.configureStep()
+ self.setProperty('buildername', 'Commit-Queue')
+ self.setProperty('first_results_exceed_failure_limit', True)
+ self.setProperty('first_run_failures', ['test{}'.format(i) for i in range(0, 300)])
+ self.setProperty('second_results_exceed_failure_limit', True)
+ self.setProperty('second_run_failures', ['test{}'.format(i) for i in range(0, 300)])
+ failure_message = 'Found 300 new test failures: test0, test1, test10, test100, test101, test102, test103, test104, test105, test106 ...'
+ self.expectOutcome(result=FAILURE, state_string=failure_message + ' (failure)')
+ rc = self.runStep()
+ self.assertEqual(self.getProperty('bugzilla_comment_text'), failure_message)
+ self.assertEqual(self.getProperty('build_finish_summary'), failure_message)
+ return rc
+
class TestCheckOutSpecificRevision(BuildStepMixinAdditions, unittest.TestCase):
def setUp(self):
self.longMessage = True
Modified: trunk/Tools/ChangeLog (264126 => 264127)
--- trunk/Tools/ChangeLog 2020-07-08 19:29:29 UTC (rev 264126)
+++ trunk/Tools/ChangeLog 2020-07-08 19:34:18 UTC (rev 264127)
@@ -1,3 +1,22 @@
+2020-07-08 Aakash Jain <aakash_j...@apple.com>
+
+ [ews] Limit layout tests failures to display in the status-bubble tooltip and buildbot summary
+ https://bugs.webkit.org/show_bug.cgi?id=214085
+
+ Reviewed by Alexey Proskuryakov.
+
+ * BuildSlaveSupport/ews-build/steps.py: Limit NUM_FAILURES_TO_DISPLAY to 10.
+ (AnalyzeLayoutTestsResults.report_failure):
+ (AnalyzeLayoutTestsResults.report_pre_existing_failures):
+ * BuildSlaveSupport/ews-build/steps_unittest.py:
+ (TestAnalyzeLayoutTestsResults.test_clean_tree_has_some_failures): Updated as per new behavior.
+ (TestAnalyzeLayoutTestsResults.test_failure_on_clean_tree): Updated to also test for build_summary.
+ (TestAnalyzeLayoutTestsResults.test_flaky_and_inconsistent_failures_with_clean_tree_failures): Ditto.
+ (TestAnalyzeLayoutTestsResults.test_mildly_flaky_patch_with_some_tree_redness_and_flakiness): Ditto.
+ (TestAnalyzeLayoutTestsResults.test_flaky_failures_in_first_run): Added unit-test.
+ (TestAnalyzeLayoutTestsResults.test_clean_tree_has_lot_of_failures_and_no_new_failure): Added unit-test.
+ (TestAnalyzeLayoutTestsResults.test_patch_introduces_lot_of_failures): Added unit-test.
+
2020-07-08 Sihui Liu <sihui_...@appe.com>
Text manipulation should ignore white spaces between nodes
_______________________________________________ webkit-changes mailing list webkit-changes@lists.webkit.org https://lists.webkit.org/mailman/listinfo/webkit-changes
