Diff
Modified: trunk/Tools/ChangeLog (175203 => 175204)
--- trunk/Tools/ChangeLog 2014-10-25 22:05:57 UTC (rev 175203)
+++ trunk/Tools/ChangeLog 2014-10-25 23:29:51 UTC (rev 175204)
@@ -1,3 +1,18 @@
+2014-10-25 Alexey Proskuryakov <a...@apple.com>
+
+ Test regressions are not detected when image result is missing
+ https://bugs.webkit.org/show_bug.cgi?id=138070
+
+ Reviewed by Simon Fraser.
+
+ * Scripts/webkitpy/layout_tests/models/test_run_results.py:
+ * Scripts/webkitpy/layout_tests/views/buildbot_results.py:
+ Count these as regressions, not as flaky tests.
+
+ * Scripts/webkitpy/port/test.py:
+ * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+ Test it.
+
2014-10-24 Timothy Horton <timothy_hor...@apple.com>
Add Conrad Shultz to the contributors list.
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_run_results.py (175203 => 175204)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_run_results.py 2014-10-25 22:05:57 UTC (rev 175203)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_run_results.py 2014-10-25 23:29:51 UTC (rev 175204)
@@ -192,7 +192,9 @@
elif retry_results:
retry_result_type = retry_results.unexpected_results_by_name[test_name].type
if result_type != retry_result_type:
- if enabled_pixel_tests_in_retry and result_type == test_expectations.TEXT and retry_result_type == test_expectations.IMAGE_PLUS_TEXT:
+ if enabled_pixel_tests_in_retry and result_type == test_expectations.TEXT and (retry_result_type == test_expectations.IMAGE_PLUS_TEXT or retry_result_type == test_expectations.MISSING):
+ if retry_result_type == test_expectations.MISSING:
+ num_missing += 1
num_regressions += 1
test_dict['report'] = 'REGRESSION'
else:
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (175203 => 175204)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2014-10-25 22:05:57 UTC (rev 175203)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2014-10-25 23:29:51 UTC (rev 175204)
@@ -671,6 +671,22 @@
self.assertFalse(json["pixel_tests_enabled"])
self.assertEqual(details.enabled_pixel_tests_in_retry, True)
+ def test_failed_text_with_missing_pixel_results_on_retry(self):
+ # Test what happens when pixel results are missing on retry.
+ host = MockHost()
+ details, err, _ = logging_run(['--no-show-results',
+ '--no-new-test-results', '--no-pixel-tests',
+ 'failures/unexpected/text-image-missing.html'],
+ tests_included=True, host=host)
+ file_list = host.filesystem.written_files.keys()
+ self.assertEqual(details.exit_code, 1)
+ expected_token = '"unexpected":{"text-image-missing.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT MISSING","is_missing_image":true}}'
+ json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
+ self.assertTrue(json_string.find(expected_token) != -1)
+ self.assertTrue(json_string.find('"num_regressions":1') != -1)
+ self.assertTrue(json_string.find('"num_flaky":0') != -1)
+ self.assertTrue(json_string.find('"num_missing":1') != -1)
+
def test_retrying_uses_retries_directory(self):
host = MockHost()
details, err, _ = logging_run(['--debug-rwt-logging', 'failures/unexpected/text-image-checksum.html'], tests_included=True, host=host)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/views/buildbot_results.py (175203 => 175204)
--- trunk/Tools/Scripts/webkitpy/layout_tests/views/buildbot_results.py 2014-10-25 22:05:57 UTC (rev 175203)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/views/buildbot_results.py 2014-10-25 23:29:51 UTC (rev 175204)
@@ -115,7 +115,7 @@
add_to_dict_of_lists(passes, 'Expected to timeout, but passed', test)
else:
add_to_dict_of_lists(passes, 'Expected to fail, but passed', test)
- elif enabled_pixel_tests_in_retry and actual == ['TEXT', 'IMAGE+TEXT']:
+ elif enabled_pixel_tests_in_retry and (actual == ['TEXT', 'IMAGE+TEXT'] or actual == ['TEXT', 'MISSING']):
add_to_dict_of_lists(regressions, actual[0], test)
elif len(actual) > 1:
# We group flaky tests by the first actual result we got.
Modified: trunk/Tools/Scripts/webkitpy/port/test.py (175203 => 175204)
--- trunk/Tools/Scripts/webkitpy/port/test.py 2014-10-25 22:05:57 UTC (rev 175203)
+++ trunk/Tools/Scripts/webkitpy/port/test.py 2014-10-25 23:29:51 UTC (rev 175204)
@@ -100,11 +100,11 @@
#
# These numbers may need to be updated whenever we add or delete tests.
#
-TOTAL_TESTS = 106
+TOTAL_TESTS = 107
TOTAL_SKIPS = 28
TOTAL_RETRIES = 14
-UNEXPECTED_PASSES = 6
+UNEXPECTED_PASSES = 7
UNEXPECTED_FAILURES = 17
def unit_test_list():
@@ -167,6 +167,9 @@
actual_text='text-image-checksum_fail-txt',
actual_image='text-image-checksum_fail-pngtEXtchecksum\x00checksum_fail',
actual_checksum='text-image-checksum_fail-checksum')
+ tests.add('failures/unexpected/text-image-missing.html',
+ actual_text='text-image-checksum_fail-txt',
+ expected_image=None)
tests.add('failures/unexpected/checksum-with-matching-image.html',
actual_checksum='text-image-checksum_fail-checksum')
tests.add('failures/unexpected/skip_pass.html')
