Diff
Modified: trunk/LayoutTests/ChangeLog (285212 => 285213)
--- trunk/LayoutTests/ChangeLog 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/LayoutTests/ChangeLog 2021-11-03 17:05:50 UTC (rev 285213)
@@ -1,3 +1,18 @@
+2021-11-03 Simon Fraser <simon.fra...@apple.com>
+
+ Expose fuzzy match data in layout test results
+ https://bugs.webkit.org/show_bug.cgi?id=232523
+
+ Reviewed by Jonathan Bedard.
+
+ Add some "image_difference" values to the test JSON and update the resulting
+ files.
+
+ * fast/harness/full_results.json:
+ * fast/harness/image-diff-template-expected.txt:
+ * fast/harness/image-diff-template.html:
+ * fast/harness/results.html:
+
2021-11-03 Jer Noble <jer.no...@apple.com>
[iOS] AVAssetResourceLoadingRequest.request does not include a Range: header on iOS 15.
Modified: trunk/LayoutTests/fast/harness/full_results.json (285212 => 285213)
--- trunk/LayoutTests/fast/harness/full_results.json 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/LayoutTests/fast/harness/full_results.json 2021-11-03 17:05:50 UTC (rev 285213)
@@ -25,13 +25,21 @@
"report": "REGRESSION",
"expected": "PASS",
"actual": "IMAGE",
- "image_diff_percent": 86.262301
+ "image_diff_percent": 86.262301,
+ "image_difference": {
+ "max_difference": 13,
+ "total_pixels": 167
+ }
},
"tiny-image-fail.html": {
"report": "REGRESSION",
"expected": "PASS",
"actual": "IMAGE",
- "image_diff_percent": 0.002
+ "image_diff_percent": 0.002,
+ "image_difference": {
+ "max_difference": 3,
+ "total_pixels": 24
+ }
},
"leaky-worker.html": {
"report": "REGRESSION",
Modified: trunk/LayoutTests/fast/harness/image-diff-template-expected.txt (285212 => 285213)
--- trunk/LayoutTests/fast/harness/image-diff-template-expected.txt 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/LayoutTests/fast/harness/image-diff-template-expected.txt 2021-11-03 17:05:50 UTC (rev 285213)
@@ -1,3 +1,5 @@
+Pixel difference: __PIXEL_DIFF__
+Fuzzy match: __FUZZY_DATA__
Expected Image Actual Image Diff Image
Animate: Expected Actual Diff
Loading...
Modified: trunk/LayoutTests/fast/harness/image-diff-template.html (285212 => 285213)
--- trunk/LayoutTests/fast/harness/image-diff-template.html 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/LayoutTests/fast/harness/image-diff-template.html 2021-11-03 17:05:50 UTC (rev 285213)
@@ -3,10 +3,17 @@
<head>
<title>__TITLE__</title>
<style>
+ body {
+ font-family: Helvetica, sans-serif;
+ font-size: 11pt;
+ }
+
.imageContainer {
position: absolute;
margin: 10px;
outline: 10px solid silver;
+ min-width: 300px;
+ min-height: 150px;
}
.imageContainer > img {
@@ -13,32 +20,38 @@
display: block;
}
-
.controls {
- margin-bottom: 20px;
+ margin-bottom: 15px;
+ padding: 10px;
}
+ .pixel-diff {
+ font-size: 10pt;
+ font-family: -apple-system;
+ }
+
+ .pixel-diff > td:first-child {
+ text-align: right;
+ }
+
+ .hidden {
+ display: none;
+ }
+
.controls button {
width: 12em;
+ padding: 4px;
}
.controls td {
- padding: 0 10px;
+ padding: 5px 10px;
}
.controls label {
- font-family: -webkit-system-font;
- font-size: 11px;
+ font-family: -apple-system;
+ font-size: 12px;
}
- .buttons {
- }
- .controls > .indicator {
- display: table-cell;
- -webkit-column-span: 1;
- border-top: 2px solid black;
- }
-
button.selected {
text-decoration: underline;
}
@@ -46,6 +59,14 @@
</head>
<body>
<table class="controls">
+ <tr class="pixel-diff __HIDE_DIFF_CLASS__">
+ <td>Pixel difference:</td>
+ <td>__PIXEL_DIFF__</td>
+ </tr>
+ <tr class="pixel-diff __HIDE_FUZZY_CLASS__">
+ <td>Fuzzy match:</td>
+ <td>__FUZZY_DATA__</td>
+ </tr>
<tr>
<td></td>
<td><button id="expected-label" data-type="expected" _onclick_="switchToImage(this)">Expected Image</button></td>
@@ -53,10 +74,10 @@
<td><button id="diff-label" data-type="diff" _onclick_="switchToImage(this)">Diff Image</button></td>
</tr>
<tr>
- <td><input type="checkbox" id="animate" _onchange_="toggleAnimate()" checked></input><label for=""
- <td><input type="checkbox" id="cycle-expected" _onchange_="updateImageCycle()" checked></input><label for=""
- <td><input type="checkbox" id="cycle-actual" _onchange_="updateImageCycle()" checked></input><label for=""
- <td><input type="checkbox" id="cycle-diff" _onchange_="updateImageCycle()"></input><label for=""
+ <td><input type="checkbox" id="animate" _onchange_="toggleAnimate()" checked><label for=""
+ <td><input type="checkbox" id="cycle-expected" _onchange_="updateImageCycle()" checked><label for=""
+ <td><input type="checkbox" id="cycle-actual" _onchange_="updateImageCycle()" checked><label for=""
+ <td><input type="checkbox" id="cycle-diff" _onchange_="updateImageCycle()"><label for=""
</tr>
</table>
Modified: trunk/LayoutTests/fast/harness/results.html (285212 => 285213)
--- trunk/LayoutTests/fast/harness/results.html 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/LayoutTests/fast/harness/results.html 2021-11-03 17:05:50 UTC (rev 285213)
@@ -1415,8 +1415,8 @@
result += TestResultsController.resultLink(testPrefix, '-actual.png', 'actual');
}
- let diff = Math.max(testResult.info.image_diff_percent, 0.01).toFixed(2);
- result += TestResultsController.resultLink(testPrefix, '-diff.png', 'diff (' + diff + '%)');
+ const diff = Math.max(testResult.info.image_diff_percent, 0.01).toFixed(2);
+ result += TestResultsController.resultLink(testPrefix, '-diff.png', `diff (${diff}%)`);
}
}
Modified: trunk/Tools/ChangeLog (285212 => 285213)
--- trunk/Tools/ChangeLog 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/Tools/ChangeLog 2021-11-03 17:05:50 UTC (rev 285213)
@@ -1,3 +1,40 @@
+2021-11-03 Simon Fraser <simon.fra...@apple.com>
+
+ Expose fuzzy match data in layout test results
+ https://bugs.webkit.org/show_bug.cgi?id=232523
+
+ Reviewed by Jonathan Bedard.
+
+ For a failing ref (or image) test, include the fuzzy matching data in full_results.json
+ in the form "image_difference": { "max_difference": 13, "total_pixels": 167 }, and
+ show that in the image comparison page that's written out for each test result.
+
+ Fix some tests that relied on JSON property serialization ordering (we don't serialize
+ with `sort_keys=True`).
+
+ * Scripts/webkitpy/layout_tests/controllers/manager.py:
+ (Manager._save_json_files):
+ * Scripts/webkitpy/layout_tests/controllers/test_result_writer.py:
+ (TestResultWriter.write_image_diff_files):
+ * Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py:
+ (JSONResultsGenerator.generate_times_ms_file):
+ * Scripts/webkitpy/layout_tests/models/test_failures.py:
+ (FailureImageHashMismatch.formatted_diff_percent):
+ (FailureImageHashMismatch):
+ (FailureImageHashMismatch.formatted_fuzzy_data):
+ (FailureImageHashMismatch.write_failure):
+ (FailureReftestMismatch.message):
+ (FailureReftestMismatch.formatted_diff_percent):
+ (FailureReftestMismatch):
+ (FailureReftestMismatch.formatted_fuzzy_data):
+ (FailureReftestMismatch.write_failure):
+ * Scripts/webkitpy/layout_tests/models/test_run_results.py:
+ (_interpret_test_failures):
+ * Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py:
+ (InterpretTestFailuresTest.test_interpret_test_failures):
+ * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+ (RunTest.test_retrying_force_pixel_tests):
+
2021-11-03 Kate Cheney <katherine_che...@apple.com>
[iOS] Need API for marking file requests as non-app-initiated
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py (285212 => 285213)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2021-11-03 17:05:50 UTC (rev 285213)
@@ -692,7 +692,7 @@
self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))
full_results_path = self._filesystem.join(self._results_directory, "full_results.json")
- # We write full_results.json out as jsonp because we need to load it from a file url and Chromium doesn't allow that.
+ # We write full_results.json out as jsonp because we need to load it from a file url and WebKit doesn't allow that.
json_results_generator.write_json(self._filesystem, summarized_results, full_results_path, callback="ADD_RESULTS")
generator = json_layout_results_generator.JSONLayoutResultsGenerator(
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py (285212 => 285213)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py 2021-11-03 17:05:50 UTC (rev 285213)
@@ -184,7 +184,7 @@
def write_image_files(self, actual_image, expected_image):
self.write_output_files('.png', actual_image, expected_image)
- def write_image_diff_files(self, image_diff):
+ def write_image_diff_files(self, image_diff, diff_percent_text=None, fuzzy_data_text=None):
diff_filename = self.output_filename(self.FILENAME_SUFFIX_IMAGE_DIFF)
self._write_binary_file(diff_filename, image_diff)
@@ -198,6 +198,18 @@
html = image_diff_file.replace('__TITLE__', self._test_name)
html = html.replace('__PREFIX__', self._output_testname(''))
+ if not diff_percent_text:
+ html = html.replace('__HIDE_DIFF_CLASS__', 'hidden')
+ else:
+ html = html.replace('__HIDE_DIFF_CLASS__', '')
+ html = html.replace('__PIXEL_DIFF__', diff_percent_text)
+
+ if not fuzzy_data_text:
+ html = html.replace('__HIDE_FUZZY_CLASS__', 'hidden')
+ else:
+ html = html.replace('__HIDE_FUZZY_CLASS__', '')
+ html = html.replace('__FUZZY_DATA__', fuzzy_data_text)
+
diffs_html_filename = self.output_filename(self.FILENAME_SUFFIX_IMAGE_DIFFS_HTML)
self._filesystem.write_text_file(diffs_html_filename, html)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py (285212 => 285213)
--- trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py 2021-11-03 17:05:50 UTC (rev 285213)
@@ -285,8 +285,6 @@
return True
def generate_times_ms_file(self):
- # FIXME: rename to generate_times_ms_file. This needs to be coordinated with
- # changing the calls to this on the chromium build workers.
times = test_timings_trie(self._port, self._test_results_map.values())
file_path = self._filesystem.join(self._results_directory, self.TIMES_MS_FILENAME)
write_json(self._filesystem, times, file_path)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py (285212 => 285213)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py 2021-11-03 17:05:50 UTC (rev 285213)
@@ -208,9 +208,18 @@
def message(self):
return "image diff"
+ def formatted_diff_percent(self):
+ return '{:.2f}%'.format(max(self.image_diff_result.diff_percent, 0.01))
+
+ def formatted_fuzzy_data(self):
+ fuzzy_data = self.image_diff_result.fuzzy_data if self.image_diff_result else None
+ if fuzzy_data:
+ return 'maxDifference={}; totalPixels={}'.format(fuzzy_data['max_difference'], fuzzy_data['total_pixels'])
+ return None
+
def write_failure(self, writer, driver_output, expected_driver_output, port):
writer.write_image_files(driver_output.image, expected_driver_output.image)
- writer.write_image_diff_files(driver_output.image_diff)
+ writer.write_image_diff_files(driver_output.image_diff, self.formatted_diff_percent(), self.formatted_fuzzy_data())
class FailureImageHashIncorrect(TestFailure):
@@ -225,8 +234,20 @@
self.image_diff_result = image_diff_result
def message(self):
+ fuzzy_data = self.image_diff_result.fuzzy_data if self.image_diff_result else None
+ if fuzzy_data:
+ return "reference mismatch maxDifference={}; totalPixels={}".format(fuzzy_data['max_difference'], fuzzy_data['total_pixels'])
return "reference mismatch"
+ def formatted_diff_percent(self):
+ return '{:.2f}%'.format(max(self.image_diff_result.diff_percent, 0.01))
+
+ def formatted_fuzzy_data(self):
+ fuzzy_data = self.image_diff_result.fuzzy_data if self.image_diff_result else None
+ if fuzzy_data:
+ return 'maxDifference={}; totalPixels={}'.format(fuzzy_data['max_difference'], fuzzy_data['total_pixels'])
+ return None
+
def write_failure(self, writer, driver_output, expected_driver_output, port):
writer.write_image_files(driver_output.image, expected_driver_output.image)
if self.image_diff_result:
@@ -236,7 +257,7 @@
else:
diff_image = self.image_diff_result.diff_image
- writer.write_image_diff_files(diff_image)
+ writer.write_image_diff_files(diff_image, self.formatted_diff_percent(), self.formatted_fuzzy_data())
else:
_log.warn('ref test mismatch did not produce an image diff.')
writer.write_reftest(self.reference_filename)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_run_results.py (285212 => 285213)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_run_results.py 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_run_results.py 2021-11-03 17:05:50 UTC (rev 285213)
@@ -215,6 +215,11 @@
if isinstance(failure, test_failures.FailureImageHashMismatch) or isinstance(failure, test_failures.FailureReftestMismatch):
test_dict['image_diff_percent'] = failure.image_diff_result.diff_percent
+ if 'image_difference' not in test_dict:
+ for failure in failures:
+ if (isinstance(failure, test_failures.FailureImageHashMismatch) or isinstance(failure, test_failures.FailureReftestMismatch)) and failure.image_diff_result.fuzzy_data:
+ test_dict['image_difference'] = failure.image_diff_result.fuzzy_data
+
return test_dict
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py (285212 => 285213)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py 2021-11-03 17:05:50 UTC (rev 285213)
@@ -128,8 +128,10 @@
test_dict = test_run_results._interpret_test_failures([test_failures.FailureImageHashMismatch(ImageDiffResult(passed=False, diff_image=b'', difference=0.42))])
self.assertEqual(test_dict['image_diff_percent'], 0.42)
- test_dict = test_run_results._interpret_test_failures([test_failures.FailureReftestMismatch(self.port.abspath_for_test('foo/reftest-expected.html'), ImageDiffResult(passed=False, diff_image=b'', difference=100.0))])
+ result_fuzzy_data = {'max_difference': 6, 'total_pixels': 50}
+ test_dict = test_run_results._interpret_test_failures([test_failures.FailureReftestMismatch(self.port.abspath_for_test('foo/reftest-expected.html'), ImageDiffResult(passed=False, diff_image=b'', difference=100.0, fuzzy_data=result_fuzzy_data))])
self.assertIn('image_diff_percent', test_dict)
+ self.assertIn('image_difference', test_dict)
test_dict = test_run_results._interpret_test_failures([test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-mismatch.html'))])
self.assertEqual(len(test_dict), 0)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (285212 => 285213)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2021-11-03 17:05:50 UTC (rev 285213)
@@ -517,7 +517,7 @@
def test_pixel_test_directories(self):
host = MockHost()
- """Both tests have faling checksum. We include only the first in pixel tests so only that should fail."""
+ """Both tests have failing checksum. We include only the first in pixel tests so only that should fail."""
args = ['--pixel-tests', '--pixel-test-directory', 'failures/unexpected/pixeldir',
'failures/unexpected/pixeldir/image_in_pixeldir.html',
'failures/unexpected/image_not_in_pixeldir.html']
@@ -524,9 +524,13 @@
details, err, _ = logging_run(extra_args=args, host=host, tests_included=True)
self.assertEqual(details.exit_code, 1)
- expected_token = '"unexpected":{"pixeldir":{"image_in_pixeldir.html":{"report":"REGRESSION","expected":"PASS","actual":"IMAGE"'
+
json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
- self.assertTrue(json_string.find(expected_token) != -1)
+ json = parse_full_results(json_string)
+ test_data = json["tests"]["failures"]["unexpected"]["pixeldir"]["image_in_pixeldir.html"]
+ self.assertEqual(test_data["expected"], "PASS")
+ self.assertEqual(test_data["actual"], "IMAGE")
+ self.assertEqual(test_data["report"], "REGRESSION")
def test_missing_and_unexpected_results_with_custom_exit_code(self):
# Test that we update expectations in place. If the expectation
@@ -710,7 +714,7 @@
json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
json = parse_full_results(json_string)
self.assertEqual(json["tests"]["failures"]["unexpected"]["text-image-checksum.html"],
- {"expected": "PASS", "actual": "TEXT IMAGE+TEXT", "image_diff_percent": 1, "report": "REGRESSION"})
+ {"expected": "PASS", "actual": "TEXT IMAGE+TEXT", "image_diff_percent": 1, 'image_difference': {'max_difference': 10, 'total_pixels': 20}, "report": "REGRESSION"})
self.assertFalse(json["pixel_tests_enabled"])
self.assertEqual(details.enabled_pixel_tests_in_retry, True)
@@ -723,9 +727,15 @@
tests_included=True, host=host)
file_list = host.filesystem.written_files.keys()
self.assertEqual(details.exit_code, 1)
- expected_token = '"unexpected":{"text-image-missing.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT MISSING","is_missing_image":true}}'
json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
- self.assertTrue(json_string.find(expected_token) != -1)
+
+ json = parse_full_results(json_string)
+ test_data = json["tests"]["failures"]["unexpected"]["text-image-missing.html"]
+ self.assertEqual(test_data["expected"], "PASS")
+ self.assertEqual(test_data["actual"], "TEXT MISSING")
+ self.assertEqual(test_data["report"], "REGRESSION")
+ self.assertEqual(test_data["is_missing_image"], True)
+
self.assertTrue(json_string.find('"num_regressions":1') != -1)
self.assertTrue(json_string.find('"num_flaky":0') != -1)
self.assertTrue(json_string.find('"num_missing":1') != -1)
