Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py (284986 => 284987)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py 2021-10-28 16:28:03 UTC (rev 284986)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py 2021-10-28 16:36:30 UTC (rev 284987)
@@ -28,6 +28,7 @@
import logging
+import os
import re
from webkitcorepy import string_utils
@@ -36,6 +37,7 @@
from webkitpy.layout_tests.models import test_expectations
from webkitpy.layout_tests.models import test_failures
from webkitpy.layout_tests.models.test_results import TestResult
+from webkitpy.w3c.test_parser import TestParser
_log = logging.getLogger(__name__)
@@ -334,6 +336,46 @@
reftest_type = set([reference_file[0] for reference_file in self._reference_files])
return TestResult(self._test_input, test_result.failures, total_test_time + test_result.test_run_time, test_result.has_stderr, reftest_type=reftest_type, pid=test_result.pid, references=reference_test_names)
+ @staticmethod
+ def _relative_reference_path(test_full_path, reference_full_path):
+ test_dir = os.path.split(test_full_path)[0]
+ reference_dir, reference_file_name = os.path.split(reference_full_path)
+ if test_dir == reference_dir:
+ return reference_file_name
+
+ relative_path = os.path.relpath(reference_dir, test_dir)
+ return os.path.join(relative_path, reference_file_name)
+
+ def _fuzzy_tolerance_for_reference(self, reference_full_path):
+ test_full_path = self._port.abspath_for_test(self._test_name)
+ test_parser = TestParser({'all': True}, filename=test_full_path, host=self._port.host)
+ fuzzy = test_parser.fuzzy_metadata()
+ if not fuzzy:
+ return {'maxDifference': [0, 0], 'totalPixels': [0, 0]}
+
+ reference_relative_path = self._relative_reference_path(test_full_path, reference_full_path)
+ tolerance = [[0, 0], [0, 0]]
+ if reference_relative_path in fuzzy:
+ tolerance = fuzzy[reference_relative_path]
+ elif None in fuzzy:
+ tolerance = fuzzy[None]
+
+ return {'max_difference': tolerance[0], 'total_pixels': tolerance[1]}
+
+ @staticmethod
+ def _test_passes_fuzzy_matching(allowed_fuzzy_values, fuzzy_result):
+ maxDifferenceMin = allowed_fuzzy_values['max_difference'][0]
+ maxDifferenceMax = allowed_fuzzy_values['max_difference'][1]
+
+ totalPixelsMin = allowed_fuzzy_values['total_pixels'][0]
+ totalPixelsMax = allowed_fuzzy_values['total_pixels'][1]
+
+ actualMaxDifference = fuzzy_result['max_difference']
+ actualTotalPixels = fuzzy_result['total_pixels']
+
+ # https://web-platform-tests.org/writing-tests/reftests.html says "in the range" but isn't precise about whether the upper bound is included.
+ return actualMaxDifference >= maxDifferenceMin and actualMaxDifference <= maxDifferenceMax and actualTotalPixels >= totalPixelsMin and actualTotalPixels <= totalPixelsMax
+
def _compare_output_with_reference(self, reference_driver_output, actual_driver_output, reference_filename, mismatch):
total_test_time = reference_driver_output.test_time + actual_driver_output.test_time
has_stderr = reference_driver_output.has_stderr() or actual_driver_output.has_stderr()
@@ -355,6 +397,9 @@
elif reference_driver_output.image_hash != actual_driver_output.image_hash:
# ImageDiff has a hard coded color distance threshold even though tolerance=0 is specified.
diff_result = self._port.diff_image(reference_driver_output.image, actual_driver_output.image, tolerance=0)
+
+ # FIXME: use the result of _fuzzy_tolerance_for_reference to allow for pass or fail based on fuzzy matching. webkit.org/b/149828
+
if not diff_result.passed:
failures.append(test_failures.FailureReftestMismatch(reference_filename, diff_result))
if diff_result.error_string:
Added: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner_unittest.py (0 => 284987)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner_unittest.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner_unittest.py 2021-10-28 16:36:30 UTC (rev 284987)
@@ -0,0 +1,115 @@
+# Copyright (C) 2021 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import unittest
+
+from webkitpy.common.host_mock import MockHost
+from webkitpy.layout_tests.controllers.single_test_runner import SingleTestRunner
+from webkitpy.layout_tests.models.test_input import Test, TestInput
+from webkitpy.port.driver import DriverOutput
+from webkitpy.port.test import TestPort
+
+
+class TestDriver:
+ def run_test(self, driver_input, stop_when_done):
+ text = ''
+ timeout = False
+ crash = False
+ return DriverOutput(text, '', '', '', crash=crash, timeout=timeout)
+
+ def start(self):
+ """do nothing"""
+
+ def stop(self):
+ """do nothing"""
+
+
+class SingleTestRunnerTest(unittest.TestCase):
+
+ def _add_file(self, port, file_path, contents):
+ filesystem = port.host.filesystem
+ file_dir, file_name = os.path.split(file_path)
+ dirname = filesystem.join(port.layout_tests_dir(), file_dir)
+ filesystem.maybe_make_directory(dirname)
+ filesystem.write_binary_file(filesystem.join(dirname, file_name), contents)
+
+ def _make_test_runner(self, test_name):
+ host = MockHost()
+ port = TestPort(host)
+ driver = TestDriver()
+ results_directory = 'layout-test-results'
+ worker_name = ''
+
+ test_input = TestInput(Test(test_name))
+ return SingleTestRunner(port, port._options, results_directory, worker_name, driver, test_input, True)
+
+ def test_fuzzy_matching_values(self):
+ single_test_runner = self._make_test_runner('fuzzy-test.html')
+ self._add_file(single_test_runner._port, 'fuzzy-test.html', b'<html><head><meta name=fuzzy content="maxDifference=15;totalPixels=300">')
+ fuzzy_data = single_test_runner._fuzzy_tolerance_for_reference('/test.checkout/LayoutTests/fuzzy-test-expected.html')
+ self.assertEqual(fuzzy_data, {'max_difference': [15, 15], 'total_pixels': [300, 300]})
+
+ def test_fuzzy_matching_values_for_ref(self):
+ test_name = 'fuzzy-test.html'
+ single_test_runner = self._make_test_runner(test_name)
+ self._add_file(single_test_runner._port, test_name, """<html><head>
+ <meta name=fuzzy content="maxDifference=15;totalPixels=300">
+ <meta name=fuzzy content="reference.html:maxDifference=5-8;totalPixels=78-84">
+ """)
+ fuzzy_data = single_test_runner._fuzzy_tolerance_for_reference('/test.checkout/LayoutTests/reference.html')
+ self.assertEqual(fuzzy_data, {'max_difference': [5, 8], 'total_pixels': [78, 84]})
+
+ def test_fuzzy_matching_values_for_relative_path_ref(self):
+ test_name = 'fast/borders/fuzzy-test.html'
+ single_test_runner = self._make_test_runner(test_name)
+ self._add_file(single_test_runner._port, test_name, """<html><head>
+ <meta name=fuzzy content="maxDifference=15;totalPixels=300">
+ <meta name=fuzzy content="../resources/common-ref.html:maxDifference=5-8;totalPixels=78-84">
+ """)
+ self._add_file(single_test_runner._port, 'fast/resources/common-ref.html', b'')
+ fuzzy_data = single_test_runner._fuzzy_tolerance_for_reference('/test.checkout/LayoutTests/fast/resources/common-ref.html')
+ self.assertEqual(fuzzy_data, {'max_difference': [5, 8], 'total_pixels': [78, 84]})
+
+ def test_fuzzy_matching_values_no_common_data(self):
+ test_name = 'fast/borders/fuzzy-test.html'
+ single_test_runner = self._make_test_runner(test_name)
+ self._add_file(single_test_runner._port, test_name, """<html><head>
+ <meta name=fuzzy content="../resources/common-ref.html:maxDifference=5-8;totalPixels=78-84">
+ """)
+
+ fuzzy_data = single_test_runner._fuzzy_tolerance_for_reference('/test.checkout/LayoutTests/reference.html')
+ self.assertEqual(fuzzy_data, {'max_difference': [0, 0], 'total_pixels': [0, 0]})
+
+ def test_fuzzy_matching_comparisons(self):
+ self.assertTrue(SingleTestRunner._test_passes_fuzzy_matching({'max_difference': [0, 0], 'total_pixels': [0, 0]}, {'max_difference': 0, 'total_pixels': 0}))
+ self.assertFalse(SingleTestRunner._test_passes_fuzzy_matching({'max_difference': [0, 0], 'total_pixels': [0, 0]}, {'max_difference': 1, 'total_pixels': 1}))
+
+ self.assertTrue(SingleTestRunner._test_passes_fuzzy_matching({'max_difference': [5, 7], 'total_pixels': [10, 12]}, {'max_difference': 5, 'total_pixels': 10}))
+ self.assertTrue(SingleTestRunner._test_passes_fuzzy_matching({'max_difference': [5, 7], 'total_pixels': [10, 12]}, {'max_difference': 6, 'total_pixels': 11}))
+ self.assertTrue(SingleTestRunner._test_passes_fuzzy_matching({'max_difference': [5, 7], 'total_pixels': [10, 12]}, {'max_difference': 7, 'total_pixels': 12}))
+
+ self.assertFalse(SingleTestRunner._test_passes_fuzzy_matching({'max_difference': [5, 7], 'total_pixels': [10, 12]}, {'max_difference': 0, 'total_pixels': 0}))
+ self.assertFalse(SingleTestRunner._test_passes_fuzzy_matching({'max_difference': [5, 7], 'total_pixels': [10, 12]}, {'max_difference': 5, 'total_pixels': 8}))
+ self.assertFalse(SingleTestRunner._test_passes_fuzzy_matching({'max_difference': [5, 7], 'total_pixels': [10, 12]}, {'max_difference': 3, 'total_pixels': 11}))
+ self.assertFalse(SingleTestRunner._test_passes_fuzzy_matching({'max_difference': [5, 7], 'total_pixels': [10, 12]}, {'max_difference': 9, 'total_pixels': 11}))
+ self.assertFalse(SingleTestRunner._test_passes_fuzzy_matching({'max_difference': [5, 7], 'total_pixels': [10, 12]}, {'max_difference': 6, 'total_pixels': 13}))
