Added: releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/common/test_expectations.py (0 => 227499)
--- releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/common/test_expectations.py (rev 0)
+++ releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/common/test_expectations.py 2018-01-24 10:01:53 UTC (rev 227499)
@@ -0,0 +1,114 @@
+# Copyright (C) 2018 Igalia S.L.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import json
+import os
+
+
+class TestExpectations(object):
+
+ def __init__(self, port_name, expectations_file, build_type='Release'):
+ self._port_name = port_name
+ self._build_type = build_type
+ if os.path.isfile(expectations_file):
+ with open(expectations_file, 'r') as fd:
+ self._expectations = json.load(fd)
+ else:
+ self._expectations = {}
+
+ def _port_name_for_expected(self, expected):
+ if self._port_name in expected:
+ return self._port_name
+
+ name_with_build = self._port_name + '@' + self._build_type
+ if name_with_build in expected:
+ return name_with_build
+
+ if 'all' in expected:
+ return 'all'
+
+ name_with_build = 'all@' + self._build_type
+ if name_with_build in expected:
+ return name_with_build
+
+ return None
+
+ def _expected_value(self, expected, value, default):
+ port_name = self._port_name_for_expected(expected)
+ if port_name is None:
+ return default
+
+ port_expected = expected[port_name]
+ if value in port_expected:
+ return port_expected[value]
+
+ return default
+
+ def skipped_tests(self):
+ skipped = []
+ for test in self._expectations:
+ if 'expected' not in self._expectations[test]:
+ continue
+
+ expected = self._expectations[test]['expected']
+ if 'SKIP' in self._expected_value(expected, 'status', []):
+ skipped.append(test)
+ return skipped
+
+ def skipped_subtests(self, test):
+ skipped = []
+ if test not in self._expectations:
+ return skipped
+
+ test_expectation = self._expectations[test]
+ if 'subtests' not in test_expectation:
+ return skipped
+
+ subtests = test_expectation['subtests']
+ for subtest in subtests:
+ if 'SKIP' in self._expected_value(subtests[subtest]['expected'], 'status', []):
+ skipped.append(subtest)
+ return skipped
+
+ def _expectation_value(self, test, subtest, value, default):
+ retval = default
+ if test not in self._expectations:
+ return retval
+
+ test_expectation = self._expectations[test]
+ if 'expected' in test_expectation:
+ retval = self._expected_value(test_expectation['expected'], value, retval)
+
+ if subtest is None or 'subtests' not in test_expectation:
+ return retval
+
+ subtests = test_expectation['subtests']
+ if subtest not in subtests:
+ return retval
+
+ return self._expected_value(subtests[subtest]['expected'], value, retval)
+
+ def is_slow(self, test, subtest=None):
+ return self._expectation_value(test, subtest, 'slow', False)
+
+ def get_expectation(self, test, subtest=None):
+ return self._expectation_value(test, subtest, 'status', ['PASS'])
Added: releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/common/test_expectations_unittest.py (0 => 227499)
--- releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/common/test_expectations_unittest.py (rev 0)
+++ releases/WebKitGTK/webkit-2.18/Tools/Scripts/webkitpy/common/test_expectations_unittest.py 2018-01-24 10:01:53 UTC (rev 227499)
@@ -0,0 +1,254 @@
+# Copyright (C) 2018 Igalia S.L.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import unittest
+
+import json
+import os
+
+from webkitpy.common.host_mock import MockHost
+from webkitpy.common.test_expectations import TestExpectations
+from webkitpy.common.webkit_finder import WebKitFinder
+
+
+class MockTestExpectations(TestExpectations):
+
+ def __init__(self, port, expectations, build_type='Release'):
+ host = MockHost()
+ port = host.port_factory.get(port)
+ self._port_name = port.name()
+ self._build_type = build_type
+ self._expectations = json.loads(expectations)
+
+ def is_skip(self, test, subtest):
+ if test in self.skipped_tests():
+ return True
+ return subtest in self.skipped_subtests(test)
+
+
+class ExpectationsTest(unittest.TestCase):
+
+ BASIC = """
+{
+ "imported/w3c/webdriver/tests/test1.py": {
+ "subtests": {
+ "test1_one": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "1234"}}
+ },
+ "test1_two": {
+ "expected": {
+ "all": {"status": ["FAIL"], "bug": "1234"},
+ "gtk": {"status": ["PASS"]}
+ }
+ }
+ }
+ },
+ "imported/w3c/webdriver/tests/test2.py": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "1234"}}
+ },
+ "imported/w3c/webdriver/tests/test3.py": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "1234"}},
+ "subtests": {
+ "test3_one": {
+ "expected": {"all": {"status": ["PASS"]}}
+ }
+ }
+ }
+}"""
+
+ SKIP = """
+{
+ "imported/w3c/webdriver/tests/test1.py": {
+ "expected": {"all": {"status": ["SKIP"], "bug": "1234"}}
+ },
+ "imported/w3c/webdriver/tests/test2.py": {
+ "expected": {"gtk": {"status": ["SKIP"], "bug": "1234"}}
+ },
+ "imported/w3c/webdriver/tests/test3.py": {
+ "expected": {"all": {"status": ["SKIP"], "bug": "1234"}},
+ "subtests": {
+ "test3_one": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "1234"}}
+ }
+ }
+ },
+ "imported/w3c/webdriver/tests/test4.py": {
+ "subtests": {
+ "test4_one": {
+ "expected": {"all": {"status": ["SKIP"], "bug": "1234"}}
+ }
+ }
+ }
+}"""
+
+ FLAKY = """
+{
+ "TestCookieManager": {
+ "subtests": {
+ "/webkit2/WebKitCookieManager/persistent-storage": {
+ "expected": {"gtk": {"status": ["FAIL", "PASS"], "bug": "1234"}}
+ }
+ }
+ },
+ "TestWebKit": {
+ "subtests": {
+ "WebKit.MouseMoveAfterCrash": {
+ "expected": {"all": {"status": ["CRASH", "PASS"], "bug": "1234"}}
+ },
+ "WebKit.WKConnection": {
+ "expected": {"wpe": {"status": ["FAIL", "TIMEOUT"], "bug": "1234"}}
+ }
+ }
+ }
+}"""
+
+ BUILD_TYPE = """
+{
+ "TestCookieManager": {
+ "subtests": {
+ "/webkit2/WebKitCookieManager/persistent-storage": {
+ "expected": {"all": {"status": ["FAIL"], "bug": "1234"}}
+ }
+ }
+ },
+ "TestWebKit": {
+ "subtests": {
+ "WebKit.MouseMoveAfterCrash": {
+ "expected": {"all@Debug": {"status": ["CRASH"], "bug": "1234"}}
+ },
+ "WebKit.WKConnection": {
+ "expected": {"gtk@Release": {"status": ["FAIL"], "bug": "1234"},
+ "gtk@Debug": {"status": ["CRASH"], "bug": "1234"}}
+ }
+ }
+ },
+ "TestWebCore": {
+ "expected": {"all@Debug": {"status": ["CRASH"]}},
+ "subtests": {
+ "ComplexTextControllerTest.InitialAdvanceWithLeftRunInRTL": {
+ "expected": {"all": {"status": ["PASS"], "bug": "1234"}}
+ },
+ "FileMonitorTest.DetectChange": {
+ "expected": {"all@Release": {"status": ["FAIL"], "bug": "1234"}}
+ }
+ }
+ },
+ "TestWebViewEditor": {
+ "expected": {"all@Release": {"status": ["SKIP"]},
+ "wpe@Debug": {"status": ["SKIP"]}},
+ "subtests": {
+ "/webkit2/WebKitWebView/editable/editable": {
+ "expected": {"gtk": {"status": ["FAIL"], "bug": "1234"}}
+ }
+ }
+ }
+}"""
+
+ def assert_exp(self, test, subtest, result):
+ self.assertIn(result, self.expectations.get_expectation(test, subtest))
+
+ def assert_not_exp(self, test, subtest, result):
+ self.assertNotIn(result, self.expectations.get_expectation(test, subtest))
+
+ def assert_bad_exp(self, test):
+ self.assertRaises(AssertionError, self.expectations.get_expectation(test))
+
+ def assert_skip(self, test, subtest, result):
+ self.assertEqual(self.expectations.is_skip(test, subtest), result)
+
+ def test_basic(self):
+ self.expectations = MockTestExpectations('gtk', self.BASIC)
+ self.assert_exp('imported/w3c/webdriver/tests/test5.py', 'test5_two', 'PASS')
+
+ self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_five', 'PASS')
+ self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_one', 'FAIL')
+ self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_two', 'PASS')
+
+ self.assert_exp('imported/w3c/webdriver/tests/test2.py', 'test2_one', 'FAIL')
+
+ self.assert_exp('imported/w3c/webdriver/tests/test3.py', 'test3_two', 'FAIL')
+ self.assert_exp('imported/w3c/webdriver/tests/test3.py', 'test3_one', 'PASS')
+
+ self.expectations = MockTestExpectations('wpe', self.BASIC)
+ self.assert_exp('imported/w3c/webdriver/tests/test1.py', 'test1_two', 'FAIL')
+
+ def test_skip(self):
+ self.expectations = MockTestExpectations('gtk', self.SKIP)
+ self.assert_skip('imported/w3c/webdriver/tests/test5.py', None, False)
+ self.assert_skip('imported/w3c/webdriver/tests/test1.py', None, True)
+ self.assert_skip('imported/w3c/webdriver/tests/test2.py', None, True)
+ self.assert_skip('imported/w3c/webdriver/tests/test3.py', None, True)
+ self.assert_skip('imported/w3c/webdriver/tests/test3.py', 'test3_one', True)
+ self.assert_skip('imported/w3c/webdriver/tests/test4.py', None, False)
+ self.assert_skip('imported/w3c/webdriver/tests/test4.py', 'test4_one', True)
+ self.assert_exp('imported/w3c/webdriver/tests/test4.py', 'test4_one', 'SKIP')
+
+ self.expectations = MockTestExpectations('wpe', self.SKIP)
+ self.assert_skip('imported/w3c/webdriver/tests/test2.py', None, False)
+
+ def test_flaky(self):
+ self.expectations = MockTestExpectations('gtk', self.FLAKY)
+ self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'PASS')
+ self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL')
+ self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'CRASH')
+ self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'PASS')
+ self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'PASS')
+ self.assert_not_exp('TestWebKit', 'WebKit.WKConnection', 'FAIL')
+ self.assert_not_exp('TestWebKit', 'WebKit.WKConnection', 'TIMEOUT')
+
+ self.expectations = MockTestExpectations('wpe', self.FLAKY)
+ self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'PASS')
+ self.assert_not_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL')
+ self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'CRASH')
+ self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'PASS')
+ self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'FAIL')
+ self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'TIMEOUT')
+
+ def test_build_type(self):
+ self.expectations = MockTestExpectations('gtk', self.BUILD_TYPE, 'Debug')
+ self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL')
+ self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'CRASH')
+ self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'CRASH')
+ self.assert_exp('TestWebCore', 'ComplexTextControllerTest.InitialAdvanceWithLeftRunInRTL', 'PASS')
+ self.assert_exp('TestWebCore', 'FileMonitorTest.DetectChange', 'CRASH')
+ self.assert_exp('TestWebCore', 'FileSystemTest.MappingMissingFile', 'CRASH')
+ self.assert_skip('TestWebViewEditor', None, False)
+ self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', False)
+ self.assert_exp('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', 'FAIL')
+
+ self.expectations = MockTestExpectations('gtk', self.BUILD_TYPE, 'Release')
+ self.assert_exp('TestCookieManager', '/webkit2/WebKitCookieManager/persistent-storage', 'FAIL')
+ self.assert_exp('TestWebKit', 'WebKit.MouseMoveAfterCrash', 'PASS')
+ self.assert_exp('TestWebKit', 'WebKit.WKConnection', 'FAIL')
+ self.assert_exp('TestWebCore', 'ComplexTextControllerTest.InitialAdvanceWithLeftRunInRTL', 'PASS')
+ self.assert_exp('TestWebCore', 'FileMonitorTest.DetectChange', 'FAIL')
+ self.assert_exp('TestWebCore', 'FileSystemTest.MappingMissingFile', 'PASS')
+ self.assert_skip('TestWebViewEditor', None, True)
+ self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', True)
+
+ self.expectations = MockTestExpectations('wpe', self.BUILD_TYPE, 'Release')
+ self.assert_skip('TestWebViewEditor', None, True)
+ self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', True)
+
+ self.expectations = MockTestExpectations('wpe', self.BUILD_TYPE, 'Debug')
+ self.assert_skip('TestWebViewEditor', None, True)
+ self.assert_skip('TestWebViewEditor', '/webkit2/WebKitWebView/editable/editable', True)
