Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py (92281 => 92282)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py 2011-08-03 15:30:26 UTC (rev 92281)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py 2011-08-03 15:33:37 UTC (rev 92282)
@@ -119,7 +119,7 @@
return 'ParseError(fatal=%s, errors=%s)' % (self.fatal, self.errors)
-class TestExpectationSerializer:
+class TestExpectationSerializer(object):
"""Provides means of serializing TestExpectationLine instances."""
def __init__(self, test_configuration_converter):
self._test_configuration_converter = test_configuration_converter
@@ -139,7 +139,7 @@
modifiers = self._parsed_modifier_string(expectation_line, specifiers)
expectations = self._parsed_expectations_string(expectation_line)
result.append(self._format_result(modifiers, expectation_line.name, expectations, expectation_line.comment))
- return "\n".join(result)
+ return "\n".join(result) if result else None
return self._format_result(" ".join(expectation_line.modifiers), expectation_line.name, " ".join(expectation_line.expectations), expectation_line.comment)
@@ -166,12 +166,12 @@
return result
@classmethod
- def list_to_string(cls, expectations, test_configuration_converter):
+ def list_to_string(cls, expectation_lines, test_configuration_converter):
serializer = cls(test_configuration_converter)
- return "\n".join([serializer.to_string(expectation) for expectation in expectations])
+ return "\n".join([line for line in [serializer.to_string(expectation_line) for expectation_line in expectation_lines] if line is not None])
-class TestExpectationParser:
+class TestExpectationParser(object):
"""Provides parsing facilities for lines in the test_expectation.txt file."""
BUG_MODIFIER_PREFIX = 'bug'
@@ -185,7 +185,7 @@
def __init__(self, port, full_test_list, allow_rebaseline_modifier):
self._port = port
- self._test_configuration_converter = TestConfigurationConverter(port.all_test_configurations(), port.configuration_specifier_macros())
+ self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros())
self._full_test_list = full_test_list
self._allow_rebaseline_modifier = allow_rebaseline_modifier
@@ -359,6 +359,9 @@
def is_invalid(self):
return self.is_malformed() or len(self.warnings) > 0
+ def is_flaky(self):
+ return len(self.parsed_expectations) > 1
+
@classmethod
def create_passing_expectation(cls, test):
expectation_line = TestExpectationLine()
@@ -370,7 +373,7 @@
# FIXME: Refactor API to be a proper CRUD.
-class TestExpectationsModel:
+class TestExpectationsModel(object):
"""Represents relational store of all expectations and provides CRUD semantics to manage it."""
def __init__(self):
@@ -431,6 +434,9 @@
def has_test(self, test):
return test in self._test_to_expectation_line
+ def get_expectation_line(self, test):
+ return self._test_to_expectation_line.get(test)
+
def get_expectations(self, test):
return self._test_to_expectations[test]
@@ -478,7 +484,7 @@
self._result_type_to_tests[SKIP].add(test)
elif expectation_line.parsed_expectations == set([PASS]):
self._result_type_to_tests[PASS].add(test)
- elif len(expectation_line.parsed_expectations) > 1:
+ elif expectation_line.is_flaky():
self._result_type_to_tests[FLAKY].add(test)
else:
self._result_type_to_tests[FAIL].add(test)
@@ -573,7 +579,119 @@
return False
-class TestExpectations:
+class BugManager(object):
+ """A simple interface for managing bugs from TestExpectationsEditor."""
+ def close_bug(self, bug_id, reference_bug_id=None):
+ raise NotImplementedError("BugManager.close_bug")
+
+ def create_bug(self):
+ """Should return a newly created bug id in the form of r"BUG[^\d].*"."""
+ raise NotImplementedError("BugManager.create_bug")
+
+
+class TestExpectationsEditor(object):
+ """
+ The editor assumes that the expectation data is error-free.
+ """
+
+ def __init__(self, expectation_lines, bug_manager):
+ self._bug_manager = bug_manager
+ self._expectation_lines = expectation_lines
+ self._tests_with_directory_paths = set()
+ # FIXME: Unify this with TestExpectationsModel.
+ self._test_to_expectation_lines = {}
+ for expectation_line in expectation_lines:
+ for test in expectation_line.matching_tests:
+ if test == expectation_line.path:
+ self._test_to_expectation_lines.setdefault(test, []).append(expectation_line)
+ else:
+ self._tests_with_directory_paths.add(test)
+
+ def remove_expectation(self, test, test_config_set, remove_flakes=False):
+ """Removes existing expectations for {test} in the of test configurations {test_config_set}.
+ If the test is flaky, the expectation is not removed, unless remove_flakes is True.
+
+ In this context, removing expectations does not imply that the test is passing -- we are merely removing
+ any information about this test from the expectations.
+
+ We do not remove the actual expectation lines here. Instead, we adjust TestExpectationLine.matching_configurations.
+ The serializer will figure out what to do:
+ * An empty matching_configurations set means that the this line matches nothing and will serialize as None.
+ * A matching_configurations set that can't be expressed as one line will be serialized as multiple lines.
+
+ Also, we do only adjust matching_configurations for lines that match tests exactly, because expectation lines with
+ better path matches are valid and always win.
+
+ For example, the expectation with the path "fast/events/shadow/" will
+ be ignored when removing expectations for the test "fast/event/shadow/awesome-crash.html", since we can just
+ add a new expectation line for "fast/event/shadow/awesome-crash.html" to influence expected results.
+ """
+ expectation_lines = self._test_to_expectation_lines.get(test, [])
+ for expectation_line in expectation_lines:
+ if (not expectation_line.is_flaky() or remove_flakes) and expectation_line.matching_configurations & test_config_set:
+ expectation_line.matching_configurations = expectation_line.matching_configurations - test_config_set
+ if not expectation_line.matching_configurations:
+ self._bug_manager.close_bug(expectation_line.parsed_bug_modifier)
+ return
+
+ def update_expectation(self, test, test_config_set, expectation_set, parsed_bug_modifier=None):
+ """Updates expectations for {test} in the set of test configuration {test_config_set} to the values of {expectation_set}.
+ If {parsed_bug_modifier} is supplied, it is used for updated expectations. Otherwise, a new bug is created.
+
+ Here, we treat updating expectations to PASS as special: if possible, the corresponding lines are completely removed.
+ """
+ # FIXME: Allow specifying modifiers (SLOW, SKIP, WONTFIX).
+ expectation_lines = self._test_to_expectation_lines.get(test, [])
+ remaining_configurations = test_config_set.copy()
+ bug_id = self._get_valid_bug_id(parsed_bug_modifier)
+ remove_expectations = expectation_set == set([PASS]) and test not in self._tests_with_directory_paths
+ for expectation_line in expectation_lines:
+ if expectation_line.matching_configurations == remaining_configurations:
+ # Tweak expectations on existing line.
+ if expectation_line.parsed_expectations == expectation_set:
+ return
+ self._bug_manager.close_bug(expectation_line.parsed_bug_modifier, bug_id)
+ if remove_expectations:
+ expectation_line.matching_configurations = set()
+ return
+ expectation_line.parsed_expectations = expectation_set
+ expectation_line.parsed_bug_modifier = bug_id
+ return
+ elif expectation_line.matching_configurations >= remaining_configurations:
+ # 1) Split up into two expectation lines:
+ # * one with old expectations (existing expectation_line)
+ # * one with new expectations (new expectation_line)
+ # 2) Finish looking, since there will be no more remaining configs to test for.
+ expectation_line.matching_configurations -= remaining_configurations
+ break
+ elif expectation_line.matching_configurations <= remaining_configurations:
+ # Remove existing expectation line.
+ self._bug_manager.close_bug(expectation_line.parsed_bug_modifier, bug_id)
+ expectation_line.matching_configurations = set()
+ else:
+ intersection = expectation_line.matching_configurations & remaining_configurations
+ if intersection:
+ expectation_line.matching_configurations -= intersection
+
+ if not remove_expectations:
+ self._expectation_lines.append(self._create_new_line(test, bug_id, remaining_configurations, expectation_set))
+
+ def _get_valid_bug_id(self, suggested_bug_id):
+ # FIXME: Flesh out creating a bug properly (title, etc.)
+ return suggested_bug_id or self._bug_manager.create_bug()
+
+ def _create_new_line(self, name, bug_id, config_set, expectation_set):
+ new_line = TestExpectationLine()
+ new_line.name = name
+ new_line.parsed_bug_modifier = bug_id
+ new_line.matching_configurations = config_set
+ new_line.parsed_expectations = expectation_set
+ # Ensure index integrity for multiple operations.
+ self._test_to_expectation_lines.setdefault(name, []).append(new_line)
+ return new_line
+
+
+class TestExpectations(object):
"""Test expectations consist of lines with specifications of what
to expect from layout test cases. The test cases can be directories
in which case the expectations apply to all test cases in that
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py (92281 => 92282)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py 2011-08-03 15:30:26 UTC (rev 92281)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py 2011-08-03 15:33:37 UTC (rev 92282)
@@ -33,7 +33,17 @@
from webkitpy.layout_tests.port import base
from webkitpy.layout_tests.models.test_configuration import *
from webkitpy.layout_tests.models.test_expectations import *
+from webkitpy.layout_tests.models.test_configuration import *
+
+class MockBugManager(object):
+ def close_bug(self, bug_id, reference_bug_id=None):
+ pass
+
+ def create_bug(self):
+ return "BUG_NEWLY_CREATED"
+
+
class FunctionsTest(unittest.TestCase):
def test_result_was_expected(self):
# test basics
@@ -495,7 +505,7 @@
expectation_line.parsed_bug_modifier = 'BUGX'
expectation_line.name = 'test/name/for/realz.html'
expectation_line.parsed_expectations = set([IMAGE])
- self.assertEqual(self._serializer.to_string(expectation_line), '')
+ self.assertEqual(self._serializer.to_string(expectation_line), None)
expectation_line.matching_configurations = set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu')])
self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP RELEASE CPU : test/name/for/realz.html = IMAGE')
expectation_line.matching_configurations = set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu'), TestConfiguration(None, 'xp', 'x86', 'release', 'gpu')])
@@ -571,5 +581,319 @@
self.assert_round_trip('FOO : bar = BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
+class TestExpectationEditorTests(unittest.TestCase):
+ WIN_RELEASE_CPU_CONFIGS = set([
+ TestConfiguration(None, 'vista', 'x86', 'release', 'cpu'),
+ TestConfiguration(None, 'win7', 'x86', 'release', 'cpu'),
+ TestConfiguration(None, 'xp', 'x86', 'release', 'cpu'),
+ ])
+
+ RELEASE_CONFIGS = set([
+ TestConfiguration(None, 'vista', 'x86', 'release', 'cpu'),
+ TestConfiguration(None, 'win7', 'x86', 'release', 'cpu'),
+ TestConfiguration(None, 'xp', 'x86', 'release', 'cpu'),
+ TestConfiguration(None, 'vista', 'x86', 'release', 'gpu'),
+ TestConfiguration(None, 'win7', 'x86', 'release', 'gpu'),
+ TestConfiguration(None, 'xp', 'x86', 'release', 'gpu'),
+ TestConfiguration(None, 'snowleopard', 'x86', 'release', 'cpu'),
+ TestConfiguration(None, 'leopard', 'x86', 'release', 'cpu'),
+ TestConfiguration(None, 'snowleopard', 'x86', 'release', 'gpu'),
+ TestConfiguration(None, 'leopard', 'x86', 'release', 'gpu'),
+ TestConfiguration(None, 'lucid', 'x86', 'release', 'cpu'),
+ TestConfiguration(None, 'lucid', 'x86_64', 'release', 'cpu'),
+ TestConfiguration(None, 'lucid', 'x86', 'release', 'gpu'),
+ TestConfiguration(None, 'lucid', 'x86_64', 'release', 'gpu'),
+ ])
+
+ def __init__(self, testFunc):
+ self.test_port = port.get('test-win-xp', None)
+ self.full_test_list = ['failures/expected/keyboard.html', 'failures/expected/audio.html']
+ unittest.TestCase.__init__(self, testFunc)
+
+ def make_parsed_expectation_lines(self, in_string):
+ expectation_lines = TestExpectationParser.tokenize_list(in_string)
+ parser = TestExpectationParser(self.test_port, self.full_test_list, allow_rebaseline_modifier=False)
+ for expectation_line in expectation_lines:
+ self.assertFalse(expectation_line.is_invalid())
+ parser.parse(expectation_line)
+ return expectation_lines
+
+ def assert_remove_roundtrip(self, in_string, test, expected_string, remove_flakes=False):
+ test_config_set = set([self.test_port.test_configuration()])
+ expectation_lines = self.make_parsed_expectation_lines(in_string)
+ editor = TestExpectationsEditor(expectation_lines, MockBugManager())
+ editor.remove_expectation(test, test_config_set, remove_flakes)
+ converter = TestConfigurationConverter(self.test_port.all_test_configurations(), self.test_port.configuration_specifier_macros())
+ result = TestExpectationSerializer.list_to_string(expectation_lines, converter)
+ self.assertEquals(result, expected_string)
+
+ def assert_update_roundtrip(self, in_string, test, expectation_set, expected_string, remove_flakes=False, parsed_bug_modifier=None, test_configs=None):
+ test_config_set = test_configs or set([self.test_port.test_configuration()])
+ expectation_lines = self.make_parsed_expectation_lines(in_string)
+ editor = TestExpectationsEditor(expectation_lines, MockBugManager())
+ editor.update_expectation(test, test_config_set, expectation_set, parsed_bug_modifier=parsed_bug_modifier)
+ converter = TestConfigurationConverter(self.test_port.all_test_configurations(), self.test_port.configuration_specifier_macros())
+ result = TestExpectationSerializer.list_to_string(expectation_lines, converter)
+ self.assertEquals(result, expected_string)
+
+ def test_remove_expectation(self):
+ self.assert_remove_roundtrip("""
+BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/hang.html', """
+BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""")
+
+ self.assert_remove_roundtrip("""
+BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
+BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""")
+
+ self.assert_remove_roundtrip("""
+BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
+BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""")
+
+ self.assert_remove_roundtrip("""
+BUGX1 MAC : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
+BUGX1 MAC : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""")
+
+ self.assert_remove_roundtrip("""
+BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
+BUGX2 WIN : failures/expected/audio.html = IMAGE""")
+
+ self.assert_remove_roundtrip("""
+BUGX1 WIN : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
+BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = IMAGE
+BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
+BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""")
+
+ self.assert_remove_roundtrip("""
+BUGX1 XP : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
+BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE
+BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""")
+
+ self.assert_remove_roundtrip("""
+BUGX1 : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""", 'failures/expected/keyboard.html', """
+BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = IMAGE
+BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
+BUGX1 LINUX MAC VISTA WIN7 : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""")
+
+ self.assert_remove_roundtrip("""
+BUGX1 WIN : failures/expected = PASS
+BUGX2 XP RELEASE : failures/expected/keyboard.html = IMAGE""", 'failures/expected/keyboard.html', """
+BUGX1 WIN : failures/expected = PASS
+BUGX2 XP RELEASE GPU : failures/expected/keyboard.html = IMAGE""")
+
+ self.assert_remove_roundtrip("""
+BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE
+BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""", 'failures/expected/keyboard.html', """
+BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""")
+
+ self.assert_remove_roundtrip("""
+BUGX1 WIN : failures/expected = FAIL""", 'failures/expected/keyboard.html', """
+BUGX1 WIN : failures/expected = FAIL""")
+
+ self.assert_remove_roundtrip("""
+BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE PASS
+BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""", 'failures/expected/keyboard.html', """
+BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = PASS IMAGE
+BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""")
+
+ self.assert_remove_roundtrip("""
+BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE PASS
+BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""", 'failures/expected/keyboard.html', """
+BUGX2 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE""", remove_flakes=True)
+
+ def test_remove_expectation_multiple(self):
+ in_string = """
+BUGX1 WIN : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE"""
+ expectation_lines = self.make_parsed_expectation_lines(in_string)
+ converter = TestConfigurationConverter(self.test_port.all_test_configurations(), self.test_port.configuration_specifier_macros())
+ editor = TestExpectationsEditor(expectation_lines, MockBugManager())
+ test = "failures/expected/keyboard.html"
+
+ editor.remove_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu')]))
+ self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
+BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = IMAGE
+BUGX1 XP DEBUG : failures/expected/keyboard.html = IMAGE
+BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""")
+
+ editor.remove_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'debug', 'cpu')]))
+ self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
+BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
+BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""")
+
+ editor.remove_expectation(test, set([TestConfiguration(None, 'vista', 'x86', 'debug', 'gpu'), TestConfiguration(None, 'win7', 'x86', 'release', 'gpu')]))
+ self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
+BUGX1 VISTA DEBUG CPU : failures/expected/keyboard.html = IMAGE
+BUGX1 WIN7 DEBUG GPU : failures/expected/keyboard.html = IMAGE
+BUGX1 WIN7 CPU : failures/expected/keyboard.html = IMAGE
+BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
+BUGX1 VISTA RELEASE : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""")
+
+ editor.remove_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'debug', 'gpu'), TestConfiguration(None, 'xp', 'x86', 'release', 'gpu')]))
+ self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
+BUGX1 VISTA DEBUG CPU : failures/expected/keyboard.html = IMAGE
+BUGX1 WIN7 RELEASE CPU : failures/expected/keyboard.html = IMAGE
+BUGX1 WIN7 DEBUG : failures/expected/keyboard.html = IMAGE
+BUGX1 VISTA RELEASE : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""")
+
+ editor.remove_expectation(test, set([TestConfiguration(None, 'vista', 'x86', 'debug', 'cpu'), TestConfiguration(None, 'vista', 'x86', 'debug', 'gpu'), TestConfiguration(None, 'vista', 'x86', 'release', 'gpu')]))
+ self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
+BUGX1 WIN7 DEBUG : failures/expected/keyboard.html = IMAGE
+BUGX1 VISTA WIN7 RELEASE CPU : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE""")
+
+ editor.remove_expectation(test, set(self.test_port.all_test_configurations()))
+ self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
+BUGX2 WIN : failures/expected/audio.html = IMAGE""")
+
+ test = "failures/expected/audio.html"
+
+ editor.remove_expectation(test, set(self.test_port.all_test_configurations()))
+ self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), "")
+
+ def test_update_expectation(self):
+ self.assert_update_roundtrip("""
+BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
+BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""")
+
+ self.assert_update_roundtrip("""
+BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([PASS]), '')
+
+ self.assert_update_roundtrip("""
+BUGX1 XP RELEASE CPU : failures/expected = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
+BUGX1 XP RELEASE CPU : failures/expected = TEXT
+BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""")
+
+ self.assert_update_roundtrip("""
+BUGX1 XP RELEASE CPU : failures/expected = TEXT""", 'failures/expected/keyboard.html', set([PASS]), """
+BUGX1 XP RELEASE CPU : failures/expected = TEXT
+BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = PASS""")
+
+ self.assert_update_roundtrip("""
+BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([TEXT]), """
+BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""")
+
+ self.assert_update_roundtrip("""
+BUGX1 XP RELEASE CPU : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
+BUGAWESOME XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""", parsed_bug_modifier='BUGAWESOME')
+
+ self.assert_update_roundtrip("""
+BUGX1 XP RELEASE : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
+BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = TEXT
+BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""")
+
+ self.assert_update_roundtrip("""
+BUGX1 XP RELEASE : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([PASS]), """
+BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = TEXT""")
+
+ self.assert_update_roundtrip("""
+BUGX1 XP RELEASE : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
+BUGX1 XP RELEASE GPU : failures/expected/keyboard.html = TEXT
+BUGAWESOME XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""", parsed_bug_modifier='BUGAWESOME')
+
+ self.assert_update_roundtrip("""
+BUGX1 WIN : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
+BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
+BUGX1 XP GPU : failures/expected/keyboard.html = TEXT
+BUGX1 VISTA WIN7 : failures/expected/keyboard.html = TEXT
+BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""")
+
+ self.assert_update_roundtrip("""
+BUGX1 WIN : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([PASS]), """
+BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
+BUGX1 XP GPU : failures/expected/keyboard.html = TEXT
+BUGX1 VISTA WIN7 : failures/expected/keyboard.html = TEXT""")
+
+ self.assert_update_roundtrip("""
+BUGX1 WIN : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
+BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
+BUGX1 XP GPU : failures/expected/keyboard.html = TEXT
+BUGX1 VISTA WIN7 : failures/expected/keyboard.html = TEXT
+BUG_NEWLY_CREATED XP RELEASE CPU : failures/expected/keyboard.html = IMAGE""")
+
+ self.assert_update_roundtrip("""
+BUGX1 XP RELEASE CPU: failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
+BUG_NEWLY_CREATED WIN RELEASE CPU : failures/expected/keyboard.html = IMAGE""", test_configs=self.WIN_RELEASE_CPU_CONFIGS)
+
+ self.assert_update_roundtrip("""
+BUGX1 XP RELEASE CPU: failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([PASS]), '', test_configs=self.WIN_RELEASE_CPU_CONFIGS)
+
+ self.assert_update_roundtrip("""
+BUGX1 RELEASE CPU: failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
+BUGX1 LINUX MAC RELEASE CPU : failures/expected/keyboard.html = TEXT
+BUG_NEWLY_CREATED WIN RELEASE CPU : failures/expected/keyboard.html = IMAGE""", test_configs=self.WIN_RELEASE_CPU_CONFIGS)
+
+ self.assert_update_roundtrip("""
+BUGX1 MAC : failures/expected/keyboard.html = TEXT""", 'failures/expected/keyboard.html', set([IMAGE]), """
+BUGX1 MAC : failures/expected/keyboard.html = TEXT
+BUG_NEWLY_CREATED WIN RELEASE CPU : failures/expected/keyboard.html = IMAGE""", test_configs=self.WIN_RELEASE_CPU_CONFIGS)
+
+ def test_update_expectation_multiple(self):
+ in_string = """
+BUGX1 WIN : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE"""
+ expectation_lines = self.make_parsed_expectation_lines(in_string)
+ converter = TestConfigurationConverter(self.test_port.all_test_configurations(), self.test_port.configuration_specifier_macros())
+ editor = TestExpectationsEditor(expectation_lines, MockBugManager())
+ test = "failures/expected/keyboard.html"
+
+ editor.update_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'release', 'cpu')]), set([IMAGE_PLUS_TEXT]), 'BUG_UPDATE1')
+ self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
+BUGX1 XP DEBUG CPU : failures/expected/keyboard.html = IMAGE
+BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
+BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE
+BUG_UPDATE1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE+TEXT""")
+
+ editor.update_expectation(test, set([TestConfiguration(None, 'xp', 'x86', 'debug', 'cpu')]), set([TEXT]), 'BUG_UPDATE2')
+ self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
+BUGX1 XP GPU : failures/expected/keyboard.html = IMAGE
+BUGX1 VISTA WIN7 : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE
+BUG_UPDATE1 XP RELEASE CPU : failures/expected/keyboard.html = IMAGE+TEXT
+BUG_UPDATE2 XP DEBUG CPU : failures/expected/keyboard.html = TEXT""")
+
+ editor.update_expectation(test, self.WIN_RELEASE_CPU_CONFIGS, set([CRASH]), 'BUG_UPDATE3')
+ self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
+BUGX1 VISTA DEBUG CPU : failures/expected/keyboard.html = IMAGE
+BUGX1 WIN7 RELEASE GPU : failures/expected/keyboard.html = IMAGE
+BUGX1 WIN7 DEBUG : failures/expected/keyboard.html = IMAGE
+BUGX1 VISTA XP GPU : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE
+BUG_UPDATE2 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
+BUG_UPDATE3 WIN RELEASE CPU : failures/expected/keyboard.html = CRASH""")
+
+ editor.update_expectation(test, self.RELEASE_CONFIGS, set([FAIL]), 'BUG_UPDATE4')
+ self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
+BUGX1 XP DEBUG GPU : failures/expected/keyboard.html = IMAGE
+BUGX1 VISTA WIN7 DEBUG : failures/expected/keyboard.html = IMAGE
+BUGX2 WIN : failures/expected/audio.html = IMAGE
+BUG_UPDATE2 XP DEBUG CPU : failures/expected/keyboard.html = TEXT
+BUG_UPDATE4 RELEASE : failures/expected/keyboard.html = FAIL""")
+
+ editor.update_expectation(test, set(self.test_port.all_test_configurations()), set([TIMEOUT]), 'BUG_UPDATE5')
+ self.assertEquals(TestExpectationSerializer.list_to_string(expectation_lines, converter), """
+BUGX2 WIN : failures/expected/audio.html = IMAGE
+BUG_UPDATE5 : failures/expected/keyboard.html = TIMEOUT""")
+
+
if __name__ == '__main__':
unittest.main()
