Diff
Modified: trunk/Tools/ChangeLog (92623 => 92624)
--- trunk/Tools/ChangeLog 2011-08-08 19:54:46 UTC (rev 92623)
+++ trunk/Tools/ChangeLog 2011-08-08 19:59:31 UTC (rev 92624)
@@ -1,3 +1,15 @@
+2011-08-05 Dimitri Glazkov <[email protected]>
+
+ Wire up updating expectations in garden-o-matic.
+ https://bugs.webkit.org/show_bug.cgi?id=65794
+
+ Reviewed by Adam Barth.
+
+ * Scripts/webkitpy/layout_tests/port/factory.py: Moved _port_for_builder from rebaseline.py to here, rename to get_from_builder_name.
+ * Scripts/webkitpy/tool/commands/rebaseline.py: Adapted callsites to new location.
+ * Scripts/webkitpy/tool/servers/gardeningserver.py: Added GardeningExpectationsUpdater, which knows how to update expectations.
+ * Scripts/webkitpy/tool/servers/gardeningserver_unittest.py: Added loads of tests.
+
2011-08-08 Jochen Eisinger <[email protected]>
webkit-patch doesn't get along with renamed files
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/factory.py (92623 => 92624)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/factory.py 2011-08-08 19:54:46 UTC (rev 92623)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/factory.py 2011-08-08 19:59:31 UTC (rev 92624)
@@ -29,12 +29,19 @@
"""Factory method to retrieve the appropriate port implementation."""
-
+import re
import sys
from webkitpy.layout_tests.port import builders
+class BuilderOptions(object):
+
+ def __init__(self, builder_name):
+ self.configuration = "Debug" if re.search(r"[d|D](ebu|b)g", builder_name) else "Release"
+ self.builder_name = builder_name
+
+
def all_port_names():
"""Return a list of all valid, fully-specified, "real" port names.
@@ -57,6 +64,14 @@
return _get_kwargs(**kwargs)
+def get_from_builder_name(builder_name):
+ port_name = builders.port_name_for_builder_name(builder_name)
+ assert(port_name) # Need to update port_name_for_builder_name
+ port = get(port_name, BuilderOptions(builder_name))
+ assert(port) # Need to update port_name_for_builder_name
+ return port
+
+
def _get_kwargs(**kwargs):
port_to_use = kwargs.get('port_name', None)
options = kwargs.get('options', None)
Modified: trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline.py (92623 => 92624)
--- trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline.py 2011-08-08 19:54:46 UTC (rev 92623)
+++ trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline.py 2011-08-08 19:59:31 UTC (rev 92624)
@@ -49,28 +49,11 @@
_baseline_suffix_list = ['png', 'txt']
-def _port_for_builder(builder_name):
- port_name = builders.port_name_for_builder_name(builder_name)
- assert(port_name) # Need to update port_name_for_builder_name
- port = factory.get(port_name, RebaseliningOptions(builder_name))
- assert(port) # Need to update port_name_for_builder_name
- return port
-
-
# FIXME: Should TestResultWriter know how to compute this string?
def _baseline_name(fs, test_name, suffix):
return fs.splitext(test_name)[0] + TestResultWriter.FILENAME_SUFFIX_EXPECTED + "." + suffix
-class RebaseliningOptions(object):
-
- DEBUG_CONFIGURATION_REGEX = r"[d|D](ebu|b)g"
-
- def __init__(self, builder_name):
- self.configuration = "Debug" if re.search(self.DEBUG_CONFIGURATION_REGEX, builder_name) else "Release"
- self.builder_name = builder_name
-
-
class RebaselineTest(AbstractDeclarativeCommand):
name = "rebaseline-test"
help_text = "Rebaseline a single test from a buildbot. (Currently works only with build.chromium.org buildbots.)"
@@ -83,7 +66,7 @@
return builder.accumulated_results_url()
def _baseline_directory(self, builder_name):
- port = _port_for_builder(builder_name)
+ port = factory.get_from_builder_name(builder_name)
return port.baseline_path()
def _save_baseline(self, data, target_baseline):
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/gardeningserver.py (92623 => 92624)
--- trunk/Tools/Scripts/webkitpy/tool/servers/gardeningserver.py 2011-08-08 19:54:46 UTC (rev 92623)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/gardeningserver.py 2011-08-08 19:59:31 UTC (rev 92624)
@@ -25,9 +25,45 @@
import BaseHTTPServer
import os
+from webkitpy.common.memoized import memoized
from webkitpy.tool.servers.reflectionhandler import ReflectionHandler
+from webkitpy.layout_tests.models.test_expectations import BugManager, TestExpectationParser, TestExpectations, TestExpectationsEditor, TestExpectationSerializer
+from webkitpy.layout_tests.models.test_configuration import TestConfigurationConverter
+from webkitpy.layout_tests.port import factory
+class GardeningExpectationsUpdater(BugManager):
+ def __init__(self, tool, port):
+ self._converter = TestConfigurationConverter(port.all_test_configurations(), port.configuration_specifier_macros())
+ self._parser = TestExpectationParser(port, [], allow_rebaseline_modifier=False)
+ self._path_to_test_expectations_file = port.path_to_test_expectations_file()
+ self._tool = tool
+
+ @memoized
+ def _builder_to_test_config(self, builder_name):
+ return factory.get_from_builder_name(builder_name).test_configuration()
+
+ def close_bug(self, bug_id, reference_bug_id=None):
+ # FIXME: Implement this properly.
+ pass
+
+ def create_bug(self):
+ return "BUG_NEW"
+
+ def update_expectations(self, failure_info_list):
+ expectation_lines = TestExpectationParser.tokenize_list(self._tool.filesystem.read_text_file(self._path_to_test_expectations_file))
+ for expectation_line in expectation_lines:
+ self._parser.parse(expectation_line)
+ editor = TestExpectationsEditor(expectation_lines, self)
+ for failure_info in failure_info_list:
+ expectation_set = set(filter(None, map(TestExpectations.expectation_from_string, failure_info['failureTypeList'])))
+ assert(expectation_set)
+ test_name = failure_info['testName']
+ assert(test_name)
+ editor.update_expectation(test_name, set([self._builder_to_test_config(failure_info['builderName'])]), expectation_set)
+ self._tool.filesystem.write_text_file(self._path_to_test_expectations_file, TestExpectationSerializer.list_to_string(expectation_lines, self._converter))
+
+
class GardeningHTTPServer(BaseHTTPServer.HTTPServer):
def __init__(self, httpd_port, config):
server_name = ''
@@ -69,6 +105,13 @@
def _run_webkit_patch(self, args):
return self.server.tool.executive.run_command([self.server.tool.path()] + args, cwd=self.server.tool.scm().checkout_root)
+ @memoized
+ def _expectations_updater(self):
+ # FIXME: Should split failure_info_list into lists per port, then edit each expectations file separately.
+ # For now, assume Chromium port.
+ port = factory.get("chromium-win-win7")
+ return GardeningExpectationsUpdater(self.server.tool, port)
+
def rollout(self):
revision = self.query['revision'][0]
reason = self.query['reason'][0]
@@ -82,8 +125,7 @@
self._serve_text('success')
def updateexpectations(self):
- failure_info_list = self._read_entity_body_as_json()
- print 'FailureInfoList:', failure_info_list
+ self._expectations_updater().update_expectations(self._read_entity_body_as_json())
self._serve_text('success')
def rebaseline(self):
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/gardeningserver_unittest.py (92623 => 92624)
--- trunk/Tools/Scripts/webkitpy/tool/servers/gardeningserver_unittest.py 2011-08-08 19:54:46 UTC (rev 92623)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/gardeningserver_unittest.py 2011-08-08 19:59:31 UTC (rev 92624)
@@ -35,15 +35,27 @@
import unittest
from webkitpy.common.system.outputcapture import OutputCapture
+from webkitpy.layout_tests.port import factory
from webkitpy.thirdparty.mock import Mock
from webkitpy.tool.mocktool import MockTool, MockExecutive
from webkitpy.tool.servers.gardeningserver import *
+class TestPortFactory(object):
+ @classmethod
+ def create(cls):
+ return factory.get("test-win-xp")
+
+ @classmethod
+ def path_to_test_expectations_file(cls):
+ return cls.create().path_to_test_expectations_file()
+
+
class MockServer(object):
def __init__(self):
self.tool = MockTool()
self.tool.executive = MockExecutive(should_log=True)
+ self.tool.filesystem.files[TestPortFactory.path_to_test_expectations_file()] = ""
# The real GardeningHTTPRequestHandler has a constructor that's too hard to
@@ -53,6 +65,9 @@
self.server = server
self.body = None
+ def _expectations_updater(self):
+ return GardeningExpectationsUpdater(self.server.tool, TestPortFactory.create())
+
def _read_entity_body(self):
return self.body if self.body else ''
@@ -67,6 +82,66 @@
print "== End JSON Response =="
+class GardeningExpectationsUpdaterTest(unittest.TestCase):
+ def __init__(self, testFunc):
+ self.tool = MockTool()
+ self.tool.executive = MockExecutive(should_log=True)
+ self.tool.filesystem.files[TestPortFactory.path_to_test_expectations_file()] = ""
+ unittest.TestCase.__init__(self, testFunc)
+
+ def assert_update(self, failure_info_list, expectations_before=None, expectations_after=None, expected_exception=None):
+ updater = GardeningExpectationsUpdater(self.tool, TestPortFactory.create())
+ path_to_test_expectations_file = TestPortFactory.path_to_test_expectations_file()
+ self.tool.filesystem.files[path_to_test_expectations_file] = expectations_before or ""
+ if expected_exception:
+ self.assertRaises(expected_exception, updater.update_expectations, (failure_info_list))
+ else:
+ updater.update_expectations(failure_info_list)
+ self.assertEquals(self.tool.filesystem.files[path_to_test_expectations_file], expectations_after)
+
+ def test_empty_expectations(self):
+ failure_info_list = []
+ expectations_before = ""
+ expectations_after = ""
+ self.assert_update(failure_info_list, expectations_before=expectations_before, expectations_after=expectations_after)
+
+ def test_unknown_builder(self):
+ failure_info_list = [{"testName": "failures/expected/image.html", "builderName": "Bob", "failureTypeList": ["IMAGE"]}]
+ self.assert_update(failure_info_list, expected_exception=AssertionError)
+
+ def test_empty_failure_type_list(self):
+ failure_info_list = [{"testName": "failures/expected/image.html", "builderName": "Webkit Win", "failureTypeList": []}]
+ self.assert_update(failure_info_list, expected_exception=AssertionError)
+
+ def test_empty_test_name(self):
+ failure_info_list = [{"testName": "", "builderName": "Webkit Win", "failureTypeList": ["TEXT"]}]
+ self.assert_update(failure_info_list, expected_exception=AssertionError)
+
+ def test_unknown_failure_type(self):
+ failure_info_list = [{"testName": "failures/expected/image.html", "builderName": "Webkit Win", "failureTypeList": ["IMAGE", "EXPLODE"]}]
+ expectations_before = ""
+ expectations_after = "\nBUG_NEW XP RELEASE CPU : failures/expected/image.html = IMAGE"
+ self.assert_update(failure_info_list, expectations_before=expectations_before, expectations_after=expectations_after)
+
+ def test_add_new_expectation(self):
+ failure_info_list = [{"testName": "failures/expected/image.html", "builderName": "Webkit Win", "failureTypeList": ["IMAGE"]}]
+ expectations_before = ""
+ expectations_after = "\nBUG_NEW XP RELEASE CPU : failures/expected/image.html = IMAGE"
+ self.assert_update(failure_info_list, expectations_before=expectations_before, expectations_after=expectations_after)
+
+ def test_replace_old_expectation(self):
+ failure_info_list = [{"testName": "failures/expected/image.html", "builderName": "Webkit Win", "failureTypeList": ["IMAGE"]}]
+ expectations_before = "BUG_OLD XP RELEASE CPU : failures/expected/image.html = TEXT"
+ expectations_after = "BUG_NEW XP RELEASE CPU : failures/expected/image.html = IMAGE"
+ self.assert_update(failure_info_list, expectations_before=expectations_before, expectations_after=expectations_after)
+
+ def test_supplement_old_expectation(self):
+ failure_info_list = [{"testName": "failures/expected/image.html", "builderName": "Webkit Win", "failureTypeList": ["IMAGE"]}]
+ expectations_before = "BUG_OLD XP RELEASE : failures/expected/image.html = TEXT"
+ expectations_after = "BUG_OLD XP RELEASE GPU : failures/expected/image.html = TEXT\nBUG_NEW XP RELEASE CPU : failures/expected/image.html = IMAGE"
+ self.assert_update(failure_info_list, expectations_before=expectations_before, expectations_after=expectations_after)
+
+
class GardeningServerTest(unittest.TestCase):
def _post_to_path(self, path, body=None, expected_stderr=None, expected_stdout=None):
handler = TestGardeningHTTPRequestHandler(MockServer())
@@ -91,5 +166,5 @@
def test_updateexpectations(self):
expected_stderr = ""
- expected_stdout = "FailureInfoList: []\n== Begin Response ==\nsuccess\n== End Response ==\n"
+ expected_stdout = "== Begin Response ==\nsuccess\n== End Response ==\n"
self._post_to_path("/updateexpectations", body="[]", expected_stderr=expected_stderr, expected_stdout=expected_stdout)
