Modified: trunk/Tools/ChangeLog (129050 => 129051)
--- trunk/Tools/ChangeLog 2012-09-19 22:29:05 UTC (rev 129050)
+++ trunk/Tools/ChangeLog 2012-09-19 22:29:45 UTC (rev 129051)
@@ -1,5 +1,59 @@
2012-09-19 Dirk Pranke <dpra...@chromium.org>
+ implement first part of support for the new TestExpectations syntax
+ https://bugs.webkit.org/show_bug.cgi?id=96569
+
+ Reviewed by Ryosuke Niwa.
+
+ This patch implements support for parsing a line of the new
+ format for the TestExpectations file and converting it back into
+ the old format for compatibility. This routine is not yet used
+ by anything.
+
+ The new format is documented at:
+ http://trac.webkit.org/wiki/TestExpectations
+
+ but, in short:
+
+ [bugs] [ "[" modifiers "]" ] test_name [ "[" expectations "]" ]
+
+ - Comments are indicated with "#" instead of "//"
+ - If no expectations are specified we default to Skip for
+ compatibility with the Skipped files (these two changes make
+ Skipped files a subset of TestExpectations files)
+
+ - All of the tokens are now CamelCase instead of ALLCAPS.
+ - FAIL -> Failure
+ - IMAGE -> ImageOnlyFailure
+ - WONTFIX -> WontFix
+ - modifiers refer to just the platforms and configurations
+ (release/debug) that the line applies to.
+ - WontFix, Rebaseline, Slow, and Skip move to the right-hand side as
+ expectations
+ - expectations will typically be written out in lexicographic order
+ - We use webkit.org/b/12345, crbug.com/12345, and Bug(dpranke)
+ instead of BUGWK12345, BUGCR12345, and BUGDPRANKE.
+
+ * Scripts/webkitpy/layout_tests/models/test_expectations.py:
+ (TestExpectationParser):
+ (TestExpectationParser._tokenize_line_using_new_format):
+ * Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py:
+ (NewExpectationSyntaxTests):
+ (NewExpectationSyntaxTests.assert_exp):
+ (NewExpectationSyntaxTests.test_bare_name):
+ (NewExpectationSyntaxTests.test_bare_name_and_bugs):
+ (NewExpectationSyntaxTests.test_comments):
+ (NewExpectationSyntaxTests.test_config_modifiers):
+ (NewExpectationSyntaxTests.test_unknown_config):
+ (NewExpectationSyntaxTests.test_unknown_expectation):
+ (NewExpectationSyntaxTests.test_skip):
+ (NewExpectationSyntaxTests.test_slow):
+ (NewExpectationSyntaxTests.test_wontfix):
+ (NewExpectationSyntaxTests.test_blank_line):
+ (NewExpectationSyntaxTests.test_warnings):
+
+2012-09-19 Dirk Pranke <dpra...@chromium.org>
+
nrwt: replace TEXT, AUDIO, and IMAGE+TEXT with FAIL
https://bugs.webkit.org/show_bug.cgi?id=96845
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py (129050 => 129051)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py 2012-09-19 22:29:05 UTC (rev 129050)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py 2012-09-19 22:29:45 UTC (rev 129051)
@@ -254,11 +254,147 @@
return expectation_line
+ # FIXME: Update the original modifiers and remove this once the old syntax is gone.
+ _configuration_tokens_list = [
+ 'Mac', 'SnowLeopard', 'Lion', 'MountainLion',
+ 'Win', 'XP', 'Vista', 'Win7',
+ 'Linux',
+ 'Android',
+ 'Release',
+ 'Debug',
+ ]
+
+ _configuration_tokens = dict((token, token.upper()) for token in _configuration_tokens_list)
+
+ # Note: we can't distinguish audio failures or image+text failures from text-only failures.
+ # FIXME: Update the original modifiers list and remove this once the old syntax is gone.
+ _expectation_tokens = {
+ 'WontFix': 'WONTFIX',
+ 'Pass': 'PASS',
+ 'Failure': 'FAIL',
+ 'ImageOnlyFailure': 'IMAGE',
+ 'Crash': 'CRASH',
+ 'Timeout': 'TIMEOUT',
+ 'Slow': 'SLOW',
+ }
+
@classmethod
def _tokenize_line_using_new_format(cls, filename, expectation_string, line_number):
- # FIXME: implement :).
- raise NotImplementedError
+ """Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance using the old format.
+ The new format for a test expectation line is:
+
+ [[bugs] [ "[" <configuration modifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>]
+
+ Any errant whitespace is not preserved.
+
+ """
+ expectation_line = TestExpectationLine()
+ expectation_line.filename = filename
+ expectation_line.line_number = line_number
+
+ comment_index = expectation_string.find("#")
+ if comment_index == -1:
+ comment_index = len(expectation_string)
+ else:
+ expectation_line.comment = expectation_string[comment_index + 1:]
+
+ remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip())
+ if len(remaining_string) == 0:
+ return expectation_line
+
+ # special-case parsing this so that we fail immediately instead of treating this as a test name
+ if remaining_string.startswith('//'):
+ expectation_line.warnings = ['use "#" instead of "//" for comments']
+ return expectation_line
+
+ bugs = []
+ modifiers = []
+ name = None
+ expectations = []
+ warnings = []
+
+ WEBKIT_BUG_PREFIX = 'webkit.org/b/'
+ CHROMIUM_BUG_PREFIX = 'crbug.com/'
+ V8_BUG_PREFIX = 'code.google.com/p/v8/issues/detail?id='
+
+ tokens = remaining_string.split()
+ state = 'start'
+ for token in tokens:
+ if (token.startswith(WEBKIT_BUG_PREFIX) or
+ token.startswith(CHROMIUM_BUG_PREFIX) or
+ token.startswith(V8_BUG_PREFIX) or
+ token.startswith('Bug(')):
+ if state != 'start':
+ warnings.append('"%s" is not at the start of the line.' % token)
+ break
+ if token.startswith(WEBKIT_BUG_PREFIX):
+ bugs.append(token.replace(WEBKIT_BUG_PREFIX, 'BUGWK'))
+ elif token.startswith(CHROMIUM_BUG_PREFIX):
+ bugs.append(token.replace(CHROMIUM_BUG_PREFIX, 'BUGCR'))
+ elif token.startswith(V8_BUG_PREFIX):
+ bugs.append(token.replace(V8_BUG_PREFIX, 'BUGV8_'))
+ else:
+ match = re.match('Bug\((\w+)\)$', token)
+ if not match:
+ warnings.append('unrecognized bug identifier "%s"' % token)
+ break
+ else:
+ bugs.append('BUG' + match.group(1).upper())
+ elif token.startswith('BUG'):
+ warnings.append('unrecognized old-style bug identifier "%s"' % token)
+ break
+ elif token == '[':
+ if state == 'start':
+ state = 'configuration'
+ elif state == 'name_found':
+ state = 'expectations'
+ else:
+ warnings.append('unexpected "["')
+ break
+ elif token == ']':
+ if state == 'configuration':
+ state = 'name'
+ elif state == 'expectations':
+ state = 'done'
+ else:
+ warnings.append('unexpected "]"')
+ break
+ elif token in ('//', ':', '='):
+ warnings.append('"%s" is not legal in the new TestExpectations syntax.' % token)
+ break
+ elif state == 'configuration':
+ modifiers.append(cls._configuration_tokens.get(token, token))
+ elif state == 'expectations':
+ if token in ('Rebaseline', 'Skip', 'Slow', 'WontFix'):
+ modifiers.append(token.upper())
+ else:
+ expectations.append(cls._expectation_tokens.get(token, token))
+ elif state == 'name_found':
+ warnings.append('expecting "[", "#", or end of line instead of "%s"' % token)
+ break
+ else:
+ name = token
+ state = 'name_found'
+
+ if not warnings:
+ if not name:
+ warnings.append('Did not find a test name.')
+ elif state not in ('name_found', 'done'):
+ warnings.append('Missing a "]"')
+
+ if not expectations:
+ if 'SKIP' not in modifiers and 'REBASELINE' not in modifiers and 'SLOW' not in modifiers:
+ modifiers.append('SKIP')
+ expectations = ['PASS']
+
+ # FIXME: expectation line should just store bugs and modifiers separately.
+ expectation_line.modifiers = bugs + modifiers
+ expectation_line.expectations = expectations
+ expectation_line.name = name
+ expectation_line.warnings = warnings
+ return expectation_line
+
@classmethod
def _split_space_separated(cls, space_separated_string):
"""Splits a space-separated string into an array."""
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py (129050 => 129051)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py 2012-09-19 22:29:05 UTC (rev 129050)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py 2012-09-19 22:29:45 UTC (rev 129051)
@@ -340,6 +340,68 @@
self.assert_exp('failures/expected/text.html', FAIL)
+class NewExpectationSyntaxTests(unittest.TestCase):
+ def assert_exp(self, line, bugs=None, modifiers=None, expectations=None, warnings=None, comment=None, name='foo.html'):
+ bugs = bugs or []
+ modifiers = modifiers or []
+ expectations = expectations or []
+ warnings = warnings or []
+ filename = 'TestExpectations'
+ line_number = 1
+ expectation_line = TestExpectationParser._tokenize_line_using_new_format(filename, line, line_number)
+ self.assertEquals(expectation_line.warnings, warnings)
+ self.assertEquals(expectation_line.name, name)
+ self.assertEquals(expectation_line.filename, filename)
+ self.assertEquals(expectation_line.line_number, line_number)
+ if not warnings:
+ self.assertEquals(expectation_line.modifiers, modifiers)
+ self.assertEquals(expectation_line.expectations, expectations)
+
+ def test_bare_name(self):
+ self.assert_exp('foo.html', modifiers=['SKIP'], expectations=['PASS'])
+
+ def test_bare_name_and_bugs(self):
+ self.assert_exp('webkit.org/b/12345 foo.html', modifiers=['BUGWK12345', 'SKIP'], expectations=['PASS'])
+ self.assert_exp('crbug.com/12345 foo.html', modifiers=['BUGCR12345', 'SKIP'], expectations=['PASS'])
+ self.assert_exp('Bug(dpranke) foo.html', modifiers=['BUGDPRANKE', 'SKIP'], expectations=['PASS'])
+ self.assert_exp('crbug.com/12345 crbug.com/34567 foo.html', modifiers=['BUGCR12345', 'BUGCR34567', 'SKIP'], expectations=['PASS'])
+
+ def test_comments(self):
+ self.assert_exp("# comment", name=None, comment="# comment")
+ self.assert_exp("foo.html # comment", comment="# comment", expectations=['PASS'], modifiers=['SKIP'])
+
+ def test_config_modifiers(self):
+ self.assert_exp('[ Mac ] foo.html', modifiers=['MAC', 'SKIP'], expectations=['PASS'])
+ self.assert_exp('[ Mac Vista ] foo.html', modifiers=['MAC', 'VISTA', 'SKIP'], expectations=['PASS'])
+ self.assert_exp('[ Mac ] foo.html [ Failure ] ', modifiers=['MAC'], expectations=['FAIL'])
+
+ def test_unknown_config(self):
+ self.assert_exp('[ Foo ] foo.html ', modifiers=['Foo', 'SKIP'], expectations=['PASS'])
+
+ def test_unknown_expectation(self):
+ self.assert_exp('foo.html [ Audio ]', expectations=['Audio'])
+
+ def test_skip(self):
+ self.assert_exp('foo.html [ Skip ]', modifiers=['SKIP'], expectations=['PASS'])
+
+ def test_slow(self):
+ self.assert_exp('foo.html [ Slow ]', modifiers=['SLOW'], expectations=['PASS'])
+
+ def test_wontfix(self):
+ self.assert_exp('foo.html [ WontFix ]', modifiers=['WONTFIX', 'SKIP'], expectations=['PASS'])
+
+ def test_blank_line(self):
+ self.assert_exp('', name=None)
+
+ def test_warnings(self):
+ self.assert_exp('[ Mac ]', warnings=['Did not find a test name.'], name=None)
+
+ self.assert_exp('[ [', warnings=['unexpected "["'], name=None)
+ self.assert_exp('crbug.com/12345 ]', warnings=['unexpected "]"'], name=None)
+
+ self.assert_exp('foo.html crbug.com/12345 ]', warnings=['"crbug.com/12345" is not at the start of the line.'])
+
+
class SemanticTests(Base):
def test_bug_format(self):
self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = FAIL', is_lint_mode=True)
