Diff
Added: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/factories.py (0 => 225334)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/factories.py (rev 0)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/factories.py 2017-11-30 19:14:04 UTC (rev 225334)
@@ -0,0 +1,193 @@
+# Copyright (C) 2017 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from buildbot.process import factory
+from buildbot.steps import trigger
+
+from steps import *
+
+
+class Factory(factory.BuildFactory):
+ def __init__(self, platform, configuration, architectures, buildOnly, additionalArguments, SVNMirror):
+ factory.BuildFactory.__init__(self)
+ self.addStep(ConfigureBuild(platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly, additionalArguments=additionalArguments, SVNMirror=SVNMirror))
+ if SVNMirror:
+ self.addStep(WaitForSVNServer())
+ self.addStep(CheckOutSource(SVNMirror=SVNMirror))
+ if not (platform == "jsc-only"):
+ self.addStep(KillOldProcesses())
+ self.addStep(CleanBuildIfScheduled())
+ self.addStep(DeleteStaleBuildFiles())
+ if platform == "win":
+ self.addStep(InstallWin32Dependencies())
+ if platform == "gtk" and additionalArguments != ["--no-experimental-features"]:
+ self.addStep(InstallGtkDependencies())
+ if platform == "wpe":
+ self.addStep(InstallWpeDependencies())
+
+
+class BuildFactory(Factory):
+ def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None):
+ Factory.__init__(self, platform, configuration, architectures, True, additionalArguments, SVNMirror)
+
+ if platform == "win":
+ self.addStep(CompileWebKit(timeout=2 * 60 * 60))
+ else:
+ self.addStep(CompileWebKit())
+
+ if triggers:
+ self.addStep(ArchiveBuiltProduct())
+ self.addStep(UploadBuiltProduct())
+ if platform.startswith('mac') or platform.startswith('ios-simulator'):
+ self.addStep(ArchiveMinifiedBuiltProduct())
+ self.addStep(UploadMinifiedBuiltProduct())
+ self.addStep(TransferToS3())
+ self.addStep(trigger.Trigger(schedulerNames=triggers))
+
+
+class TestFactory(Factory):
+ JSCTestClass = RunJavaScriptCoreTests
+ LayoutTestClass = RunWebKitTests
+
+ def getProduct(self):
+ self.addStep(DownloadBuiltProduct())
+ self.addStep(ExtractBuiltProduct())
+
+ def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
+ Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
+ self.getProduct()
+ if self.JSCTestClass:
+ self.addStep(self.JSCTestClass())
+ if self.LayoutTestClass:
+ self.addStep(self.LayoutTestClass())
+
+ if platform == 'win' or platform.startswith('mac') or platform.startswith('ios-simulator'):
+ self.addStep(RunUnitTests())
+ self.addStep(RunPythonTests())
+ self.addStep(RunPerlTests())
+ self.addStep(RunBindingsTests())
+ self.addStep(RunBuiltinsTests())
+ if platform != 'win':
+ self.addStep(RunDashboardTests())
+ if self.LayoutTestClass:
+ self.addStep(ArchiveTestResults())
+ self.addStep(UploadTestResults())
+ self.addStep(ExtractTestResults())
+ if platform == "gtk":
+ self.addStep(RunGtkAPITests())
+ if platform == "wpe":
+ self.addStep(RunWPEAPITests())
+
+
+class BuildAndTestFactory(TestFactory):
+ def getProduct(self):
+ self.addStep(CompileWebKit())
+
+ def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
+ TestFactory.__init__(self, platform, configuration, architectures, additionalArguments, SVNMirror, **kwargs)
+ if triggers:
+ self.addStep(ArchiveBuiltProduct())
+ self.addStep(UploadBuiltProduct())
+ self.addStep(trigger.Trigger(schedulerNames=triggers))
+
+
+class BuildAndTestLLINTCLoopFactory(Factory):
+ def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
+ Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
+ self.addStep(CompileLLINTCLoop())
+ self.addStep(RunLLINTCLoopTests())
+
+
+class BuildAndTest32bitJSCFactory(Factory):
+ def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
+ Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
+ self.addStep(Compile32bitJSC())
+ self.addStep(Run32bitJSCTests())
+
+
+class BuildAndNonLayoutTestFactory(BuildAndTestFactory):
+ LayoutTestClass = None
+
+
+class BuildAndRemoteJSCTestsFactory(Factory):
+ def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None):
+ Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
+ self.addStep(CompileJSCOnly(timeout=60 * 60))
+ self.addStep(RunRemoteJavaScriptCoreTests(timeout=60 * 60))
+
+
+class TestWebKit1LeaksFactory(Factory):
+ def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None):
+ Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
+ self.addStep(DownloadBuiltProduct())
+ self.addStep(ExtractBuiltProduct())
+ self.addStep(RunWebKit1LeakTests())
+ self.addStep(ArchiveTestResults())
+ self.addStep(UploadTestResults())
+ self.addStep(ExtractTestResultsAndLeaks())
+
+
+class TestAllButJSCFactory(TestFactory):
+ JSCTestClass = None
+
+
+class TestJSCFactory(Factory):
+ def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None):
+ Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
+ self.addStep(DownloadBuiltProduct())
+ self.addStep(ExtractBuiltProduct())
+ self.addStep(RunJavaScriptCoreTests())
+
+
+class Test262Factory(Factory):
+ def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None):
+ Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
+ self.addStep(DownloadBuiltProduct())
+ self.addStep(ExtractBuiltProduct())
+ self.addStep(RunTest262Tests())
+
+
+class TestWebKit1Factory(TestFactory):
+ LayoutTestClass = RunWebKit1Tests
+
+
+class TestWebKit1AllButJSCFactory(TestWebKit1Factory):
+ JSCTestClass = None
+
+
+class BuildAndPerfTestFactory(Factory):
+ def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
+ Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
+ self.addStep(CompileWebKit())
+ self.addStep(RunAndUploadPerfTests())
+ if platform == "gtk":
+ self.addStep(RunBenchmarkTests())
+
+
+class DownloadAndPerfTestFactory(Factory):
+ def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
+ Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
+ self.addStep(DownloadBuiltProduct())
+ self.addStep(ExtractBuiltProduct())
+ self.addStep(RunAndUploadPerfTests())
+ if platform == "gtk":
+ self.addStep(RunBenchmarkTests())
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg (225333 => 225334)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg 2017-11-30 19:12:01 UTC (rev 225333)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg 2017-11-30 19:14:04 UTC (rev 225334)
@@ -9,17 +9,13 @@
from buildbot.status import html
from buildbot.status.web.authz import Authz
from buildbot.process import buildstep, factory, properties
-from buildbot.steps import master, shell, source, transfer, trigger
-from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION
+from factories import *
from twisted.internet import defer
-import os
import re
import json
import operator
-import cStringIO
-import urllib
from committer_auth import CommitterAuth
import wkbuild
@@ -60,981 +56,11 @@
c['eventHorizon'] = 200
c['buildCacheSize'] = 60
-WithProperties = properties.WithProperties
-
-class TestWithFailureCount(shell.Test):
- failedTestsFormatString = "%d test%s failed"
-
- def countFailures(self, cmd):
- return 0
-
- def commandComplete(self, cmd):
- shell.Test.commandComplete(self, cmd)
- self.failedTestCount = self.countFailures(cmd)
- self.failedTestPluralSuffix = "" if self.failedTestCount == 1 else "s"
-
- def evaluateCommand(self, cmd):
- if self.failedTestCount:
- return FAILURE
-
- if cmd.rc != 0:
- return FAILURE
-
- return SUCCESS
-
- def getText(self, cmd, results):
- return self.getText2(cmd, results)
-
- def getText2(self, cmd, results):
- if results != SUCCESS and self.failedTestCount:
- return [self.failedTestsFormatString % (self.failedTestCount, self.failedTestPluralSuffix)]
-
- return [self.name]
-
-
-class ConfigureBuild(buildstep.BuildStep):
- name = "configure build"
- description = ["configuring build"]
- descriptionDone = ["configured build"]
- def __init__(self, platform, configuration, architecture, buildOnly, additionalArguments, SVNMirror, *args, **kwargs):
- buildstep.BuildStep.__init__(self, *args, **kwargs)
- self.platform = platform
- if platform != 'jsc-only':
- self.platform = platform.split('-', 1)[0]
- self.fullPlatform = platform
- self.configuration = configuration
- self.architecture = architecture
- self.buildOnly = buildOnly
- self.additionalArguments = additionalArguments
- self.SVNMirror = SVNMirror
- self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, additionalArguments=additionalArguments, SVNMirror=SVNMirror)
-
- def start(self):
- self.setProperty("platform", self.platform)
- self.setProperty("fullPlatform", self.fullPlatform)
- self.setProperty("configuration", self.configuration)
- self.setProperty("architecture", self.architecture)
- self.setProperty("buildOnly", self.buildOnly)
- self.setProperty("additionalArguments", self.additionalArguments)
- self.setProperty("SVNMirror", self.SVNMirror)
- self.finished(SUCCESS)
- return defer.succeed(None)
-
-
-class CheckOutSource(source.SVN):
- mode = "update"
- def __init__(self, SVNMirror, **kwargs):
- kwargs['baseURL'] = SVNMirror or "https://svn.webkit.org/repository/webkit/"
- kwargs['defaultBranch'] = "trunk"
- kwargs['mode'] = self.mode
- source.SVN.__init__(self, **kwargs)
- self.addFactoryArguments(SVNMirror=SVNMirror)
-
-
-class WaitForSVNServer(shell.ShellCommand):
- name = "wait-for-svn-server"
- command = ["python", "./Tools/BuildSlaveSupport/wait-for-SVN-server.py", "-r", WithProperties("%(revision)s"), "-s", WithProperties("%(SVNMirror)s")]
- description = ["waiting for SVN server"]
- descriptionDone = ["SVN server is ready"]
- warnOnFailure = True
-
- def evaluateCommand(self, cmd):
- if cmd.rc != 0:
- return WARNINGS
- return SUCCESS
-
-
-class InstallWin32Dependencies(shell.Compile):
- description = ["installing dependencies"]
- descriptionDone = ["installed dependencies"]
- command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"]
-
-
-class KillOldProcesses(shell.Compile):
- name = "kill old processes"
- description = ["killing old processes"]
- descriptionDone = ["killed old processes"]
- command = ["python", "./Tools/BuildSlaveSupport/kill-old-processes", "buildbot"]
-
-
-class CleanBuildIfScheduled(shell.Compile):
- name = "delete WebKitBuild directory"
- description = ["deleting WebKitBuild directory"]
- descriptionDone = ["deleted WebKitBuild directory"]
- command = ["python", "./Tools/BuildSlaveSupport/clean-build", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
-
- def start(self):
- if not self.getProperty('is_clean'):
- self.hideStepIf = True
- return SKIPPED
- return shell.Compile.start(self)
-
-
-class DeleteStaleBuildFiles(shell.Compile):
- name = "delete stale build files"
- description = ["deleting stale build files"]
- descriptionDone = ["deleted stale build files"]
- command = ["python", "./Tools/BuildSlaveSupport/delete-stale-build-files", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
-
- def start(self):
- if self.getProperty('is_clean'): # Nothing to be done if WebKitBuild had been removed.
- self.hideStepIf = True
- return SKIPPED
- return shell.Compile.start(self)
-
-
-class InstallGtkDependencies(shell.ShellCommand):
- name = "jhbuild"
- description = ["updating gtk dependencies"]
- descriptionDone = ["updated gtk dependencies"]
- command = ["perl", "./Tools/Scripts/update-webkitgtk-libs"]
- haltOnFailure = True
-
-
-class InstallWpeDependencies(shell.ShellCommand):
- name = "jhbuild"
- description = ["updating wpe dependencies"]
- descriptionDone = ["updated wpe dependencies"]
- command = ["perl", "./Tools/Scripts/update-webkitwpe-libs"]
- haltOnFailure = True
-
-def appendCustomBuildFlags(step, platform, fullPlatform):
- if platform not in ('gtk', 'wincairo', 'ios', 'jsc-only', 'wpe'):
- return
- if fullPlatform.startswith('ios-simulator'):
- platform = 'ios-simulator'
- elif platform == 'ios':
- platform = 'device'
- step.setCommand(step.command + ['--' + platform])
-
-
-class CompileWebKit(shell.Compile):
- command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
- env = {'MFLAGS':''}
- name = "compile-webkit"
- description = ["compiling"]
- descriptionDone = ["compiled"]
- warningPattern = ".*arning: .*"
-
- def start(self):
- platform = self.getProperty('platform')
- buildOnly = self.getProperty('buildOnly')
- architecture = self.getProperty('architecture')
- additionalArguments = self.getProperty('additionalArguments')
-
- if additionalArguments:
- self.setCommand(self.command + additionalArguments)
- if platform in ('mac', 'ios') and architecture:
- self.setCommand(self.command + ['ARCHS=' + architecture])
- if platform == 'ios':
- self.setCommand(self.command + ['_ONLY_ACTIVE_ARCH_=NO'])
- # Generating dSYM files is slow, but these are needed to have line numbers in crash reports on testers.
- # Debug builds on Yosemite can't use dSYMs, because crash logs end up unsymbolicated.
- if platform in ('mac', 'ios') and buildOnly and (self.getProperty('fullPlatform') != "mac-yosemite" or self.getProperty('configuration') != "debug"):
- self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
-
- appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
-
- return shell.Compile.start(self)
-
- def createSummary(self, log):
- platform = self.getProperty('platform')
- if platform.startswith('mac'):
- warnings = []
- errors = []
- sio = cStringIO.StringIO(log.getText())
- for line in sio.readlines():
- if "arning:" in line:
- warnings.append(line)
- if "rror:" in line:
- errors.append(line)
- if warnings:
- self.addCompleteLog('warnings', "".join(warnings))
- if errors:
- self.addCompleteLog('errors', "".join(errors))
-
-
-class CompileLLINTCLoop(CompileWebKit):
- command = ["perl", "./Tools/Scripts/build-jsc", "--cloop", WithProperties("--%(configuration)s")]
-
-
-class Compile32bitJSC(CompileWebKit):
- command = ["perl", "./Tools/Scripts/build-jsc", "--32-bit", WithProperties("--%(configuration)s")]
-
-
-class CompileJSCOnly(CompileWebKit):
- command = ["perl", "./Tools/Scripts/build-jsc", WithProperties("--%(configuration)s")]
-
-
-class ArchiveBuiltProduct(shell.ShellCommand):
- command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
- WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive"]
- name = "archive-built-product"
- description = ["archiving built product"]
- descriptionDone = ["archived built product"]
- haltOnFailure = True
-
-
-class ArchiveMinifiedBuiltProduct(ArchiveBuiltProduct):
- command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
- WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive", "--minify"]
-
-
-class ExtractBuiltProduct(shell.ShellCommand):
- command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
- WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "extract"]
- name = "extract-built-product"
- description = ["extracting built product"]
- descriptionDone = ["extracted built product"]
- haltOnFailure = True
-
-
-class UploadBuiltProduct(transfer.FileUpload):
- slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
- masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
- haltOnFailure = True
-
- def __init__(self, **kwargs):
- kwargs['slavesrc'] = self.slavesrc
- kwargs['masterdest'] = self.masterdest
- kwargs['mode'] = 0644
- kwargs['blocksize'] = 1024*256
- transfer.FileUpload.__init__(self, **kwargs)
-
-
-class UploadMinifiedBuiltProduct(UploadBuiltProduct):
- slavesrc = WithProperties("WebKitBuild/minified-%(configuration)s.zip")
- masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/minified-%(got_revision)s.zip")
-
-
-class DownloadBuiltProduct(shell.ShellCommand):
- command = ["python", "./Tools/BuildSlaveSupport/download-built-product",
- WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"),
- WithProperties(S3URL + "archives.webkit.org/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")]
- name = "download-built-product"
- description = ["downloading built product"]
- descriptionDone = ["downloaded built product"]
- haltOnFailure = True
- flunkOnFailure = True
-
- def start(self):
- if 'apple' in self.getProperty('buildername').lower():
- self.slaveEnvironment['HTTPS_PROXY'] = APPLE_WEBKIT_AWS_PROXY # curl env var to use a proxy
- return shell.ShellCommand.start(self)
-
-
-class RunJavaScriptCoreTests(TestWithFailureCount):
- name = "jscore-test"
- description = ["jscore-tests running"]
- descriptionDone = ["jscore-tests"]
- jsonFileName = "jsc_results.json"
- command = ["perl", "./Tools/Scripts/run-_javascript_core-tests", "--no-build", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
- failedTestsFormatString = "%d JSC test%s failed"
- logfiles = {"json": jsonFileName}
-
- def start(self):
- platform = self.getProperty('platform')
- # Linux bots have currently problems with JSC tests that try to use large amounts of memory.
- # Check: https://bugs.webkit.org/show_bug.cgi?id=175140
- if platform in ('gtk', 'wpe'):
- self.setCommand(self.command + ['--memory-limited'])
- appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
- return shell.Test.start(self)
-
- def countFailures(self, cmd):
- logText = cmd.logs['stdio'].getText()
-
- match = re.search(r'^Results for JSC stress tests:\r?\n\s+(\d+) failure', logText, re.MULTILINE)
- if match:
- return int(match.group(1))
-
- match = re.search(r'^Results for Mozilla tests:\r?\n\s+(\d+) regression', logText, re.MULTILINE)
- if match:
- return int(match.group(1))
-
- return 0
-
-
-class RunRemoteJavaScriptCoreTests(RunJavaScriptCoreTests):
- def start(self):
- self.setCommand(self.command + ["--memory-limited", "--remote-config-file", "../../remote-jsc-tests-config.json"])
- return RunJavaScriptCoreTests.start(self)
-
-
-class RunTest262Tests(TestWithFailureCount):
- name = "test262-test"
- description = ["test262-tests running"]
- descriptionDone = ["test262-tests"]
- failedTestsFormatString = "%d Test262 test%s failed"
- command = ["perl", "./Tools/Scripts/run-jsc-stress-tests", WithProperties("--%(configuration)s"), "JSTests/test262.yaml"]
-
- def start(self):
- appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
- return shell.Test.start(self)
-
- def countFailures(self, cmd):
- logText = cmd.logs['stdio'].getText()
- matches = re.findall(r'^FAIL:', logText, flags=re.MULTILINE)
- if matches:
- return len(matches)
- return 0
-
-
-class RunWebKitTests(shell.Test):
- name = "layout-test"
- description = ["layout-tests running"]
- descriptionDone = ["layout-tests"]
- resultDirectory = "layout-test-results"
- command = ["python", "./Tools/Scripts/run-webkit-tests",
- "--no-build",
- "--no-show-results",
- "--no-new-test-results",
- "--builder-name", WithProperties("%(buildername)s"),
- "--build-number", WithProperties("%(buildnumber)s"),
- "--master-name", "webkit.org",
- "--test-results-server", "webkit-test-results.webkit.org",
- "--exit-after-n-crashes-or-timeouts", "50",
- "--exit-after-n-failures", "500",
- WithProperties("--%(configuration)s")]
-
- def start(self):
- platform = self.getProperty('platform')
- appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
- additionalArguments = self.getProperty('additionalArguments')
-
- self.setCommand(self.command + ["--results-directory", self.resultDirectory])
- self.setCommand(self.command + ['--debug-rwt-logging'])
-
- if platform == "win":
- self.setCommand(self.command + ['--batch-size', '100', '--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin32")])
-
- if additionalArguments:
- self.setCommand(self.command + additionalArguments)
- return shell.Test.start(self)
-
- # FIXME: This will break if run-webkit-tests changes its default log formatter.
- nrwt_log_message_regexp = re.compile(r'\d{2}:\d{2}:\d{2}(\.\d+)?\s+\d+\s+(?P<message>.*)')
-
- def _strip_python_logging_prefix(self, line):
- match_object = self.nrwt_log_message_regexp.match(line)
- if match_object:
- return match_object.group('message')
- return line
-
- def _parseRunWebKitTestsOutput(self, logText):
- incorrectLayoutLines = []
- expressions = [
- ('flakes', re.compile(r'Unexpected flakiness.+\((\d+)\)')),
- ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
- ('missing results', re.compile(r'Regressions: Unexpected missing results\s+\((\d+)\)')),
- ('failures', re.compile(r'Regressions: Unexpected.+\((\d+)\)')),
- ]
- testFailures = {}
-
- for line in logText.splitlines():
- if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
- incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
- continue
- for name, _expression_ in expressions:
- match = _expression_.search(line)
-
- if match:
- testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
- break
-
- # FIXME: Parse file names and put them in results
-
- for name in testFailures:
- incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)
-
- self.incorrectLayoutLines = incorrectLayoutLines
-
- def commandComplete(self, cmd):
- shell.Test.commandComplete(self, cmd)
-
- logText = cmd.logs['stdio'].getText()
- self._parseRunWebKitTestsOutput(logText)
-
- def evaluateCommand(self, cmd):
- result = SUCCESS
-
- if self.incorrectLayoutLines:
- if len(self.incorrectLayoutLines) == 1:
- line = self.incorrectLayoutLines[0]
- if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
- return WARNINGS
-
- for line in self.incorrectLayoutLines:
- if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
- result = WARNINGS
- else:
- return FAILURE
-
- # Return code from Tools/Scripts/layout_tests/run_webkit_tests.py.
- # This means that an exception was raised when running run-webkit-tests and
- # was never handled.
- if cmd.rc == 254:
- return EXCEPTION
- if cmd.rc != 0:
- return FAILURE
-
- return result
-
- def getText(self, cmd, results):
- return self.getText2(cmd, results)
-
- def getText2(self, cmd, results):
- if results != SUCCESS and self.incorrectLayoutLines:
- return self.incorrectLayoutLines
-
- return [self.name]
-
-
-class RunDashboardTests(RunWebKitTests):
- name = "dashboard-tests"
- description = ["dashboard-tests running"]
- descriptionDone = ["dashboard-tests"]
- resultDirectory = os.path.join(RunWebKitTests.resultDirectory, "dashboard-layout-test-results")
-
- def start(self):
- self.setCommand(self.command + ["--layout-tests-directory", "./Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/tests"])
- return RunWebKitTests.start(self)
-
-
-class RunUnitTests(TestWithFailureCount):
- name = "run-api-tests"
- description = ["unit tests running"]
- descriptionDone = ["unit-tests"]
- command = ["perl", "./Tools/Scripts/run-api-tests", "--no-build", WithProperties("--%(configuration)s"), "--verbose"]
- failedTestsFormatString = "%d unit test%s failed or timed out"
-
- def start(self):
- appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
- return shell.Test.start(self)
-
- def countFailures(self, cmd):
- log_text = cmd.logs['stdio'].getText()
- count = 0
-
- split = re.split(r'\sTests that timed out:\s', log_text)
- if len(split) > 1:
- count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
-
- split = re.split(r'\sTests that failed:\s', split[0])
- if len(split) > 1:
- count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
-
- return count
-
-
-class RunPythonTests(TestWithFailureCount):
- name = "webkitpy-test"
- description = ["python-tests running"]
- descriptionDone = ["python-tests"]
- command = ["python", "./Tools/Scripts/test-webkitpy", "--verbose"]
- failedTestsFormatString = "%d python test%s failed"
-
- def start(self):
- platform = self.getProperty('platform')
- # Python tests are flaky on the GTK builders, running them serially
- # helps and does not significantly prolong the cycle time.
- if platform == 'gtk':
- self.setCommand(self.command + ['--child-processes', '1'])
- # Python tests fail on windows bots when running more than one child process
- # https://bugs.webkit.org/show_bug.cgi?id=97465
- if platform == 'win':
- self.setCommand(self.command + ['--child-processes', '1'])
- return shell.Test.start(self)
-
- def countFailures(self, cmd):
- logText = cmd.logs['stdio'].getText()
- # We're looking for the line that looks like this: FAILED (failures=2, errors=1)
- regex = re.compile(r'^FAILED \((?P<counts>[^)]+)\)')
- for line in logText.splitlines():
- match = regex.match(line)
- if not match:
- continue
- return sum(int(component.split('=')[1]) for component in match.group('counts').split(', '))
- return 0
-
-
-class RunPerlTests(TestWithFailureCount):
- name = "webkitperl-test"
- description = ["perl-tests running"]
- descriptionDone = ["perl-tests"]
- command = ["perl", "./Tools/Scripts/test-webkitperl"]
- failedTestsFormatString = "%d perl test%s failed"
-
- def countFailures(self, cmd):
- logText = cmd.logs['stdio'].getText()
- # We're looking for the line that looks like this: Failed 2/19 test programs. 5/363 subtests failed.
- regex = re.compile(r'^Failed \d+/\d+ test programs\. (?P<count>\d+)/\d+ subtests failed\.')
- for line in logText.splitlines():
- match = regex.match(line)
- if not match:
- continue
- return int(match.group('count'))
- return 0
-
-
-class RunLLINTCLoopTests(TestWithFailureCount):
- name = "webkit-jsc-cloop-test"
- description = ["cloop-tests running"]
- descriptionDone = ["cloop-tests"]
- jsonFileName = "jsc_cloop.json"
- command = ["perl", "./Tools/Scripts/run-_javascript_core-tests", "--cloop", "--no-build", "--no-jsc-stress", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
- failedTestsFormatString = "%d regression%s found."
- logfiles = {"json": jsonFileName}
-
- def countFailures(self, cmd):
- logText = cmd.logs['stdio'].getText()
- # We're looking for the line that looks like this: 0 regressions found.
- regex = re.compile(r'\s*(?P<count>\d+) regressions? found.')
- for line in logText.splitlines():
- match = regex.match(line)
- if not match:
- continue
- return int(match.group('count'))
- return 0
-
-
-class Run32bitJSCTests(TestWithFailureCount):
- name = "webkit-32bit-jsc-test"
- description = ["32bit-jsc-tests running"]
- descriptionDone = ["32bit-jsc-tests"]
- jsonFileName = "jsc_32bit.json"
- command = ["perl", "./Tools/Scripts/run-_javascript_core-tests", "--32-bit", "--no-build", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
- failedTestsFormatString = "%d regression%s found."
- logfiles = {"json": jsonFileName}
-
- def countFailures(self, cmd):
- logText = cmd.logs['stdio'].getText()
- # We're looking for the line that looks like this: 0 failures found.
- regex = re.compile(r'\s*(?P<count>\d+) failures? found.')
- for line in logText.splitlines():
- match = regex.match(line)
- if not match:
- continue
- return int(match.group('count'))
- return 0
-
-
-class RunBindingsTests(shell.Test):
- name = "bindings-generation-tests"
- description = ["bindings-tests running"]
- descriptionDone = ["bindings-tests"]
- command = ["python", "./Tools/Scripts/run-bindings-tests"]
-
-
-class RunBuiltinsTests(shell.Test):
- name = "builtins-generator-tests"
- description = ["builtins-generator-tests running"]
- descriptionDone = ["builtins-generator-tests"]
- command = ["python", "./Tools/Scripts/run-builtins-generator-tests"]
-
-
-class RunGLibAPITests(shell.Test):
- name = "API tests"
- description = ["API tests running"]
- descriptionDone = ["API tests"]
-
- def start(self):
- additionalArguments = self.getProperty("additionalArguments")
- if additionalArguments:
- self.command += additionalArguments
- self.setCommand(self.command)
- return shell.Test.start(self)
-
- def commandComplete(self, cmd):
- shell.Test.commandComplete(self, cmd)
-
- logText = cmd.logs['stdio'].getText()
-
- self.incorrectTests = 0
- self.crashedTests = 0
- self.timedOutTests = 0
- self.skippedTests = 0
- self.statusLine = []
-
- foundItems = re.findall("Unexpected failures \((\d+)\)", logText)
- if (foundItems):
- self.incorrectTests = int(foundItems[0])
-
- foundItems = re.findall("Unexpected crashes \((\d+)\)", logText)
- if (foundItems):
- self.crashedTests = int(foundItems[0])
-
- foundItems = re.findall("Unexpected timeouts \((\d+)\)", logText)
- if (foundItems):
- self.timedOutTests = int(foundItems[0])
-
- self.totalFailedTests = self.incorrectTests + self.crashedTests + self.timedOutTests
-
- if self.totalFailedTests > 0:
- self.statusLine = [
- "%d API tests failed, %d crashed, %d timed out" % (self.incorrectTests, self.crashedTests, self.timedOutTests)
- ]
-
- def evaluateCommand(self, cmd):
- if self.totalFailedTests > 0:
- return FAILURE
-
- if cmd.rc != 0:
- return FAILURE
-
- return SUCCESS
-
- def getText(self, cmd, results):
- return self.getText2(cmd, results)
-
- def getText2(self, cmd, results):
- if results != SUCCESS and self.totalFailedTests > 0:
- return self.statusLine
-
- return [self.name]
-
-
-class RunGtkAPITests(RunGLibAPITests):
- command = ["python", "./Tools/Scripts/run-gtk-tests", "--verbose", WithProperties("--%(configuration)s")]
-
-
-class RunWPEAPITests(RunGLibAPITests):
- command = ["python", "./Tools/Scripts/run-wpe-tests", "--verbose", WithProperties("--%(configuration)s")]
-
-
-class RunWebKit1Tests(RunWebKitTests):
- def start(self):
- self.setCommand(self.command + ["--dump-render-tree"])
-
- return RunWebKitTests.start(self)
-
-
-class RunWebKit1LeakTests(RunWebKit1Tests):
- want_stdout = False
- want_stderr = False
- warnOnWarnings = True
- def start(self):
- self.setCommand(self.command + ["--leaks"])
- return RunWebKit1Tests.start(self)
-
-
-class RunAndUploadPerfTests(shell.Test):
- name = "perf-test"
- description = ["perf-tests running"]
- descriptionDone = ["perf-tests"]
- command = ["python", "./Tools/Scripts/run-perf-tests",
- "--output-json-path", "perf-test-results.json",
- "--slave-config-json-path", "../../perf-test-config.json",
- "--no-show-results",
- "--reset-results",
- "--test-results-server", "perf.webkit.org",
- "--builder-name", WithProperties("%(buildername)s"),
- "--build-number", WithProperties("%(buildnumber)s"),
- "--platform", WithProperties("%(fullPlatform)s"),
- "--no-build",
- WithProperties("--%(configuration)s")]
-
- def start(self):
- additionalArguments = self.getProperty("additionalArguments")
- if additionalArguments:
- self.command += additionalArguments
- self.setCommand(self.command)
- return shell.Test.start(self)
-
- def getText(self, cmd, results):
- return self.getText2(cmd, results)
-
- def getText2(self, cmd, results):
- if results != SUCCESS:
- if cmd.rc == -1 & 0xff:
- return ["build not up to date"]
- elif cmd.rc == -2 & 0xff:
- return ["slave config JSON error"]
- elif cmd.rc == -3 & 0xff:
- return ["output JSON merge error"]
- elif cmd.rc == -4 & 0xff:
- return ["upload error"]
- elif cmd.rc == -5 & 0xff:
- return ["system dependency error"]
- elif cmd.rc == -1:
- return ["timeout"]
- else:
- return ["%d perf tests failed" % cmd.rc]
-
- return [self.name]
-
-
-class RunBenchmarkTests(shell.Test):
- name = "benchmark-test"
- description = ["benchmark tests running"]
- descriptionDone = ["benchmark tests"]
- # Buildbot default timeout without output for a step is 1200.
- # The current maximum timeout for a benchmark plan is also 1200.
- # So raise the buildbot timeout to avoid aborting this whole step when a test timeouts.
- timeout = 1500
- command = ["python", "./Tools/Scripts/run-benchmark", "--allplans"]
-
- def start(self):
- platform = self.getProperty("platform")
- if platform == "gtk":
- self.command += ["--browser", "minibrowser-gtk"]
- self.setCommand(self.command)
- return shell.Test.start(self)
-
- def getText(self, cmd, results):
- return self.getText2(cmd, results)
-
- def getText2(self, cmd, results):
- if results != SUCCESS:
- return ["%d benchmark tests failed" % cmd.rc]
- return [self.name]
-
-
-class ArchiveTestResults(shell.ShellCommand):
- command = ["python", "./Tools/BuildSlaveSupport/test-result-archive",
- WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
- name = "archive-test-results"
- description = ["archiving test results"]
- descriptionDone = ["archived test results"]
- haltOnFailure = True
-
-
-class UploadTestResults(transfer.FileUpload):
- slavesrc = "layout-test-results.zip"
- masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
-
- def __init__(self, **kwargs):
- kwargs['slavesrc'] = self.slavesrc
- kwargs['masterdest'] = self.masterdest
- kwargs['mode'] = 0644
- transfer.FileUpload.__init__(self, **kwargs)
-
-
-class TransferToS3(master.MasterShellCommand):
- name = "transfer-to-s3"
- description = ["transferring to s3"]
- descriptionDone = ["transferred to s3"]
- archive = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
- minifiedArchive = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/minified-%(got_revision)s.zip")
- identifier = WithProperties("%(fullPlatform)s-%(architecture)s-%(configuration)s")
- revision = WithProperties("%(got_revision)s")
- command = ["python", "./transfer-archive-to-s3", "--revision", revision, "--identifier", identifier, "--archive", archive]
- haltOnFailure = True
-
- def __init__(self, **kwargs):
- kwargs['command'] = self.command
- master.MasterShellCommand.__init__(self, **kwargs)
-
- def start(self):
- return master.MasterShellCommand.start(self)
-
- def finished(self, result):
- return master.MasterShellCommand.finished(self, result)
-
-
-class ExtractTestResults(master.MasterShellCommand):
- zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
- resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
- descriptionDone = ["uploaded results"]
-
- def __init__(self, **kwargs):
- kwargs['command'] = ""
- master.MasterShellCommand.__init__(self, **kwargs)
-
- def resultDirectoryURL(self):
- return self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + "/"
-
- def start(self):
- self.command = ["unzip", self.build.getProperties().render(self.zipFile), "-d", self.build.getProperties().render(self.resultDirectory)]
- return master.MasterShellCommand.start(self)
-
- def addCustomURLs(self):
- self.addURL("view layout test results", self.resultDirectoryURL() + "results.html")
- self.addURL("view dashboard test results", self.resultDirectoryURL() + "dashboard-layout-test-results/results.html")
-
- def finished(self, result):
- self.addCustomURLs()
- return master.MasterShellCommand.finished(self, result)
-
-
-class ExtractTestResultsAndLeaks(ExtractTestResults):
- def addCustomURLs(self):
- ExtractTestResults.addCustomURLs(self)
- url = "" + urllib.quote(self.resultDirectoryURL(), safe="")
- self.addURL("view leaks", url)
-
-
-class Factory(factory.BuildFactory):
- def __init__(self, platform, configuration, architectures, buildOnly, additionalArguments, SVNMirror):
- factory.BuildFactory.__init__(self)
- self.addStep(ConfigureBuild(platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly, additionalArguments=additionalArguments, SVNMirror=SVNMirror))
- if SVNMirror:
- self.addStep(WaitForSVNServer())
- self.addStep(CheckOutSource(SVNMirror=SVNMirror))
- if not (platform == "jsc-only"):
- self.addStep(KillOldProcesses())
- self.addStep(CleanBuildIfScheduled())
- self.addStep(DeleteStaleBuildFiles())
- if platform == "win":
- self.addStep(InstallWin32Dependencies())
- if platform == "gtk" and additionalArguments != ["--no-experimental-features"]:
- self.addStep(InstallGtkDependencies())
- if platform == "wpe":
- self.addStep(InstallWpeDependencies())
-
-
-class BuildFactory(Factory):
- def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None):
- Factory.__init__(self, platform, configuration, architectures, True, additionalArguments, SVNMirror)
-
- if platform == "win":
- self.addStep(CompileWebKit(timeout=2*60*60))
- else:
- self.addStep(CompileWebKit())
-
- if triggers:
- self.addStep(ArchiveBuiltProduct())
- self.addStep(UploadBuiltProduct())
- if platform.startswith('mac') or platform.startswith('ios-simulator'):
- self.addStep(ArchiveMinifiedBuiltProduct())
- self.addStep(UploadMinifiedBuiltProduct())
- self.addStep(TransferToS3())
- self.addStep(trigger.Trigger(schedulerNames=triggers))
-
def pickLatestBuild(builder, requests):
return max(requests, key=operator.attrgetter("submittedAt"))
-class TestFactory(Factory):
- JSCTestClass = RunJavaScriptCoreTests
- LayoutTestClass = RunWebKitTests
-
- def getProduct(self):
- self.addStep(DownloadBuiltProduct())
- self.addStep(ExtractBuiltProduct())
-
- def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
- Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
- self.getProduct()
- if self.JSCTestClass:
- self.addStep(self.JSCTestClass())
- if self.LayoutTestClass:
- self.addStep(self.LayoutTestClass())
-
- if platform == 'win' or platform.startswith('mac') or platform.startswith('ios-simulator'):
- self.addStep(RunUnitTests())
- self.addStep(RunPythonTests())
- self.addStep(RunPerlTests())
- self.addStep(RunBindingsTests())
- self.addStep(RunBuiltinsTests())
- if platform != 'win':
- self.addStep(RunDashboardTests())
- if self.LayoutTestClass:
- self.addStep(ArchiveTestResults())
- self.addStep(UploadTestResults())
- self.addStep(ExtractTestResults())
- if platform == "gtk":
- self.addStep(RunGtkAPITests())
- if platform == "wpe":
- self.addStep(RunWPEAPITests())
-
-
-class BuildAndTestFactory(TestFactory):
- def getProduct(self):
- self.addStep(CompileWebKit())
-
- def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
- TestFactory.__init__(self, platform, configuration, architectures, additionalArguments, SVNMirror, **kwargs)
- if triggers:
- self.addStep(ArchiveBuiltProduct())
- self.addStep(UploadBuiltProduct())
- self.addStep(trigger.Trigger(schedulerNames=triggers))
-
-
-class BuildAndTestLLINTCLoopFactory(Factory):
- def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
- Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
- self.addStep(CompileLLINTCLoop())
- self.addStep(RunLLINTCLoopTests())
-
-
-class BuildAndTest32bitJSCFactory(Factory):
- def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None, **kwargs):
- Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
- self.addStep(Compile32bitJSC())
- self.addStep(Run32bitJSCTests())
-
-
-class BuildAndNonLayoutTestFactory(BuildAndTestFactory):
- LayoutTestClass = None
-
-
-class BuildAndRemoteJSCTestsFactory(Factory):
- def __init__(self, platform, configuration, architectures, triggers=None, additionalArguments=None, SVNMirror=None):
- Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
- self.addStep(CompileJSCOnly(timeout=60*60))
- self.addStep(RunRemoteJavaScriptCoreTests(timeout=60*60))
-
-
-class TestWebKit1LeaksFactory(Factory):
- def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None):
- Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
- self.addStep(DownloadBuiltProduct())
- self.addStep(ExtractBuiltProduct())
- self.addStep(RunWebKit1LeakTests())
- self.addStep(ArchiveTestResults())
- self.addStep(UploadTestResults())
- self.addStep(ExtractTestResultsAndLeaks())
-
-
-class TestAllButJSCFactory(TestFactory):
- JSCTestClass = None
-
-
-class TestJSCFactory(Factory):
- def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None):
- Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
- self.addStep(DownloadBuiltProduct())
- self.addStep(ExtractBuiltProduct())
- self.addStep(RunJavaScriptCoreTests())
-
-
-class Test262Factory(Factory):
- def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None):
- Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror)
- self.addStep(DownloadBuiltProduct())
- self.addStep(ExtractBuiltProduct())
- self.addStep(RunTest262Tests())
-
-
-class TestWebKit1Factory(TestFactory):
- LayoutTestClass = RunWebKit1Tests
-
-
-class TestWebKit1AllButJSCFactory(TestWebKit1Factory):
- JSCTestClass = None
-
-
-class BuildAndPerfTestFactory(Factory):
- def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
- Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
- self.addStep(CompileWebKit())
- self.addStep(RunAndUploadPerfTests())
- if platform == "gtk":
- self.addStep(RunBenchmarkTests())
-
-
-class DownloadAndPerfTestFactory(Factory):
- def __init__(self, platform, configuration, architectures, additionalArguments=None, SVNMirror=None, **kwargs):
- Factory.__init__(self, platform, configuration, architectures, False, additionalArguments, SVNMirror, **kwargs)
- self.addStep(DownloadBuiltProduct())
- self.addStep(ExtractBuiltProduct())
- self.addStep(RunAndUploadPerfTests())
- if platform == "gtk":
- self.addStep(RunBenchmarkTests())
-
-
class PlatformSpecificScheduler(AnyBranchScheduler):
def __init__(self, platform, branch, **kwargs):
self.platform = platform
Copied: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/steps.py (from rev 225333, trunk/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg) (0 => 225334)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/steps.py (rev 0)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/steps.py 2017-11-30 19:14:04 UTC (rev 225334)
@@ -0,0 +1,843 @@
+# Copyright (C) 2017 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from buildbot.process import buildstep, factory, properties
+from buildbot.steps import master, shell, source, transfer, trigger
+from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION
+
+from factories import *
+
+from twisted.internet import defer
+
+import os
+import re
+import json
+import cStringIO
+import urllib
+
+APPLE_WEBKIT_AWS_PROXY = "http://54.190.50.182:873"
+S3URL = "https://s3-us-west-2.amazonaws.com/"
+WithProperties = properties.WithProperties
+
+
+class TestWithFailureCount(shell.Test):
+ failedTestsFormatString = "%d test%s failed"
+
+ def countFailures(self, cmd):
+ return 0
+
+ def commandComplete(self, cmd):
+ shell.Test.commandComplete(self, cmd)
+ self.failedTestCount = self.countFailures(cmd)
+ self.failedTestPluralSuffix = "" if self.failedTestCount == 1 else "s"
+
+ def evaluateCommand(self, cmd):
+ if self.failedTestCount:
+ return FAILURE
+
+ if cmd.rc != 0:
+ return FAILURE
+
+ return SUCCESS
+
+ def getText(self, cmd, results):
+ return self.getText2(cmd, results)
+
+ def getText2(self, cmd, results):
+ if results != SUCCESS and self.failedTestCount:
+ return [self.failedTestsFormatString % (self.failedTestCount, self.failedTestPluralSuffix)]
+
+ return [self.name]
+
+
+class ConfigureBuild(buildstep.BuildStep):
+ name = "configure build"
+ description = ["configuring build"]
+ descriptionDone = ["configured build"]
+
+ def __init__(self, platform, configuration, architecture, buildOnly, additionalArguments, SVNMirror, *args, **kwargs):
+ buildstep.BuildStep.__init__(self, *args, **kwargs)
+ self.platform = platform
+ if platform != 'jsc-only':
+ self.platform = platform.split('-', 1)[0]
+ self.fullPlatform = platform
+ self.configuration = configuration
+ self.architecture = architecture
+ self.buildOnly = buildOnly
+ self.additionalArguments = additionalArguments
+ self.SVNMirror = SVNMirror
+ self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, additionalArguments=additionalArguments, SVNMirror=SVNMirror)
+
+ def start(self):
+ self.setProperty("platform", self.platform)
+ self.setProperty("fullPlatform", self.fullPlatform)
+ self.setProperty("configuration", self.configuration)
+ self.setProperty("architecture", self.architecture)
+ self.setProperty("buildOnly", self.buildOnly)
+ self.setProperty("additionalArguments", self.additionalArguments)
+ self.setProperty("SVNMirror", self.SVNMirror)
+ self.finished(SUCCESS)
+ return defer.succeed(None)
+
+
+class CheckOutSource(source.SVN):
+ mode = "update"
+
+ def __init__(self, SVNMirror, **kwargs):
+ kwargs['baseURL'] = SVNMirror or "https://svn.webkit.org/repository/webkit/"
+ kwargs['defaultBranch'] = "trunk"
+ kwargs['mode'] = self.mode
+ source.SVN.__init__(self, **kwargs)
+ self.addFactoryArguments(SVNMirror=SVNMirror)
+
+
+class WaitForSVNServer(shell.ShellCommand):
+ name = "wait-for-svn-server"
+ command = ["python", "./Tools/BuildSlaveSupport/wait-for-SVN-server.py", "-r", WithProperties("%(revision)s"), "-s", WithProperties("%(SVNMirror)s")]
+ description = ["waiting for SVN server"]
+ descriptionDone = ["SVN server is ready"]
+ warnOnFailure = True
+
+ def evaluateCommand(self, cmd):
+ if cmd.rc != 0:
+ return WARNINGS
+ return SUCCESS
+
+
+class InstallWin32Dependencies(shell.Compile):
+ description = ["installing dependencies"]
+ descriptionDone = ["installed dependencies"]
+ command = ["perl", "./Tools/Scripts/update-webkit-auxiliary-libs"]
+
+
+class KillOldProcesses(shell.Compile):
+ name = "kill old processes"
+ description = ["killing old processes"]
+ descriptionDone = ["killed old processes"]
+ command = ["python", "./Tools/BuildSlaveSupport/kill-old-processes", "buildbot"]
+
+
+class CleanBuildIfScheduled(shell.Compile):
+ name = "delete WebKitBuild directory"
+ description = ["deleting WebKitBuild directory"]
+ descriptionDone = ["deleted WebKitBuild directory"]
+ command = ["python", "./Tools/BuildSlaveSupport/clean-build", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
+
+ def start(self):
+ if not self.getProperty('is_clean'):
+ self.hideStepIf = True
+ return SKIPPED
+ return shell.Compile.start(self)
+
+
+class DeleteStaleBuildFiles(shell.Compile):
+ name = "delete stale build files"
+ description = ["deleting stale build files"]
+ descriptionDone = ["deleted stale build files"]
+ command = ["python", "./Tools/BuildSlaveSupport/delete-stale-build-files", WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s")]
+
+ def start(self):
+ if self.getProperty('is_clean'): # Nothing to be done if WebKitBuild had been removed.
+ self.hideStepIf = True
+ return SKIPPED
+ return shell.Compile.start(self)
+
+
+class InstallGtkDependencies(shell.ShellCommand):
+ name = "jhbuild"
+ description = ["updating gtk dependencies"]
+ descriptionDone = ["updated gtk dependencies"]
+ command = ["perl", "./Tools/Scripts/update-webkitgtk-libs"]
+ haltOnFailure = True
+
+
+class InstallWpeDependencies(shell.ShellCommand):
+ name = "jhbuild"
+ description = ["updating wpe dependencies"]
+ descriptionDone = ["updated wpe dependencies"]
+ command = ["perl", "./Tools/Scripts/update-webkitwpe-libs"]
+ haltOnFailure = True
+
+
+def appendCustomBuildFlags(step, platform, fullPlatform):
+ if platform not in ('gtk', 'wincairo', 'ios', 'jsc-only', 'wpe'):
+ return
+ if fullPlatform.startswith('ios-simulator'):
+ platform = 'ios-simulator'
+ elif platform == 'ios':
+ platform = 'device'
+ step.setCommand(step.command + ['--' + platform])
+
+
+class CompileWebKit(shell.Compile):
+ command = ["perl", "./Tools/Scripts/build-webkit", WithProperties("--%(configuration)s")]
+ env = {'MFLAGS': ''}
+ name = "compile-webkit"
+ description = ["compiling"]
+ descriptionDone = ["compiled"]
+ warningPattern = ".*arning: .*"
+
+ def start(self):
+ platform = self.getProperty('platform')
+ buildOnly = self.getProperty('buildOnly')
+ architecture = self.getProperty('architecture')
+ additionalArguments = self.getProperty('additionalArguments')
+
+ if additionalArguments:
+ self.setCommand(self.command + additionalArguments)
+ if platform in ('mac', 'ios') and architecture:
+ self.setCommand(self.command + ['ARCHS=' + architecture])
+ if platform == 'ios':
+ self.setCommand(self.command + ['_ONLY_ACTIVE_ARCH_=NO'])
+ # Generating dSYM files is slow, but these are needed to have line numbers in crash reports on testers.
+ # Debug builds on Yosemite can't use dSYMs, because crash logs end up unsymbolicated.
+ if platform in ('mac', 'ios') and buildOnly and (self.getProperty('fullPlatform') != "mac-yosemite" or self.getProperty('configuration') != "debug"):
+ self.setCommand(self.command + ['DEBUG_INFORMATION_FORMAT=dwarf-with-dsym'])
+
+ appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
+
+ return shell.Compile.start(self)
+
+ def createSummary(self, log):
+ platform = self.getProperty('platform')
+ if platform.startswith('mac'):
+ warnings = []
+ errors = []
+ sio = cStringIO.StringIO(log.getText())
+ for line in sio.readlines():
+ if "arning:" in line:
+ warnings.append(line)
+ if "rror:" in line:
+ errors.append(line)
+ if warnings:
+ self.addCompleteLog('warnings', "".join(warnings))
+ if errors:
+ self.addCompleteLog('errors', "".join(errors))
+
+
+class CompileLLINTCLoop(CompileWebKit):
+ command = ["perl", "./Tools/Scripts/build-jsc", "--cloop", WithProperties("--%(configuration)s")]
+
+
+class Compile32bitJSC(CompileWebKit):
+ command = ["perl", "./Tools/Scripts/build-jsc", "--32-bit", WithProperties("--%(configuration)s")]
+
+
+class CompileJSCOnly(CompileWebKit):
+ command = ["perl", "./Tools/Scripts/build-jsc", WithProperties("--%(configuration)s")]
+
+
+class ArchiveBuiltProduct(shell.ShellCommand):
+ command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
+ WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive"]
+ name = "archive-built-product"
+ description = ["archiving built product"]
+ descriptionDone = ["archived built product"]
+ haltOnFailure = True
+
+
+class ArchiveMinifiedBuiltProduct(ArchiveBuiltProduct):
+ command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
+ WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "archive", "--minify"]
+
+
+class ExtractBuiltProduct(shell.ShellCommand):
+ command = ["python", "./Tools/BuildSlaveSupport/built-product-archive",
+ WithProperties("--platform=%(fullPlatform)s"), WithProperties("--%(configuration)s"), "extract"]
+ name = "extract-built-product"
+ description = ["extracting built product"]
+ descriptionDone = ["extracted built product"]
+ haltOnFailure = True
+
+
+class UploadBuiltProduct(transfer.FileUpload):
+ slavesrc = WithProperties("WebKitBuild/%(configuration)s.zip")
+ masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
+ haltOnFailure = True
+
+ def __init__(self, **kwargs):
+ kwargs['slavesrc'] = self.slavesrc
+ kwargs['masterdest'] = self.masterdest
+ kwargs['mode'] = 0644
+ kwargs['blocksize'] = 1024 * 256
+ transfer.FileUpload.__init__(self, **kwargs)
+
+
+class UploadMinifiedBuiltProduct(UploadBuiltProduct):
+ slavesrc = WithProperties("WebKitBuild/minified-%(configuration)s.zip")
+ masterdest = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/minified-%(got_revision)s.zip")
+
+
+class DownloadBuiltProduct(shell.ShellCommand):
+ command = ["python", "./Tools/BuildSlaveSupport/download-built-product",
+ WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"),
+ WithProperties(S3URL + "archives.webkit.org/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")]
+ name = "download-built-product"
+ description = ["downloading built product"]
+ descriptionDone = ["downloaded built product"]
+ haltOnFailure = True
+ flunkOnFailure = True
+
+ def start(self):
+ if 'apple' in self.getProperty('buildername').lower():
+ self.slaveEnvironment['HTTPS_PROXY'] = APPLE_WEBKIT_AWS_PROXY # curl env var to use a proxy
+ return shell.ShellCommand.start(self)
+
+
+class RunJavaScriptCoreTests(TestWithFailureCount):
+ name = "jscore-test"
+ description = ["jscore-tests running"]
+ descriptionDone = ["jscore-tests"]
+ jsonFileName = "jsc_results.json"
+ command = ["perl", "./Tools/Scripts/run-_javascript_core-tests", "--no-build", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
+ failedTestsFormatString = "%d JSC test%s failed"
+ logfiles = {"json": jsonFileName}
+
+ def start(self):
+ platform = self.getProperty('platform')
+ # Linux bots have currently problems with JSC tests that try to use large amounts of memory.
+ # Check: https://bugs.webkit.org/show_bug.cgi?id=175140
+ if platform in ('gtk', 'wpe'):
+ self.setCommand(self.command + ['--memory-limited'])
+ appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
+ return shell.Test.start(self)
+
+ def countFailures(self, cmd):
+ logText = cmd.logs['stdio'].getText()
+
+ match = re.search(r'^Results for JSC stress tests:\r?\n\s+(\d+) failure', logText, re.MULTILINE)
+ if match:
+ return int(match.group(1))
+
+ match = re.search(r'^Results for Mozilla tests:\r?\n\s+(\d+) regression', logText, re.MULTILINE)
+ if match:
+ return int(match.group(1))
+
+ return 0
+
+
+class RunRemoteJavaScriptCoreTests(RunJavaScriptCoreTests):
+ def start(self):
+ self.setCommand(self.command + ["--memory-limited", "--remote-config-file", "../../remote-jsc-tests-config.json"])
+ return RunJavaScriptCoreTests.start(self)
+
+
+class RunTest262Tests(TestWithFailureCount):
+ name = "test262-test"
+ description = ["test262-tests running"]
+ descriptionDone = ["test262-tests"]
+ failedTestsFormatString = "%d Test262 test%s failed"
+ command = ["perl", "./Tools/Scripts/run-jsc-stress-tests", WithProperties("--%(configuration)s"), "JSTests/test262.yaml"]
+
+ def start(self):
+ appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
+ return shell.Test.start(self)
+
+ def countFailures(self, cmd):
+ logText = cmd.logs['stdio'].getText()
+ matches = re.findall(r'^FAIL:', logText, flags=re.MULTILINE)
+ if matches:
+ return len(matches)
+ return 0
+
+
+class RunWebKitTests(shell.Test):
+ name = "layout-test"
+ description = ["layout-tests running"]
+ descriptionDone = ["layout-tests"]
+ resultDirectory = "layout-test-results"
+ command = ["python", "./Tools/Scripts/run-webkit-tests",
+ "--no-build",
+ "--no-show-results",
+ "--no-new-test-results",
+ "--builder-name", WithProperties("%(buildername)s"),
+ "--build-number", WithProperties("%(buildnumber)s"),
+ "--master-name", "webkit.org",
+ "--test-results-server", "webkit-test-results.webkit.org",
+ "--exit-after-n-crashes-or-timeouts", "50",
+ "--exit-after-n-failures", "500",
+ WithProperties("--%(configuration)s")]
+
+ def start(self):
+ platform = self.getProperty('platform')
+ appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
+ additionalArguments = self.getProperty('additionalArguments')
+
+ self.setCommand(self.command + ["--results-directory", self.resultDirectory])
+ self.setCommand(self.command + ['--debug-rwt-logging'])
+
+ if platform == "win":
+ self.setCommand(self.command + ['--batch-size', '100', '--root=' + os.path.join("WebKitBuild", self.getProperty('configuration'), "bin32")])
+
+ if additionalArguments:
+ self.setCommand(self.command + additionalArguments)
+ return shell.Test.start(self)
+
+ # FIXME: This will break if run-webkit-tests changes its default log formatter.
+ nrwt_log_message_regexp = re.compile(r'\d{2}:\d{2}:\d{2}(\.\d+)?\s+\d+\s+(?P<message>.*)')
+
+ def _strip_python_logging_prefix(self, line):
+ match_object = self.nrwt_log_message_regexp.match(line)
+ if match_object:
+ return match_object.group('message')
+ return line
+
+ def _parseRunWebKitTestsOutput(self, logText):
+ incorrectLayoutLines = []
+ expressions = [
+ ('flakes', re.compile(r'Unexpected flakiness.+\((\d+)\)')),
+ ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
+ ('missing results', re.compile(r'Regressions: Unexpected missing results\s+\((\d+)\)')),
+ ('failures', re.compile(r'Regressions: Unexpected.+\((\d+)\)')),
+ ]
+ testFailures = {}
+
+ for line in logText.splitlines():
+ if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
+ incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
+ continue
+ for name, _expression_ in expressions:
+ match = _expression_.search(line)
+
+ if match:
+ testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
+ break
+
+ # FIXME: Parse file names and put them in results
+
+ for name in testFailures:
+ incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)
+
+ self.incorrectLayoutLines = incorrectLayoutLines
+
+ def commandComplete(self, cmd):
+ shell.Test.commandComplete(self, cmd)
+
+ logText = cmd.logs['stdio'].getText()
+ self._parseRunWebKitTestsOutput(logText)
+
+ def evaluateCommand(self, cmd):
+ result = SUCCESS
+
+ if self.incorrectLayoutLines:
+ if len(self.incorrectLayoutLines) == 1:
+ line = self.incorrectLayoutLines[0]
+ if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
+ return WARNINGS
+
+ for line in self.incorrectLayoutLines:
+ if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
+ result = WARNINGS
+ else:
+ return FAILURE
+
+ # Return code from Tools/Scripts/layout_tests/run_webkit_tests.py.
+ # This means that an exception was raised when running run-webkit-tests and
+ # was never handled.
+ if cmd.rc == 254:
+ return EXCEPTION
+ if cmd.rc != 0:
+ return FAILURE
+
+ return result
+
+ def getText(self, cmd, results):
+ return self.getText2(cmd, results)
+
+ def getText2(self, cmd, results):
+ if results != SUCCESS and self.incorrectLayoutLines:
+ return self.incorrectLayoutLines
+
+ return [self.name]
+
+
+class RunDashboardTests(RunWebKitTests):
+ name = "dashboard-tests"
+ description = ["dashboard-tests running"]
+ descriptionDone = ["dashboard-tests"]
+ resultDirectory = os.path.join(RunWebKitTests.resultDirectory, "dashboard-layout-test-results")
+
+ def start(self):
+ self.setCommand(self.command + ["--layout-tests-directory", "./Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/tests"])
+ return RunWebKitTests.start(self)
+
+
+class RunUnitTests(TestWithFailureCount):
+ name = "run-api-tests"
+ description = ["unit tests running"]
+ descriptionDone = ["unit-tests"]
+ command = ["perl", "./Tools/Scripts/run-api-tests", "--no-build", WithProperties("--%(configuration)s"), "--verbose"]
+ failedTestsFormatString = "%d unit test%s failed or timed out"
+
+ def start(self):
+ appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
+ return shell.Test.start(self)
+
+ def countFailures(self, cmd):
+ log_text = cmd.logs['stdio'].getText()
+ count = 0
+
+ split = re.split(r'\sTests that timed out:\s', log_text)
+ if len(split) > 1:
+ count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
+
+ split = re.split(r'\sTests that failed:\s', split[0])
+ if len(split) > 1:
+ count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
+
+ return count
+
+
+class RunPythonTests(TestWithFailureCount):
+ name = "webkitpy-test"
+ description = ["python-tests running"]
+ descriptionDone = ["python-tests"]
+ command = ["python", "./Tools/Scripts/test-webkitpy", "--verbose"]
+ failedTestsFormatString = "%d python test%s failed"
+
+ def start(self):
+ platform = self.getProperty('platform')
+ # Python tests are flaky on the GTK builders, running them serially
+ # helps and does not significantly prolong the cycle time.
+ if platform == 'gtk':
+ self.setCommand(self.command + ['--child-processes', '1'])
+ # Python tests fail on windows bots when running more than one child process
+ # https://bugs.webkit.org/show_bug.cgi?id=97465
+ if platform == 'win':
+ self.setCommand(self.command + ['--child-processes', '1'])
+ return shell.Test.start(self)
+
+ def countFailures(self, cmd):
+ logText = cmd.logs['stdio'].getText()
+ # We're looking for the line that looks like this: FAILED (failures=2, errors=1)
+ regex = re.compile(r'^FAILED \((?P<counts>[^)]+)\)')
+ for line in logText.splitlines():
+ match = regex.match(line)
+ if not match:
+ continue
+ return sum(int(component.split('=')[1]) for component in match.group('counts').split(', '))
+ return 0
+
+
+class RunPerlTests(TestWithFailureCount):
+ name = "webkitperl-test"
+ description = ["perl-tests running"]
+ descriptionDone = ["perl-tests"]
+ command = ["perl", "./Tools/Scripts/test-webkitperl"]
+ failedTestsFormatString = "%d perl test%s failed"
+
+ def countFailures(self, cmd):
+ logText = cmd.logs['stdio'].getText()
+ # We're looking for the line that looks like this: Failed 2/19 test programs. 5/363 subtests failed.
+ regex = re.compile(r'^Failed \d+/\d+ test programs\. (?P<count>\d+)/\d+ subtests failed\.')
+ for line in logText.splitlines():
+ match = regex.match(line)
+ if not match:
+ continue
+ return int(match.group('count'))
+ return 0
+
+
+class RunLLINTCLoopTests(TestWithFailureCount):
+ name = "webkit-jsc-cloop-test"
+ description = ["cloop-tests running"]
+ descriptionDone = ["cloop-tests"]
+ jsonFileName = "jsc_cloop.json"
+ command = ["perl", "./Tools/Scripts/run-_javascript_core-tests", "--cloop", "--no-build", "--no-jsc-stress", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
+ failedTestsFormatString = "%d regression%s found."
+ logfiles = {"json": jsonFileName}
+
+ def countFailures(self, cmd):
+ logText = cmd.logs['stdio'].getText()
+ # We're looking for the line that looks like this: 0 regressions found.
+ regex = re.compile(r'\s*(?P<count>\d+) regressions? found.')
+ for line in logText.splitlines():
+ match = regex.match(line)
+ if not match:
+ continue
+ return int(match.group('count'))
+ return 0
+
+
+class Run32bitJSCTests(TestWithFailureCount):
+ name = "webkit-32bit-jsc-test"
+ description = ["32bit-jsc-tests running"]
+ descriptionDone = ["32bit-jsc-tests"]
+ jsonFileName = "jsc_32bit.json"
+ command = ["perl", "./Tools/Scripts/run-_javascript_core-tests", "--32-bit", "--no-build", "--no-fail-fast", "--json-output={0}".format(jsonFileName), WithProperties("--%(configuration)s")]
+ failedTestsFormatString = "%d regression%s found."
+ logfiles = {"json": jsonFileName}
+
+ def countFailures(self, cmd):
+ logText = cmd.logs['stdio'].getText()
+ # We're looking for the line that looks like this: 0 failures found.
+ regex = re.compile(r'\s*(?P<count>\d+) failures? found.')
+ for line in logText.splitlines():
+ match = regex.match(line)
+ if not match:
+ continue
+ return int(match.group('count'))
+ return 0
+
+
+class RunBindingsTests(shell.Test):
+ name = "bindings-generation-tests"
+ description = ["bindings-tests running"]
+ descriptionDone = ["bindings-tests"]
+ command = ["python", "./Tools/Scripts/run-bindings-tests"]
+
+
+class RunBuiltinsTests(shell.Test):
+ name = "builtins-generator-tests"
+ description = ["builtins-generator-tests running"]
+ descriptionDone = ["builtins-generator-tests"]
+ command = ["python", "./Tools/Scripts/run-builtins-generator-tests"]
+
+
+class RunGLibAPITests(shell.Test):
+ name = "API tests"
+ description = ["API tests running"]
+ descriptionDone = ["API tests"]
+
+ def start(self):
+ additionalArguments = self.getProperty("additionalArguments")
+ if additionalArguments:
+ self.command += additionalArguments
+ self.setCommand(self.command)
+ return shell.Test.start(self)
+
+ def commandComplete(self, cmd):
+ shell.Test.commandComplete(self, cmd)
+
+ logText = cmd.logs['stdio'].getText()
+
+ self.incorrectTests = 0
+ self.crashedTests = 0
+ self.timedOutTests = 0
+ self.skippedTests = 0
+ self.statusLine = []
+
+ foundItems = re.findall("Unexpected failures \((\d+)\)", logText)
+ if (foundItems):
+ self.incorrectTests = int(foundItems[0])
+
+ foundItems = re.findall("Unexpected crashes \((\d+)\)", logText)
+ if (foundItems):
+ self.crashedTests = int(foundItems[0])
+
+ foundItems = re.findall("Unexpected timeouts \((\d+)\)", logText)
+ if (foundItems):
+ self.timedOutTests = int(foundItems[0])
+
+ self.totalFailedTests = self.incorrectTests + self.crashedTests + self.timedOutTests
+
+ if self.totalFailedTests > 0:
+ self.statusLine = ["%d API tests failed, %d crashed, %d timed out" % (self.incorrectTests, self.crashedTests, self.timedOutTests)]
+
+ def evaluateCommand(self, cmd):
+ if self.totalFailedTests > 0:
+ return FAILURE
+
+ if cmd.rc != 0:
+ return FAILURE
+
+ return SUCCESS
+
+ def getText(self, cmd, results):
+ return self.getText2(cmd, results)
+
+ def getText2(self, cmd, results):
+ if results != SUCCESS and self.totalFailedTests > 0:
+ return self.statusLine
+
+ return [self.name]
+
+
+class RunGtkAPITests(RunGLibAPITests):
+ command = ["python", "./Tools/Scripts/run-gtk-tests", "--verbose", WithProperties("--%(configuration)s")]
+
+
+class RunWPEAPITests(RunGLibAPITests):
+ command = ["python", "./Tools/Scripts/run-wpe-tests", "--verbose", WithProperties("--%(configuration)s")]
+
+
+class RunWebKit1Tests(RunWebKitTests):
+ def start(self):
+ self.setCommand(self.command + ["--dump-render-tree"])
+
+ return RunWebKitTests.start(self)
+
+
+class RunWebKit1LeakTests(RunWebKit1Tests):
+ want_stdout = False
+ want_stderr = False
+ warnOnWarnings = True
+
+ def start(self):
+ self.setCommand(self.command + ["--leaks"])
+ return RunWebKit1Tests.start(self)
+
+
+class RunAndUploadPerfTests(shell.Test):
+ name = "perf-test"
+ description = ["perf-tests running"]
+ descriptionDone = ["perf-tests"]
+ command = ["python", "./Tools/Scripts/run-perf-tests",
+ "--output-json-path", "perf-test-results.json",
+ "--slave-config-json-path", "../../perf-test-config.json",
+ "--no-show-results",
+ "--reset-results",
+ "--test-results-server", "perf.webkit.org",
+ "--builder-name", WithProperties("%(buildername)s"),
+ "--build-number", WithProperties("%(buildnumber)s"),
+ "--platform", WithProperties("%(fullPlatform)s"),
+ "--no-build",
+ WithProperties("--%(configuration)s")]
+
+ def start(self):
+ additionalArguments = self.getProperty("additionalArguments")
+ if additionalArguments:
+ self.command += additionalArguments
+ self.setCommand(self.command)
+ return shell.Test.start(self)
+
+ def getText(self, cmd, results):
+ return self.getText2(cmd, results)
+
+ def getText2(self, cmd, results):
+ if results != SUCCESS:
+ if cmd.rc == -1 & 0xff:
+ return ["build not up to date"]
+ elif cmd.rc == -2 & 0xff:
+ return ["slave config JSON error"]
+ elif cmd.rc == -3 & 0xff:
+ return ["output JSON merge error"]
+ elif cmd.rc == -4 & 0xff:
+ return ["upload error"]
+ elif cmd.rc == -5 & 0xff:
+ return ["system dependency error"]
+ elif cmd.rc == -1:
+ return ["timeout"]
+ else:
+ return ["%d perf tests failed" % cmd.rc]
+
+ return [self.name]
+
+
+class RunBenchmarkTests(shell.Test):
+ name = "benchmark-test"
+ description = ["benchmark tests running"]
+ descriptionDone = ["benchmark tests"]
+ # Buildbot default timeout without output for a step is 1200.
+ # The current maximum timeout for a benchmark plan is also 1200.
+ # So raise the buildbot timeout to avoid aborting this whole step when a test timeouts.
+ timeout = 1500
+ command = ["python", "./Tools/Scripts/run-benchmark", "--allplans"]
+
+ def start(self):
+ platform = self.getProperty("platform")
+ if platform == "gtk":
+ self.command += ["--browser", "minibrowser-gtk"]
+ self.setCommand(self.command)
+ return shell.Test.start(self)
+
+ def getText(self, cmd, results):
+ return self.getText2(cmd, results)
+
+ def getText2(self, cmd, results):
+ if results != SUCCESS:
+ return ["%d benchmark tests failed" % cmd.rc]
+ return [self.name]
+
+
+class ArchiveTestResults(shell.ShellCommand):
+ command = ["python", "./Tools/BuildSlaveSupport/test-result-archive",
+ WithProperties("--platform=%(platform)s"), WithProperties("--%(configuration)s"), "archive"]
+ name = "archive-test-results"
+ description = ["archiving test results"]
+ descriptionDone = ["archived test results"]
+ haltOnFailure = True
+
+
+class UploadTestResults(transfer.FileUpload):
+ slavesrc = "layout-test-results.zip"
+ masterdest = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
+
+ def __init__(self, **kwargs):
+ kwargs['slavesrc'] = self.slavesrc
+ kwargs['masterdest'] = self.masterdest
+ kwargs['mode'] = 0644
+ transfer.FileUpload.__init__(self, **kwargs)
+
+
+class TransferToS3(master.MasterShellCommand):
+ name = "transfer-to-s3"
+ description = ["transferring to s3"]
+ descriptionDone = ["transferred to s3"]
+ archive = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/%(got_revision)s.zip")
+ minifiedArchive = WithProperties("archives/%(fullPlatform)s-%(architecture)s-%(configuration)s/minified-%(got_revision)s.zip")
+ identifier = WithProperties("%(fullPlatform)s-%(architecture)s-%(configuration)s")
+ revision = WithProperties("%(got_revision)s")
+ command = ["python", "./transfer-archive-to-s3", "--revision", revision, "--identifier", identifier, "--archive", archive]
+ haltOnFailure = True
+
+ def __init__(self, **kwargs):
+ kwargs['command'] = self.command
+ master.MasterShellCommand.__init__(self, **kwargs)
+
+ def start(self):
+ return master.MasterShellCommand.start(self)
+
+ def finished(self, result):
+ return master.MasterShellCommand.finished(self, result)
+
+
+class ExtractTestResults(master.MasterShellCommand):
+ zipFile = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s).zip")
+ resultDirectory = WithProperties("public_html/results/%(buildername)s/r%(got_revision)s (%(buildnumber)s)")
+ descriptionDone = ["uploaded results"]
+
+ def __init__(self, **kwargs):
+ kwargs['command'] = ""
+ master.MasterShellCommand.__init__(self, **kwargs)
+
+ def resultDirectoryURL(self):
+ return self.build.getProperties().render(self.resultDirectory).replace("public_html/", "/") + "/"
+
+ def start(self):
+ self.command = ["unzip", self.build.getProperties().render(self.zipFile), "-d", self.build.getProperties().render(self.resultDirectory)]
+ return master.MasterShellCommand.start(self)
+
+ def addCustomURLs(self):
+ self.addURL("view layout test results", self.resultDirectoryURL() + "results.html")
+ self.addURL("view dashboard test results", self.resultDirectoryURL() + "dashboard-layout-test-results/results.html")
+
+ def finished(self, result):
+ self.addCustomURLs()
+ return master.MasterShellCommand.finished(self, result)
+
+
+class ExtractTestResultsAndLeaks(ExtractTestResults):
+ def addCustomURLs(self):
+ ExtractTestResults.addCustomURLs(self)
+ url = "" + urllib.quote(self.resultDirectoryURL(), safe="")
+ self.addURL("view leaks", url)
Modified: trunk/Tools/ChangeLog (225333 => 225334)
--- trunk/Tools/ChangeLog 2017-11-30 19:12:01 UTC (rev 225333)
+++ trunk/Tools/ChangeLog 2017-11-30 19:14:04 UTC (rev 225334)
@@ -1,3 +1,14 @@
+2017-11-30 Aakash Jain <aakash_j...@apple.com>
+
+ [build.webkit.org] Move python code from master.cfg in separate files
+ https://bugs.webkit.org/show_bug.cgi?id=180180
+
+ Reviewed by Alexey Proskuryakov.
+
+ * BuildSlaveSupport/build.webkit.org-config/master.cfg: Moved code to steps.py and master.cfg appropriately.
+ * BuildSlaveSupport/build.webkit.org-config/factories.py: Moved all factory related code from master.cfg.
+ * BuildSlaveSupport/build.webkit.org-config/steps.py: Moved all steps related code from master.cfg
+
2017-11-30 Wenson Hsieh <wenson_hs...@apple.com>
WKAttachmentTests.InPlaceImageAttachmentToggleDisplayMode times out on macOS bots
