Modified: trunk/Tools/Scripts/run-jsc-stress-tests (159954 => 159955)
--- trunk/Tools/Scripts/run-jsc-stress-tests 2013-12-02 20:52:11 UTC (rev 159954)
+++ trunk/Tools/Scripts/run-jsc-stress-tests 2013-12-02 20:55:28 UTC (rev 159955)
@@ -23,6 +23,7 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+require 'fileutils'
require 'getoptlong'
require 'pathname'
require 'yaml'
@@ -36,7 +37,7 @@
HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
-IMPORTANT_ENVS = ["JSC_timeout", "DYLD_FRAMEWORK_PATH"]
+IMPORTANT_ENVS = ["JSC_timeout"]
begin
require 'shellwords'
@@ -81,10 +82,10 @@
$jscPath = nil
$enableFTL = false
-$collections = []
$outputDir = Pathname.new("results")
$verbosity = 0
-$errorOnFailure = false
+$bundle = nil
+$tarball = false
def usage
puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
@@ -92,8 +93,9 @@
puts "--jsc (-j) Path to _javascript_Core. This option is required."
puts "--ftl-jit Indicate that we have the FTL JIT."
puts "--output-dir (-o) Path where to put results. Default is #{$outputDir}."
- puts "--[no-]error-on-failure Exit with exit code 1 if any tests fail. Default is #{$errorOnFailure}."
puts "--verbose (-v) Print more things while running."
+ puts "--run-bundle Runs a bundle previously created by run-jsc-stress-tests."
+ puts "--tarball Creates a tarball of the final bundle."
puts "--help (-h) Print this message."
exit 1
end
@@ -102,9 +104,9 @@
['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
['--ftl-jit', GetoptLong::NO_ARGUMENT],
['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
- ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
- ['--error-on-failure', GetoptLong::NO_ARGUMENT],
- ['--no-error-on-failure', GetoptLong::NO_ARGUMENT]).each {
+ ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
+ ['--tarball', GetoptLong::NO_ARGUMENT],
+ ['--verbose', '-v', GetoptLong::NO_ARGUMENT]).each {
| opt, arg |
case opt
when '--help'
@@ -115,17 +117,22 @@
$outputDir = Pathname.new(arg)
when '--ftl-jit'
$enableFTL = true
- when '--error-on-failure'
- $errorOnFailure = true
- when '--no-error-on-failure'
- $errorOnFailure = false
when '--verbose'
$verbosity += 1
+ when '--run-bundle'
+ $bundle = Pathname.new(arg)
+ when '--tarball'
+ $tarball = true
end
}
$progressMeter = ($verbosity == 0 and $stdin.tty?)
+if $bundle
+ $jscPath = $bundle + ".vm" + "_javascript_Core.framework" + "Resources" + "jsc"
+ $outputDir = $bundle
+end
+
unless $jscPath
$stderr.puts "Error: must specify -j <path>."
exit 1
@@ -137,6 +144,55 @@
$runlist = []
+def frameworkFromJSCPath(jscPath)
+ if jscPath.dirname.basename.to_s == "Resources" and jscPath.dirname.dirname.basename.to_s == "_javascript_Core.framework"
+ jscPath.dirname.dirname
+ elsif jscPath.dirname.dirname.basename.to_s == "WebKitBuild"
+ parentDirName = jscPath.dirname.basename.to_s
+ if parentDirName == "Debug" or parentDirName == "Release"
+ jscPath.dirname + "_javascript_Core.framework"
+ else
+ raise "Unknown JSC path, cannot copy full framework: #{jscPath}"
+ end
+ else
+ $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
+ nil
+ end
+end
+
+def prepareFramework(jscPath)
+ frameworkPath = frameworkFromJSCPath(jscPath)
+ $frameworkPath = Pathname.new(".vm") + "_javascript_Core.framework"
+ $jscPath = $frameworkPath + "Resources" + "jsc"
+
+ if frameworkPath
+ FileUtils.cp_r frameworkPath, vmDir
+ else
+ Dir.chdir($outputDir) {
+ FileUtils.mkdir_p $jscPath.dirname
+ FileUtils.cp jscPath, $jscPath
+ }
+ end
+end
+
+def copyVMToBundle
+ raise if $bundle
+
+ vmDir = $outputDir + ".vm"
+ FileUtils.mkdir_p vmDir
+
+ prepareFramework($jscPath)
+end
+
+def pathToVM
+ dir = Pathname.new(".")
+ $benchmarkDirectory.each_filename {
+ | pathComponent |
+ dir += ".."
+ }
+ dir + $jscPath
+end
+
def prefixCommand(prefix)
"awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
end
@@ -149,7 +205,7 @@
def silentOutputHandler
Proc.new {
| name |
- " | " + pipeAndPrefixCommand(($outputDir + (name + ".out")).to_s, name)
+ " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
}
end
@@ -157,7 +213,7 @@
def noisyOutputHandler
Proc.new {
| name |
- " | cat > " + Shellwords.shellescape(($outputDir + (name + ".out")).to_s)
+ " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
}
end
@@ -179,16 +235,16 @@
def diffErrorHandler(expectedFilename)
Proc.new {
| outp, plan |
- outputFilename = Shellwords.shellescape(($outputDir + (plan.name + ".out")).to_s)
- diffFilename = Shellwords.shellescape(($outputDir + (plan.name + ".diff")).to_s)
+ outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
+ diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + prefixCommand(plan.name)
outp.puts " " + plan.failCommand
- outp.puts "elif test -e #{Shellwords.shellescape(expectedFilename)}"
+ outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
outp.puts "then"
- outp.puts " diff -u #{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
+ outp.puts " diff -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
outp.puts " if [ $? -eq 0 ]"
outp.puts " then"
outp.puts " " + plan.successCommand
@@ -208,13 +264,13 @@
def mozillaErrorHandler
Proc.new {
| outp, plan |
- outputFilename = Shellwords.shellescape(($outputDir + (plan.name + ".out")).to_s)
+ outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + prefixCommand(plan.name)
outp.puts " " + plan.failCommand
- outp.puts "elif ruby " + Shellwords.shellescape((HELPERS_PATH + "check-mozilla-failure").to_s) + " #{outputFilename}"
+ outp.puts "elif grep -i -q failed! #{outputFilename}"
outp.puts "then"
outp.puts " (echo Detected failures: && cat #{outputFilename}) | " + prefixCommand(plan.name)
outp.puts " " + plan.failCommand
@@ -229,12 +285,12 @@
def mozillaFailErrorHandler
Proc.new {
| outp, plan |
- outputFilename = Shellwords.shellescape(($outputDir + (plan.name + ".out")).to_s)
+ outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " " + plan.successCommand
- outp.puts "elif ruby " + Shellwords.shellescape((HELPERS_PATH + "check-mozilla-failure").to_s) + " #{outputFilename}"
+ outp.puts "elif grep -i -q failed! #{outputFilename}"
outp.puts "then"
outp.puts " " + plan.successCommand
outp.puts "else"
@@ -249,13 +305,13 @@
def mozillaExit3ErrorHandler
Proc.new {
| outp, plan |
- outputFilename = Shellwords.shellescape(($outputDir + (plan.name + ".out")).to_s)
+ outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " if [ `cat #{plan.failFile}` -eq 3 ]"
outp.puts " then"
- outp.puts " if ruby " + Shellwords.shellescape((HELPERS_PATH + "check-mozilla-failure").to_s) + " #{outputFilename}"
+ outp.puts " if grep -i -q failed! #{outputFilename}"
outp.puts " then"
outp.puts " (echo Detected failures: && cat #{outputFilename}) | " + prefixCommand(plan.name)
outp.puts " " + plan.failCommand
@@ -280,7 +336,7 @@
attr_accessor :index
def initialize(directory, arguments, name, outputHandler, errorHandler)
- @directory = directory.realpath
+ @directory = directory
@arguments = arguments
@name = name
@outputHandler = outputHandler
@@ -289,7 +345,10 @@
end
def shellCommand
- "(cd #{Shellwords.shellescape(@directory.to_s)} && \"$@\" " + @arguments.map{
+ # It's important to remember that the test is actually run in a subshell, so if we change directory
+ # in the subshell when we return we will be in our original directory. This is nice because we don't
+ # have to bend over backwards to do things relative to the root.
+ "(cd ../#{Shellwords.shellescape(@directory.to_s)} && \"$@\" " + @arguments.map{
| v |
raise "Detected a non-string in #{inspect}" unless v.is_a? String
Shellwords.shellescape(v)
@@ -297,7 +356,17 @@
end
def reproScriptCommand
- script = ""
+ # We have to find our way back to the .parallel directory since that's where all of the relative
+ # paths assume they start out from.
+ script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
+ script += "cd $CURRENT_DIR\n"
+ Pathname.new(@name).dirname.each_filename {
+ | pathComponent |
+ script += "cd ..\n"
+ }
+ script += "cd .parallel\n"
+
+ script += "export DYLD_FRAMEWORK_PATH=$(cd ../#{$frameworkPath.dirname}; pwd)\n"
IMPORTANT_ENVS.each {
| key |
if ENV[key]
@@ -305,7 +374,7 @@
end
}
script += "#{shellCommand} || exit 1"
- "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape(($outputDir + @name).to_s)}"
+ "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
end
def failCommand
@@ -366,22 +435,26 @@
# returns false, in which case you're supposed to add your own run commands.
def parseRunCommands
didRun = false
- File.open($benchmarkDirectory + $benchmark) {
- | inp |
- inp.each_line {
- | line |
- begin
- doesMatch = line =~ /^\/\/@/
- rescue Exception => e
- # Apparently this happens in the case of some UTF8 stuff in some files, where
- # Ruby tries to be strict and throw exceptions.
- next
- end
- next unless doesMatch
- eval $~.post_match
- didRun = true
+
+ Dir.chdir($outputDir) {
+ File.open($benchmarkDirectory + $benchmark) {
+ | inp |
+ inp.each_line {
+ | line |
+ begin
+ doesMatch = line =~ /^\/\/@/
+ rescue Exception => e
+ # Apparently this happens in the case of some UTF8 stuff in some files, where
+ # Ruby tries to be strict and throw exceptions.
+ next
+ end
+ next unless doesMatch
+ eval $~.post_match
+ didRun = true
+ }
}
}
+
didRun
end
@@ -390,7 +463,7 @@
end
def run(kind, *options)
- addRunCommand(kind, [$jscPath.to_s] + options + [$benchmark.to_s], silentOutputHandler, simpleErrorHandler)
+ addRunCommand(kind, [pathToVM.to_s] + options + [$benchmark.to_s], silentOutputHandler, simpleErrorHandler)
end
def runDefault
@@ -478,7 +551,7 @@
def runProfiler
profilerOutput = uniqueFilename(".json")
if $canRunDisplayProfilerOutput
- addRunCommand("profiler", ["ruby", (HELPERS_PATH + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, $jscPath.to_s, "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
+ addRunCommand("profiler", ["ruby", (HELPERS_PATH + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
else
puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
run("profiler-simple", "-p", profilerOutput.to_s)
@@ -493,12 +566,15 @@
else
kind = "layout"
end
- args =
- [$jscPath.to_s] + options +
- [(LAYOUTTESTS_PATH + "resources" + "standalone-pre.js").to_s,
+
+ prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
+ prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
+
+ args = [pathToVM.to_s] + options +
+ [(Pathname.new("resources") + "standalone-pre.js").to_s,
$benchmark.to_s,
- (LAYOUTTESTS_PATH + "resources" + "standalone-post.js").to_s]
- addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory.dirname + "#{testName}-expected.txt").to_s))
+ (Pathname.new("resources") + "standalone-post.js").to_s]
+ addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
end
def runLayoutTestDefault
@@ -524,13 +600,40 @@
runLayoutTestDFGEagerNoCJIT
end
+def prepareExtraRelativeFiles(extraFiles, destination)
+ Dir.chdir($outputDir) {
+ extraFiles.each {
+ | file |
+ FileUtils.cp $extraFilesBaseDir + file, destination + file
+ }
+ }
+end
+
+def baseDirForCollection(collectionName)
+ Pathname(".tests") + collectionName
+end
+
+def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
+ raise unless absoluteBase.absolute?
+ Dir.chdir($outputDir) {
+ collectionBaseDir = baseDirForCollection($collectionName)
+ extraFiles.each {
+ | file |
+ destination = collectionBaseDir + file
+ FileUtils.mkdir_p destination.dirname unless destination.directory?
+ FileUtils.cp absoluteBase + file, destination
+ }
+ }
+end
+
def runMozillaTest(kind, mode, extraFiles, *options)
if kind
kind = "mozilla-" + kind
else
kind = "mozilla"
end
- args = [$jscPath.to_s] + options + extraFiles.map{|v| ($benchmarkDirectory + v).to_s} + [$benchmark.to_s]
+ prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
+ args = [pathToVM.to_s] + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
case mode
when :normal
errorHandler = mozillaErrorHandler
@@ -573,14 +676,6 @@
puts "Skipping #{$collectionName}/#{$benchmark}"
end
-Dir.mkdir($outputDir) unless $outputDir.directory?
-begin
- File.delete($outputDir + "failed")
-rescue
-end
-
-$outputDir = $outputDir.realpath
-
def allJSFiles(path)
if path.file?
[path]
@@ -622,11 +717,29 @@
end
def prepareCollection(name)
- dir = $outputDir
- Pathname.new(name).each_filename {
- | filename |
- dir = dir + filename
- Dir.mkdir(dir) unless dir.directory?
+ FileUtils.mkdir_p $outputDir + name
+
+ absoluteCollection = $collection.realpath
+
+ Dir.chdir($outputDir) {
+ bundleDir = baseDirForCollection(name)
+
+ # Create the proper directory structures.
+ FileUtils.mkdir_p bundleDir
+ if bundleDir.basename == $collection.basename
+ FileUtils.cp_r absoluteCollection, bundleDir.dirname
+ else
+ FileUtils.cp_r absoluteCollection, bundleDir
+ end
+
+ $extraFilesBaseDir = absoluteCollection
+
+ # Redirect the collection's location to the newly constructed bundle.
+ if absoluteCollection.directory?
+ $collection = bundleDir
+ else
+ $collection = bundleDir + $collection.basename
+ end
}
end
@@ -634,7 +747,7 @@
def handleCollectionFile(collection)
collectionName = simplifyCollectionName(collection)
-
+
paths = {}
subCollections = []
YAML::load(IO::read(collection)).each {
@@ -670,17 +783,17 @@
$collectionName = $collectionName.to_s
prepareCollection($collectionName)
-
- allJSFiles(subCollection).each {
- | path |
-
- path = path.realpath
-
- $benchmark = path.basename
- $benchmarkDirectory = path.dirname
-
- $runCommandOptions = {}
- eval entry["cmd"]
+
+ Dir.chdir($outputDir) {
+ allJSFiles($collection).each {
+ | path |
+
+ $benchmark = path.basename
+ $benchmarkDirectory = path.dirname
+
+ $runCommandOptions = {}
+ eval entry["cmd"]
+ }
}
}
@@ -693,18 +806,20 @@
def handleCollectionDirectory(collection)
collectionName = simplifyCollectionName(collection)
- prepareCollection(collectionName)
-
$collection = collection
$collectionName = collectionName
- $benchmarkDirectory = $collection
- allJSFiles($collection).each {
- | path |
-
- $benchmark = path.basename
-
- $runCommandOptions = {}
- defaultRun unless parseRunCommands
+ prepareCollection(collectionName)
+
+ Dir.chdir($outputDir) {
+ $benchmarkDirectory = $collection
+ allJSFiles($collection).each {
+ | path |
+
+ $benchmark = path.basename
+
+ $runCommandOptions = {}
+ defaultRun unless parseRunCommands
+ }
}
end
@@ -718,11 +833,6 @@
end
end
-ARGV.each {
- | collection |
- handleCollection(collection)
-}
-
def appendFailure(plan)
File.open($outputDir + "failed", "a") {
| outp |
@@ -731,6 +841,126 @@
$numFailures += 1
end
+def prepareBundle
+ raise if $bundle
+
+ copyVMToBundle
+
+ ARGV.each {
+ | collection |
+ handleCollection(collection)
+ }
+end
+
+def cleanOldResults
+ raise unless $bundle
+
+ eachResultFile($outputDir) {
+ | path |
+ FileUtils.rm_f path
+ }
+end
+
+def cleanEmptyResultFiles
+ eachResultFile($outputDir) {
+ | path |
+ next unless path.basename.to_s =~ /\.out$/
+ next unless FileTest.size(path) == 0
+ FileUtils.rm_f path
+ }
+end
+
+def eachResultFile(startingDir, &block)
+ dirsToClean = [startingDir]
+ until dirsToClean.empty? do
+ nextDir = dirsToClean.pop
+ Dir.foreach(nextDir) {
+ | entry |
+ next if entry =~ /^\./
+ path = nextDir + entry
+ if path.directory?
+ dirsToClean.push(path)
+ else
+ block.call(path)
+ end
+ }
+ end
+end
+
+def prepareParallelTestRunner
+ # The goals of our parallel test runner are scalability and simplicity. The
+ # simplicity part is particularly important. We don't want to have to have
+ # a full-time contributor just philosophising about parallel testing.
+ #
+ # As such, we just pass off all of the hard work to 'make'. This creates a
+ # dummy directory ("$outputDir/.parallel") in which we create a dummy
+ # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
+ # That is, for each test we know we will run, there is a rule in the
+ # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
+ # Makefile results in 'make' doing all of the hard work:
+ #
+ # - Load balancing just works. Most systems have a great load balancer in
+ # 'make'. If your system doesn't then just install a real 'make'.
+ #
+ # - Interruptions just work. For example Ctrl-C handling in 'make' is
+ # exactly right. You don't have to worry about zombie processes.
+ #
+ # We then do some tricks to make failure detection work and to make this
+ # totally sound. If a test fails, we don't want the whole 'make' job to
+ # stop. We also don't have any facility for makefile-escaping of path names.
+ # We do have such a thing for shell-escaping, though. We fix both problems
+ # by having the actual work for each of the test rules be done in a shell
+ # script on the side. There is one such script per test. The script responds
+ # to failure by printing something on the console and then touching a
+ # failure file for that test, but then still returns 0. This makes 'make'
+ # continue past that failure and complete all the tests anyway.
+ #
+ # In the end, this script collects all of the failures by searching for
+ # files in the .parallel directory whose name matches /^test_fail_/, where
+ # the thing after the 'fail_' is the test index. Those are the files that
+ # would be created by the test scripts if they detect failure. We're
+ # basically using the filesystem as a concurrent database of test failures.
+ # Even if two tests fail at the same time, since they're touching different
+ # files we won't miss any failures.
+
+ runIndices = []
+ $runlist.each_with_index {
+ | plan, index |
+ runIndices << index
+ plan.index = index
+ }
+
+ Dir.mkdir($parallelDir) unless $parallelDir.directory?
+ toDelete = []
+ Dir.foreach($parallelDir) {
+ | filename |
+ if filename =~ /^test_/
+ toDelete << filename
+ end
+ }
+
+ toDelete.each {
+ | filename |
+ File.unlink($parallelDir + filename)
+ }
+
+ $runlist.each {
+ | plan |
+ plan.writeRunScript($parallelDir + "test_script_#{plan.index}")
+ }
+
+ File.open($parallelDir + "Makefile", "w") {
+ | outp |
+ outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
+ runIndices.each {
+ | index |
+ plan = $runlist[index]
+ outp.puts "test_done_#{index}:"
+ outp.puts "\tsh test_script_#{plan.index}"
+ }
+ }
+end
+
if $enableFTL and ENV["JSC_timeout"]
# Currently, using the FTL is a performance regression particularly in real
# (i.e. non-loopy) benchmarks. Account for this in the timeout.
@@ -742,154 +972,117 @@
# Increase the timeout proportionally to the number of processors.
ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i.to_f * Math.sqrt($numProcessors)).to_i.to_s
end
-
-# The goals of our parallel test runner are scalability and simplicity. The
-# simplicity part is particularly important. We don't want to have to have
-# a full-time contributor just philosophising about parallel testing.
-#
-# As such, we just pass off all of the hard work to 'make'. This creates a
-# dummy directory ("$outputDir/.parallel") in which we create a dummy
-# Makefile. The Makefile has an 'all' rule that depends on all of the tests.
-# That is, for each test we know we will run, there is a rule in the
-# Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
-# Makefile results in 'make' doing all of the hard work:
-#
-# - Load balancing just works. Most systems have a great load balancer in
-# 'make'. If your system doesn't then just install a real 'make'.
-#
-# - Interruptions just work. For example Ctrl-C handling in 'make' is
-# exactly right. You don't have to worry about zombie processes.
-#
-# We then do some tricks to make failure detection work and to make this
-# totally sound. If a test fails, we don't want the whole 'make' job to
-# stop. We also don't have any facility for makefile-escaping of path names.
-# We do have such a thing for shell-escaping, though. We fix both problems
-# by having the actual work for each of the test rules be done in a shell
-# script on the side. There is one such script per test. The script responds
-# to failure by printing something on the console and then touching a
-# failure file for that test, but then still returns 0. This makes 'make'
-# continue past that failure and complete all the tests anyway.
-#
-# In the end, this script collects all of the failures by searching for
-# files in the .parallel directory whose name matches /^test_fail_/, where
-# the thing after the 'fail_' is the test index. Those are the files that
-# would be created by the test scripts if they detect failure. We're
-# basically using the filesystem as a concurrent database of test failures.
-# Even if two tests fail at the same time, since they're touching different
-# files we won't miss any failures.
-
-runIndices = []
-$runlist.each_with_index {
- | plan, index |
- runIndices << index
- plan.index = index
-}
-
-parallelDir = $outputDir + ".parallel"
-Dir.mkdir(parallelDir) unless parallelDir.directory?
-toDelete = []
-Dir.foreach(parallelDir) {
- | filename |
- if filename =~ /^test_/
- toDelete << filename
- end
-}
-
-toDelete.each {
- | filename |
- File.unlink(parallelDir + filename)
-}
-
+
puts
-$runlist.each {
- | plan |
- plan.writeRunScript(parallelDir + "test_script_#{plan.index}")
-}
-
-File.open(parallelDir + "Makefile", "w") {
- | outp |
- outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
- runIndices.each {
- | index |
- plan = $runlist[index]
- outp.puts "test_done_#{index}:"
- outp.puts "\tsh test_script_#{plan.index}"
+def cleanParallelDirectory
+ raise unless $bundle
+ Dir.foreach($parallelDir) {
+ | filename |
+ next unless filename =~ /^test_fail/
+ FileUtils.rm_f $parallelDir + filename
}
-}
+end
-Dir.chdir(parallelDir) {
- unless $progressMeter
- mysys("make", "-j", $numProcessors.to_s, "-s", "-f", "Makefile")
- else
- cmd = "make -j #{$numProcessors} -s -f Makefile"
- running = {}
- didRun = {}
- didFail = {}
- blankLine = true
- prevStringLength = 0
- IO.popen(cmd, "r") {
- | inp |
- inp.each_line {
- | line |
- line.chomp!
- if line =~ /^Running /
- running[$~.post_match] = true
- elsif line =~ /^PASS: /
- didRun[$~.post_match] = true
- elsif line =~ /^FAIL: /
- didRun[$~.post_match] = true
- didFail[$~.post_match] = true
- else
- unless blankLine
- print("\r" + " " * prevStringLength + "\r")
- end
- puts line
- blankLine = true
- end
-
- def lpad(str, chars)
- str = str.to_s
- if str.length > chars
- str
+def runParallelTestRunner
+ Dir.chdir($parallelDir) {
+ # -1 for the Makefile, and -2 for '..' and '.'
+ numberOfTests = Dir.entries(".").count - 3
+ unless $progressMeter
+ mysys("make", "-j", $numProcessors.to_s, "-s", "-f", "Makefile")
+ else
+ cmd = "make -j #{$numProcessors} -s -f Makefile"
+ running = {}
+ didRun = {}
+ didFail = {}
+ blankLine = true
+ prevStringLength = 0
+ IO.popen(cmd, "r") {
+ | inp |
+ inp.each_line {
+ | line |
+ line.chomp!
+ if line =~ /^Running /
+ running[$~.post_match] = true
+ elsif line =~ /^PASS: /
+ didRun[$~.post_match] = true
+ elsif line =~ /^FAIL: /
+ didRun[$~.post_match] = true
+ didFail[$~.post_match] = true
else
- "%#{chars}s"%(str)
+ unless blankLine
+ print("\r" + " " * prevStringLength + "\r")
+ end
+ puts line
+ blankLine = true
end
- end
-
- string = ""
- string += "\r#{lpad(didRun.size, $runlist.size.to_s.size)}/#{$runlist.size}"
- unless didFail.empty?
- string += " (failed #{didFail.size})"
- end
- string += " "
- (running.size - didRun.size).times {
- string += "."
+
+ def lpad(str, chars)
+ str = str.to_s
+ if str.length > chars
+ str
+ else
+ "%#{chars}s"%(str)
+ end
+ end
+
+ string = ""
+ string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
+ unless didFail.empty?
+ string += " (failed #{didFail.size})"
+ end
+ string += " "
+ (running.size - didRun.size).times {
+ string += "."
+ }
+ if string.length < prevStringLength
+ print string
+ print(" " * (prevStringLength - string.length))
+ end
+ print string
+ prevStringLength = string.length
+ blankLine = false
+ $stdout.flush
}
- if string.length < prevStringLength
- print string
- print(" " * (prevStringLength - string.length))
- end
- print string
- prevStringLength = string.length
- blankLine = false
- $stdout.flush
}
- }
- puts
- raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
- end
-}
+ puts
+ raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
+ end
+ }
+end
-# Delete empty .out files to make life less confusing.
-$runlist.each {
- | plan |
- outputFilename = $outputDir + (plan.name + ".out")
- File.unlink outputFilename if FileTest.size(outputFilename) == 0
-}
+def detectFailures
+ raise if $bundle
-Dir.foreach(parallelDir) {
- | filename |
- next unless filename =~ /test_fail_/
- appendFailure($runlist[$~.post_match.to_i])
-}
+ Dir.foreach($parallelDir) {
+ | filename |
+ next unless filename =~ /test_fail_/
+ appendFailure($runlist[$~.post_match.to_i])
+ }
+end
+
+def compressBundle
+ cmd = "cd #{$outputDir}/.. && tar -czf payload.tar.gz #{$outputDir.basename}"
+ $stderr.puts ">> #{cmd}" if $verbosity >= 2
+ raise unless system(cmd)
+end
+
+FileUtils.rm_rf $outputDir if $outputDir.directory? and not $bundle
+Dir.mkdir($outputDir) unless $outputDir.directory?
+
+begin
+ File.delete($outputDir + "failed")
+rescue
+end
+
+$outputDir = $outputDir.realpath
+$parallelDir = $outputDir + ".parallel"
+
+prepareBundle unless $bundle
+prepareParallelTestRunner unless $bundle
+cleanParallelDirectory if $bundle
+cleanOldResults if $bundle
+runParallelTestRunner
+cleanEmptyResultFiles
+detectFailures unless $bundle
+compressBundle if $tarball
