Author: Carsten Senger <[email protected]>
Branch:
Changeset: r166:0c6207da7e15
Date: 2012-01-22 16:30 +0100
http://bitbucket.org/pypy/benchmarks/changeset/0c6207da7e15/
Log: Refactoring, support script params executeable, project, url
* Refactoring: move runner upload code from run_and_store and
perform_upload to main.
* factor out code to guess the executable
* Remove hardcoded values for project, executable and codespeed url
and add them as parameters to the runner script (--update-
executable,
--update-project and --upload-urls) and the saveresults script
(--executable, --project and --url)
* adjust parameters of saveresults.save and .send
* remove unused parameter options from save()
* adjust tests
diff --git a/runner.py b/runner.py
--- a/runner.py
+++ b/runner.py
@@ -10,73 +10,63 @@
from saveresults import save
from unladen_swallow import perf
+BENCHMARK_SET = ['richards', 'slowspitfire', 'django', 'spambayes',
+ 'rietveld', 'html5lib', 'ai']
+BENCHMARK_SET += perf._FindAllBenchmarks(benchmarks.__dict__).keys()
-def perform_upload(pypy_c_path, args, force_host, options, res, revision,
- changed=True, postfix='', branch='default', project='PyPy',
- executable=None):
- if executable is None:
- if "--jit" in args:
- executable = "pypy-c" + postfix
- else:
- executable = "pypy-c-jit" + postfix
+# executeablenames
+EX_CPY_PSYCO = "cpython psyco-profile"
+EX_PYPY_C = 'pypy-c'
+EX_PYPY_C_JIT = 'pypy-c-jit'
+
+
+class WrongBenchmark(Exception):
+ pass
+
+
+def guess_executeable(pypy_c_path, postfix, args, changed):
+
+ if ',' in args:
+ args_baseline, args_changed = args.split(',')
+ args = args_changed if changed else args_baseline
if "psyco.sh" in pypy_c_path:
- executable = "cpython psyco-profile"
- revision = 100
- project = 'cpython'
+ executable_prefix = EX_CPY_PSYCO
+ elif '--jit' in args:
+ executable_prefix = EX_PYPY_C
+ else:
+ executable_prefix = EX_PYPY_C_JIT
- host = force_host if force_host else socket.gethostname()
- print save(project, revision, res, options, executable, host,
- changed=changed, branch=branch)
+ return executable_prefix + postfix
def run_and_store(benchmark_set, result_filename, pypy_c_path, revision=0,
options='', branch='default', args='', upload=False,
- force_host=None, fast=False, baseline=sys.executable,
- full_store=False, postfix=''):
+ fast=False, baseline=sys.executable, full_store=False):
funcs = perf.BENCH_FUNCS.copy()
funcs.update(perf._FindAllBenchmarks(benchmarks.__dict__))
- opts = ['-b', ','.join(benchmark_set), '--inherit_env=PATH',
+ opts = ['-b', ','.join(benchmark_set),
+ '--inherit_env=PATH',
'--no_charts']
if fast:
opts += ['--fast']
if args:
opts += ['--args', args]
if full_store:
- opts.append('--no_statistics')
+ opts += ['--no_statistics']
opts += [baseline, pypy_c_path]
results = perf.main(opts, funcs)
f = open(str(result_filename), "w")
- res = [(name, result.__class__.__name__, result.__dict__)
+ results = [(name, result.__class__.__name__, result.__dict__)
for name, result in results]
f.write(json.dumps({
'revision': revision,
- 'results': res,
+ 'results': results,
'options': options,
'branch': branch,
}))
f.close()
-
- if upload:
- if ',' in args:
- argsbase, argschanged = args.split(',')
- else:
- argsbase, argschanged = args, args
- if 'pypy' in baseline:
- perform_upload(pypy_c_path, argsbase, force_host, options, res,
- revision, changed=False, postfix=postfix,
- branch=branch)
- perform_upload(pypy_c_path, argschanged, force_host, options, res,
- revision, changed=True, postfix=postfix, branch=branch)
-
-
-BENCHMARK_SET = ['richards', 'slowspitfire', 'django', 'spambayes',
- 'rietveld', 'html5lib', 'ai']
-BENCHMARK_SET += perf._FindAllBenchmarks(benchmarks.__dict__).keys()
-
-
-class WrongBenchmark(Exception):
- pass
+ return results
def main(argv):
@@ -110,8 +100,17 @@
" python, and the arguments after are passed to"
" the changed python. If there's no comma, the"
" same options are passed to both."))
- parser.add_option("--upload", default=False, action="store_true",
- help="Upload results to speed.pypy.org")
+ parser.add_option("--upload", default=None, action="store_true",
+ help=("Upload results to speed.pypy.org (unless "
+ "--upload-url is given)."))
+ parser.add_option("--upload-urls", default="http://speed.pypy.org/";,
+ help=("Comma seperated urls of the codespeed instance to"
+ " upload to (default: http://speed.pypy.org/)."))
+ parser.add_option("--upload-project", default="PyPy",
+ help="The project name in codespeed (default: PyPy).")
+ parser.add_option("--upload-executable", default=None,
+ help=("The executable name in codespeed "
+ "(guessed if possible and not given)."))
parser.add_option("--force-host", default=None, action="store",
help="Force the hostname")
parser.add_option("--fast", default=False, action="store_true",
@@ -130,11 +129,42 @@
for benchmark in benchmarks:
if benchmark not in BENCHMARK_SET:
raise WrongBenchmark(benchmark)
- run_and_store(benchmarks, options.output_filename, options.pypy_c,
- options.revision, args=options.args, upload=options.upload,
- force_host=options.force_host, fast=options.fast,
- baseline=options.baseline, full_store=options.full_store,
- postfix=options.postfix, branch=options.branch)
+
+ pypy_c_path = options.pypy_c
+ baseline = options.baseline
+ fast = options.fast
+ args = options.args
+ full_store = options.full_store
+ output_filename = options.output_filename
+
+ project = options.upload_project
+ executable = options.upload_executable
+ postfix = options.postfix
+ branch = options.branch
+ revision = options.revision
+
+ force_host = options.force_host
+
+ results = run_and_store(benchmarks, output_filename, pypy_c_path, revision,
+ args=args, fast=fast, baseline=baseline,
+ full_store=full_store, branch=branch)
+
+ if options.upload:
+ changed = 'pypy' not in baseline
+ if executable is None:
+ executable = guess_executeable(pypy_c_path, postfix, args, changed)
+
+ if executable == EX_CPY_PSYCO:
+ revision = 100
+ project = 'cpython'
+
+ changed = False
+
+ host = force_host if force_host else socket.gethostname()
+ for url in options.upload_urls.split(','):
+ print save(project, revision, results, executable, host, url,
+ changed=changed, branch=branch)
+
if __name__ == '__main__':
main(sys.argv[1:])
diff --git a/saveresults.py b/saveresults.py
--- a/saveresults.py
+++ b/saveresults.py
@@ -29,10 +29,7 @@
import urllib2
-SPEEDURL = "http://speed.pypy.org/";
-
-
-def save(project, revision, results, options, executeable, host, testing=False,
+def save(project, revision, results, executeable, host, url, testing=False,
changed=True, branch='default'):
testparams = []
#Parse data
@@ -85,7 +82,7 @@
if testing:
testparams.append(data)
else:
- error |= send(data)
+ error |= send(data, url)
if error:
raise IOError("Saving failed. See messages above.")
@@ -95,7 +92,7 @@
return 0
-def send(data):
+def send(data, url):
#save results
params = urllib.urlencode(data)
f = None
@@ -108,7 +105,7 @@
retries = [1, 2, 3, 6]
while True:
try:
- f = urllib2.urlopen(SPEEDURL + 'result/add/', params)
+ f = urllib2.urlopen(url + 'result/add/', params)
response = f.read()
f.close()
break
@@ -128,7 +125,7 @@
print response
with open('error.html', 'w') as error_file:
error_file.write(response)
- print("Server (%s) response written to error.html" % (SPEEDURL,))
+ print("Server (%s) response written to error.html" % (url,))
print(' Error code: %s\n' % (e,))
return 1
print "saved correctly!\n"
@@ -141,24 +138,35 @@
data = simplejson.load(f)
results = data['results']
print 'uploading results...',
- save('PyPy', options.revision, results, '', options.name, options.host,
- changed=options.changed)
+ save(options.project, options.revision, results, options.executable,
+ options.host, options.url, changed=options.changed)
print 'done'
if __name__ == '__main__':
parser = optparse.OptionParser(usage="%prog result.json [options]")
parser.add_option('-r', '--revision', dest='revision',
- default=None, type=str)
- parser.add_option('-n', '--name', dest='name', default=None, type=str)
+ default=None, type=str, help='VCS revision (required)')
+ parser.add_option('-n', '--name', dest='executable',
+ default=None, type=str,
+ help=('Name of the executable for codespeed.'
+ 'Deprecated. Use --e/--executable instead'))
+ parser.add_option('-e', '--executable', dest='executable',
+ default=None, type=str,
+ help='Name of the Executable for codespeed (required).')
parser.add_option('-H', '--host', dest='host', default=None, type=str)
parser.add_option('-b', '--baseline', dest='changed', default=True,
action='store_false',
help='upload the results as baseline instead of changed')
+ parser.add_option('-P', '--project', dest='project', default='PyPy')
+ parser.add_option('-u', '--url', dest='url',
+ default="http://speed.pypy.org/";,
+ help=('Url of the codespeed instance '
+ '(default: http://speed.pypy.org)'))
parser.format_description = lambda fmt: __doc__
parser.description = __doc__
options, args = parser.parse_args()
- if (options.revision is None or options.name is None or
+ if (options.revision is None or options.executable is None or
options.host is None or len(args) != 1):
parser.print_help()
sys.exit(2)
diff --git a/test/test_runner.py b/test/test_runner.py
--- a/test/test_runner.py
+++ b/test/test_runner.py
@@ -1,9 +1,9 @@
-
import py
import json
import sys
from runner import run_and_store
+
def test_run_and_store():
tmpdir = py.test.ensuretemp('bench_runner')
resfile = tmpdir.join('results')
diff --git a/test/test_saveresults.py b/test/test_saveresults.py
--- a/test/test_saveresults.py
+++ b/test/test_saveresults.py
@@ -22,7 +22,8 @@
def test_good_input(self):
'''Given correct result data, check that every result being saved has
the right parameters'''
- for resultparams in saveresults.save("PyPy", 71212, self.fixture, "",
"pypy-c-jit", 'host', True):
+ for resultparams in saveresults.save("PyPy", 71212, self.fixture,
"pypy-c-jit", 'host', 'url',
+ testing=True):
assert resultparams['project'] == "PyPy"
assert resultparams['commitid'] == 71212
assert resultparams['executable'] == "pypy-c-jit"
_______________________________________________
pypy-commit mailing list
[email protected]
http://mail.python.org/mailman/listinfo/pypy-commit
