Diff
Modified: trunk/Tools/ChangeLog (246302 => 246303)
--- trunk/Tools/ChangeLog 2019-06-11 01:14:23 UTC (rev 246302)
+++ trunk/Tools/ChangeLog 2019-06-11 05:05:19 UTC (rev 246303)
@@ -1,3 +1,25 @@
+2019-06-10 Dewei Zhu <dewei_...@apple.com>
+
+ Extend run-benchmark to allow diagnosing before closing browser on test failure.
+ https://bugs.webkit.org/show_bug.cgi?id=198729
+
+ Reviewed by Ryosuke Niwa.
+
+ Add '--diagnose-directory' option to store diagnose information when test failed.
+
+ * Scripts/webkitpy/benchmark_runner/benchmark_runner.py:
+ (BenchmarkRunner.__init__):
+ * Scripts/webkitpy/benchmark_runner/browser_driver/browser_driver.py:
+ (BrowserDriver.diagnose_test_failure): Add default no-op function to base class.
+ * Scripts/webkitpy/benchmark_runner/run_benchmark.py: Added '--diagnose-directory' option.
+ (parse_args):
+ (run_benchmark_plan):
+ * Scripts/webkitpy/benchmark_runner/webdriver_benchmark_runner.py:
+ (WebDriverBenchmarkRunner._run_one_test): Added 'diagnose_test_failure' invocation on test failure.
+ * Scripts/webkitpy/benchmark_runner/webserver_benchmark_runner.py:
+ (WebServerBenchmarkRunner.__init__):
+ (WebServerBenchmarkRunner._run_one_test): Added 'diagnose_test_failure' invocation on test failure.
+
2019-06-10 Sam Weinig <wei...@apple.com>
Remove Dashboard support
Modified: trunk/Tools/Scripts/webkitpy/benchmark_runner/benchmark_runner.py (246302 => 246303)
--- trunk/Tools/Scripts/webkitpy/benchmark_runner/benchmark_runner.py 2019-06-11 01:14:23 UTC (rev 246302)
+++ trunk/Tools/Scripts/webkitpy/benchmark_runner/benchmark_runner.py 2019-06-11 05:05:19 UTC (rev 246303)
@@ -23,7 +23,7 @@
class BenchmarkRunner(object):
name = 'benchmark_runner'
- def __init__(self, plan_file, local_copy, count_override, build_dir, output_file, platform, browser, browser_path, scale_unit=True, show_iteration_values=False, device_id=None):
+ def __init__(self, plan_file, local_copy, count_override, build_dir, output_file, platform, browser, browser_path, scale_unit=True, show_iteration_values=False, device_id=None, diagnose_dir=None):
try:
plan_file = self._find_plan_file(plan_file)
with open(plan_file, 'r') as fp:
@@ -38,6 +38,7 @@
self._browser_driver = BrowserDriverFactory.create(platform, browser)
self._browser_path = browser_path
self._build_dir = os.path.abspath(build_dir) if build_dir else None
+ self._diagnose_dir = os.path.abspath(diagnose_dir) if diagnose_dir else None
self._output_file = output_file
self._scale_unit = scale_unit
self._show_iteration_values = show_iteration_values
Modified: trunk/Tools/Scripts/webkitpy/benchmark_runner/browser_driver/browser_driver.py (246302 => 246303)
--- trunk/Tools/Scripts/webkitpy/benchmark_runner/browser_driver/browser_driver.py 2019-06-11 01:14:23 UTC (rev 246302)
+++ trunk/Tools/Scripts/webkitpy/benchmark_runner/browser_driver/browser_driver.py 2019-06-11 05:05:19 UTC (rev 246303)
@@ -42,6 +42,9 @@
def restore_env_after_all_testing(self):
pass
+ def diagnose_test_failure(self, debug_directory, error):
+ pass
+
@property
def webdriver_binary_path(self):
return get_driver_binary_path(self.browser_name)
Modified: trunk/Tools/Scripts/webkitpy/benchmark_runner/run_benchmark.py (246302 => 246303)
--- trunk/Tools/Scripts/webkitpy/benchmark_runner/run_benchmark.py 2019-06-11 01:14:23 UTC (rev 246302)
+++ trunk/Tools/Scripts/webkitpy/benchmark_runner/run_benchmark.py 2019-06-11 05:05:19 UTC (rev 246303)
@@ -47,6 +47,7 @@
parser.add_argument('--local-copy', help='Path to a local copy of the benchmark (e.g. PerformanceTests/SunSpider/).')
parser.add_argument('--device-id', default=None, help='Undocumented option for mobile device testing.')
parser.add_argument('--debug', action='', help='Enable debug logging.')
+ parser.add_argument('--diagnose-directory', dest='diagnose_dir', default=None, help='Directory for storing diagnose information on test failure. It\'s up to browser driver implementation when this option is not specified.')
parser.add_argument('--no-adjust-unit', dest='scale_unit', action='', help="Don't convert to scientific notation.")
parser.add_argument('--show-iteration-values', dest='show_iteration_values', action='', help="Show the measured value for each iteration in addition to averages.")
@@ -68,7 +69,7 @@
def run_benchmark_plan(args, plan):
benchmark_runner_class = benchmark_runner_subclasses[args.driver]
- runner = benchmark_runner_class(plan, args.local_copy, args.count, args.build_dir, args.output_file, args.platform, args.browser, args.browser_path, args.scale_unit, args.show_iteration_values, args.device_id)
+ runner = benchmark_runner_class(plan, args.local_copy, args.count, args.build_dir, args.output_file, args.platform, args.browser, args.browser_path, args.scale_unit, args.show_iteration_values, args.device_id, args.diagnose_dir)
runner.execute()
Modified: trunk/Tools/Scripts/webkitpy/benchmark_runner/webdriver_benchmark_runner.py (246302 => 246303)
--- trunk/Tools/Scripts/webkitpy/benchmark_runner/webdriver_benchmark_runner.py 2019-06-11 01:14:23 UTC (rev 246302)
+++ trunk/Tools/Scripts/webkitpy/benchmark_runner/webdriver_benchmark_runner.py 2019-06-11 05:05:19 UTC (rev 246303)
@@ -26,6 +26,9 @@
_log.info('Waiting on results from web browser')
result = WebDriverWait(driver, self._plan['timeout'], poll_frequency=1.0).until(self._get_result)
driver.quit()
+ except Exception as error:
+ self._browser_driver.diagnose_test_failure(self._diagnose_dir, error)
+ raise error
finally:
self._browser_driver.close_browsers()
Modified: trunk/Tools/Scripts/webkitpy/benchmark_runner/webserver_benchmark_runner.py (246302 => 246303)
--- trunk/Tools/Scripts/webkitpy/benchmark_runner/webserver_benchmark_runner.py 2019-06-11 01:14:23 UTC (rev 246302)
+++ trunk/Tools/Scripts/webkitpy/benchmark_runner/webserver_benchmark_runner.py 2019-06-11 05:05:19 UTC (rev 246303)
@@ -15,10 +15,10 @@
class WebServerBenchmarkRunner(BenchmarkRunner):
name = 'webserver'
- def __init__(self, plan_file, local_copy, count_override, build_dir, output_file, platform, browser, browser_path, scale_unit=True, show_iteration_values=False, device_id=None):
+ def __init__(self, plan_file, local_copy, count_override, build_dir, output_file, platform, browser, browser_path, scale_unit=True, show_iteration_values=False, device_id=None, diagnose_dir=None):
self._http_server_driver = HTTPServerDriverFactory.create(platform)
self._http_server_driver.set_device_id(device_id)
- super(WebServerBenchmarkRunner, self).__init__(plan_file, local_copy, count_override, build_dir, output_file, platform, browser, browser_path, scale_unit, show_iteration_values, device_id)
+ super(WebServerBenchmarkRunner, self).__init__(plan_file, local_copy, count_override, build_dir, output_file, platform, browser, browser_path, scale_unit, show_iteration_values, device_id, diagnose_dir)
def _get_result(self, test_url):
result = self._browser_driver.add_additional_results(test_url, self._http_server_driver.fetch_result())
@@ -33,6 +33,9 @@
self._browser_driver.launch_url(url, self._plan['options'], self._build_dir, self._browser_path)
with Timeout(self._plan['timeout']):
result = self._get_result(url)
+ except Exception as error:
+ self._browser_driver.diagnose_test_failure(self._diagnose_dir, error)
+ raise error
finally:
self._browser_driver.close_browsers()
self._http_server_driver.kill_server()
