Title: [286374] trunk/Tools
- Revision
- 286374
- Author
- w...@apple.com
- Date
- 2021-12-01 11:23:32 -0800 (Wed, 01 Dec 2021)
Log Message
[macOS] run-benchmark should take diagnostic screenshots upon test timeout
https://bugs.webkit.org/show_bug.cgi?id=232736
Reviewed by Dewei Zhu.
Added defaults for the --diagnose-directory flag depending on platform.
Added diagnose_test_failure method to OSXBrowserDriver to take screenshots and save to diagnostic directory.
* Scripts/webkitpy/benchmark_runner/browser_driver/osx_browser_driver.py:
(OSXBrowserDriver._save_screenshot_to_path):
(OSXBrowserDriver):
(OSXBrowserDriver.diagnose_test_failure):
Modified Paths
Diff
Modified: trunk/Tools/ChangeLog (286373 => 286374)
--- trunk/Tools/ChangeLog 2021-12-01 19:14:23 UTC (rev 286373)
+++ trunk/Tools/ChangeLog 2021-12-01 19:23:32 UTC (rev 286374)
@@ -1,3 +1,18 @@
+2021-12-01 W.D. Xiong <w...@apple.com>
+
+ [macOS] run-benchmark should take diagnostic screenshots upon test timeout
+ https://bugs.webkit.org/show_bug.cgi?id=232736
+
+ Reviewed by Dewei Zhu.
+
+ Added defaults for the --diagnose-directory flag depending on platform.
+ Added diagnose_test_failure method to OSXBrowserDriver to take screenshots and save to diagnostic directory.
+
+ * Scripts/webkitpy/benchmark_runner/browser_driver/osx_browser_driver.py:
+ (OSXBrowserDriver._save_screenshot_to_path):
+ (OSXBrowserDriver):
+ (OSXBrowserDriver.diagnose_test_failure):
+
2021-12-01 Jonathan Bedard <jbed...@apple.com>
[webkitscmpy] Do not query remotes when determining branches containing commit
Modified: trunk/Tools/Scripts/webkitpy/benchmark_runner/browser_driver/osx_browser_driver.py (286373 => 286374)
--- trunk/Tools/Scripts/webkitpy/benchmark_runner/browser_driver/osx_browser_driver.py 2021-12-01 19:14:23 UTC (rev 286373)
+++ trunk/Tools/Scripts/webkitpy/benchmark_runner/browser_driver/osx_browser_driver.py 2021-12-01 19:23:32 UTC (rev 286374)
@@ -1,6 +1,7 @@
import logging
import os
+import shutil
import subprocess
import time
@@ -38,6 +39,36 @@
def close_browsers(self):
self._terminate_processes(self.process_name, self.bundle_id)
+ def _save_screenshot_to_path(self, output_directory, filename):
+ jpg_image_path = os.path.join(output_directory, filename)
+ try:
+ subprocess.call(['screencapture', jpg_image_path])
+ _log.info('Saved screenshot to "{}"'.format(jpg_image_path))
+ except subprocess.CalledProcessError as error:
+ _log.error('Failed to save screenshot - Error: {error}'.format(error=error))
+
+ def diagnose_test_failure(self, diagnose_directory, error):
+ _log.info('Diagnosing benchmark failure: "{}"'.format(error))
+
+ if not diagnose_directory:
+ _log.info('Diagnose directory is not specified, will skip diagnosing.')
+ return
+
+ if os.path.exists(diagnose_directory):
+ _log.info('Diagnose directory: "{}" already exists, cleaning it up'.format(diagnose_directory))
+ try:
+ if os.path.isdir(diagnose_directory):
+ if len(os.listdir(diagnose_directory)):
+ shutil.rmtree(diagnose_directory)
+ elif os.path.isfile(diagnose_directory):
+ os.remove(diagnose_directory)
+ except Exception as error:
+ _log.error('Could not remove diagnose directory {} - error: {}'.format(diagnose_directory, error))
+ if not os.path.exists(diagnose_directory):
+ os.makedirs(diagnose_directory)
+
+ self._save_screenshot_to_path(diagnose_directory, 'test-failure-screenshot-{}.jpg'.format(int(time.time())))
+
@classmethod
def _launch_process(cls, build_dir, app_name, url, args):
if not build_dir:
Modified: trunk/Tools/Scripts/webkitpy/benchmark_runner/run_benchmark.py (286373 => 286374)
--- trunk/Tools/Scripts/webkitpy/benchmark_runner/run_benchmark.py 2021-12-01 19:14:23 UTC (rev 286373)
+++ trunk/Tools/Scripts/webkitpy/benchmark_runner/run_benchmark.py 2021-12-01 19:23:32 UTC (rev 286374)
@@ -5,6 +5,7 @@
import logging
import os
import sys
+import time
from webkitpy.benchmark_runner.browser_driver.browser_driver_factory import BrowserDriverFactory
from webkitpy.benchmark_runner.benchmark_runner import BenchmarkRunner
@@ -31,6 +32,10 @@
return 'safari'
+def default_diagnose_dir():
+ return '/tmp/run-benchmark-diagnostics-{}/'.format(int(time.time()))
+
+
def config_argument_parser():
parser = argparse.ArgumentParser(description='Run browser based performance benchmarks. To run a single benchmark in the recommended way, use run-benchmark --plan. To see the vailable benchmarks, use run-benchmark --list-plans. This script passes through the __XPC variables in its environment to the Safari process.')
mutual_group = parser.add_mutually_exclusive_group(required=True)
@@ -46,7 +51,7 @@
parser.add_argument('--local-copy', help='Path to a local copy of the benchmark (e.g. PerformanceTests/SunSpider/).')
parser.add_argument('--device-id', default=None, help='Undocumented option for mobile device testing.')
parser.add_argument('--debug', action='', help='Enable debug logging.')
- parser.add_argument('--diagnose-directory', dest='diagnose_dir', default=None, help='Directory for storing diagnose information on test failure. It\'s up to browser driver implementation when this option is not specified.')
+ parser.add_argument('--diagnose-directory', dest='diagnose_dir', default=default_diagnose_dir(), help='Directory for storing diagnose information on test failure. Defaults to {}.'.format(default_diagnose_dir()))
parser.add_argument('--no-adjust-unit', dest='scale_unit', action='', help="Don't convert to scientific notation.")
parser.add_argument('--show-iteration-values', dest='show_iteration_values', action='', help="Show the measured value for each iteration in addition to averages.")
_______________________________________________
webkit-changes mailing list
webkit-changes@lists.webkit.org
https://lists.webkit.org/mailman/listinfo/webkit-changes
