Diff
Modified: trunk/Tools/ChangeLog (273998 => 273999)
--- trunk/Tools/ChangeLog 2021-03-05 21:09:42 UTC (rev 273998)
+++ trunk/Tools/ChangeLog 2021-03-05 21:11:19 UTC (rev 273999)
@@ -1,3 +1,38 @@
+2021-03-05 Jonathan Bedard <jbed...@apple.com>
+
+ [run-webkit-tests] Delete old results server code
+ https://bugs.webkit.org/show_bug.cgi?id=222393
+ <rdar://problem/74722220>
+
+ Rubber-stamped by Alexey Proskuryakov.
+
+ * Scripts/webkitpy/layout_tests/controllers/manager.py:
+ (Manager._end_test_run): Remove upload code.
+ (Manager._save_json_files): Renamed from _upload_json_files.
+ (Manager._upload_json_files): Renamed to _save_json_files.
+ (Manager.upload_results): Deleted.
+ * Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py:
+ (JSONLayoutResultsGenerator.__init__): Remove buildbot arguments.
+ * Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py:
+ (JSONResultsGenerator):
+ (JSONResultsGenerator.__init__): Remove buildbot arguments.
+ (JSONResultsGenerator.get_json): Remove concept of builder.
+ (JSONResultsGenerator.upload_json_files): Deleted.
+ (JSONResultsGenerator._get_svn_revision): Deleted.
+ (JSONResultsGenerator._get_archived_json_results): Deleted.
+ (JSONResultsGenerator._insert_generic_metadata): Deleted.
+ (JSONResultsGenerator._convert_json_to_current_version): Deleted.
+ (JSONResultsGenerator._convert_tests_to_trie): Deleted.
+ (JSONResultsGenerator._remove_items_over_max_number_of_builds): Deleted.
+ (JSONResultsGenerator._normalize_results_json): Deleted.
+ (JSONResultsGenerator._is_results_all_of_type): Deleted.
+ * Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py:
+ (JSONGeneratorTest.setUp):
+ (JSONGeneratorTest._test_json_generation):
+ (JSONGeneratorTest._verify_json_results):
+ * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+ (parse_args): Remove arguments used to upload to old results database.
+
2021-03-05 Chris Dumez <cdu...@apple.com>
Regression(r268097): WKWebView.URL is nil in the processDidTerminate delegate
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py (273998 => 273999)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2021-03-05 21:09:42 UTC (rev 273998)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2021-03-05 21:11:19 UTC (rev 273999)
@@ -399,9 +399,6 @@
_log.debug("summarizing results")
summarized_results = test_run_results.summarize_results(self._port, self._expectations, initial_results, retry_results, enabled_pixel_tests_in_retry)
- results_including_passes = None
- if self._options.results_server_host:
- results_including_passes = test_run_results.summarize_results(self._port, self._expectations, initial_results, retry_results, enabled_pixel_tests_in_retry, include_passes=True, include_time_and_modifiers=True)
self._printer.print_results(end_time - start_time, initial_results, summarized_results)
exit_code = -1
@@ -408,7 +405,7 @@
if not self._options.dry_run:
self._port.print_leaks_summary()
self._output_perf_metrics(end_time - start_time, initial_results)
- self._upload_json_files(summarized_results, initial_results, results_including_passes, start_time, end_time)
+ self._save_json_files(summarized_results, initial_results)
results_path = self._filesystem.join(self._results_directory, "results.html")
self._copy_results_html_file(results_path)
@@ -534,7 +531,7 @@
), results_trie)
return results_trie
- def _upload_json_files(self, summarized_results, initial_results, results_including_passes=None, start_time=None, end_time=None):
+ def _save_json_files(self, summarized_results, initial_results):
"""Writes the results of the test run as JSON files into the results
dir and upload the files to the appengine server.
@@ -557,17 +554,10 @@
# We write full_results.json out as jsonp because we need to load it from a file url and Chromium doesn't allow that.
json_results_generator.write_json(self._filesystem, summarized_results, full_results_path, callback="ADD_RESULTS")
- results_json_path = self._filesystem.join(self._results_directory, "results_including_passes.json")
- if results_including_passes:
- json_results_generator.write_json(self._filesystem, results_including_passes, results_json_path)
-
generator = json_layout_results_generator.JSONLayoutResultsGenerator(
- self._port, self._options.builder_name, self._options.build_name,
- self._options.build_number, self._results_directory,
+ self._port, self._results_directory,
self._expectations, initial_results,
- self._options.test_results_server,
- "layout-tests",
- self._options.master_name)
+ "layout-tests")
if generator.generate_json_output():
_log.debug("Finished writing JSON file for the test results server.")
@@ -575,12 +565,6 @@
_log.debug("Failed to generate JSON file for the test results server.")
return
- json_files = ["incremental_results.json", "full_results.json", "times_ms.json"]
-
- generator.upload_json_files(json_files)
- if results_including_passes:
- self.upload_results(results_json_path, start_time, end_time)
-
incremental_results_path = self._filesystem.join(self._results_directory, "incremental_results.json")
# Remove these files from the results directory so they don't take up too much space on the buildbot.
@@ -587,67 +571,7 @@
# The tools use the version we uploaded to the results server anyway.
self._filesystem.remove(times_json_path)
self._filesystem.remove(incremental_results_path)
- if results_including_passes:
- self._filesystem.remove(results_json_path)
- def upload_results(self, results_json_path, start_time, end_time):
- if not self._options.results_server_host:
- return
- master_name = self._options.master_name
- builder_name = self._options.builder_name
- build_number = self._options.build_number
- build_worker = self._options.build_slave
- if not master_name or not builder_name or not build_number or not build_worker:
- _log.error("--results-server-host was set, but --master-name, --builder-name, --build-number, or --build-slave was not. Not uploading JSON files.")
- return
-
- revisions = {}
- # FIXME: This code is duplicated in PerfTestRunner._generate_results_dict
- for (name, path) in self._port.repository_paths():
- scm = SCMDetector(self._port.host.filesystem, self._port.host.executive).detect_scm_system(path) or self._port.host.scm()
- revision = scm.native_revision(path)
- revisions[name] = {'revision': revision, 'timestamp': scm.timestamp_of_native_revision(path, revision)}
-
- for hostname in self._options.results_server_host:
- _log.info("Uploading JSON files for master: {} builder: {} build: {} worker: {} to {}".format(master_name, builder_name, build_number, build_worker, hostname))
-
- attrs = [
- ('master', 'build.webkit.org' if master_name == 'webkit.org' else master_name), # FIXME: Pass in build.webkit.org.
- ('builder_name', builder_name),
- ('build_number', build_number),
- ('build_slave', build_worker),
- ('revisions', json.dumps(revisions)),
- ('start_time', str(start_time)),
- ('end_time', str(end_time)),
- ]
-
- uploader = FileUploader("http://%s/api/report" % hostname, 360)
- try:
- response = uploader.upload_as_multipart_form_data(self._filesystem, [('results.json', results_json_path)], attrs)
- if not response:
- _log.error("JSON upload failed; no response returned")
- continue
-
- if response.code != 200:
- _log.error("JSON upload failed, %d: '%s'" % (response.code, response.read()))
- continue
-
- response_text = response.read()
- try:
- response_json = json.loads(response_text)
- except ValueError:
- _log.error("JSON upload failed; failed to parse the response: %s", response_text)
- continue
-
- if response_json['status'] != 'OK':
- _log.error("JSON upload failed, %s: %s", response_json['status'], response_text)
- continue
-
- _log.info("JSON uploaded.")
- except Exception as error:
- _log.error("Upload failed: %s" % error)
- continue
-
def _copy_results_html_file(self, destination_path):
base_dir = self._port.path_from_webkit_base('LayoutTests', 'fast', 'harness')
results_file = self._filesystem.join(base_dir, 'results.html')
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py (273998 => 273999)
--- trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py 2021-03-05 21:09:42 UTC (rev 273998)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py 2021-03-05 21:11:19 UTC (rev 273999)
@@ -53,10 +53,10 @@
test_expectations.LEAK: "L",
test_expectations.IMAGE_PLUS_TEXT: "Z"}
- def __init__(self, port, builder_name, build_name, build_number,
+ def __init__(self, port,
results_file_base_path,
expectations_by_type, run_results,
- test_results_servers=[], test_type="", master_name=""):
+ test_type=""):
"""Modifies the results.json file. Grabs it off the archive directory
if it is not found locally.
@@ -64,9 +64,9 @@
run_results: TestRunResults object storing the details of the test run.
"""
super(JSONLayoutResultsGenerator, self).__init__(
- port, builder_name, build_name, build_number, results_file_base_path,
+ port, results_file_base_path,
{}, port.repository_paths(),
- test_results_servers, test_type, master_name)
+ test_type)
self._expectations = expectations_by_type
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py (273998 => 273999)
--- trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py 2021-03-05 21:09:42 UTC (rev 273998)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py 2021-03-05 21:11:19 UTC (rev 273999)
@@ -241,11 +241,8 @@
TestResult.FAILS: FAIL_RESULT,
TestResult.FLAKY: FLAKY_RESULT}
- VERSION = 4
- VERSION_KEY = "version"
RESULTS = "results"
TIMES = "times"
- BUILD_NUMBERS = "buildNumbers"
TIME = "secondsSinceEpoch"
TESTS = "tests"
@@ -259,20 +256,15 @@
URL_FOR_TEST_LIST_JSON = "http://%s/testfile?builder=%s&name=%s&testlistjson=1&testtype=%s&master=%s"
- def __init__(self, port, builder_name, build_name, build_number,
+ def __init__(self, port,
results_file_base_path,
test_results_map, svn_repositories=None,
- test_results_server=[],
- test_type="",
- master_name=""):
+ test_type=""):
"""Modifies the results.json file. Grabs it off the archive directory
if it is not found locally.
Args
port: port-specific wrapper
- builder_name: the builder name (e.g. Webkit).
- build_name: the build name (e.g. webkit-rel).
- build_number: the build number.
results_file_base_path: Absolute path to the directory containing the
results json file.
test_results_map: A dictionary that maps test_name to TestResult.
@@ -279,16 +271,11 @@
svn_repositories: A (json_field_name, svn_path) pair for SVN
repositories that tests rely on. The SVN revision will be
included in the JSON with the given json_field_name.
- test_results_server: server that hosts test results json.
test_type: test type string (e.g. 'layout-tests').
- master_name: the name of the buildbot master.
"""
self._port = port
self._filesystem = port._filesystem
self._executive = port._executive
- self._builder_name = builder_name
- self._build_name = build_name
- self._build_number = build_number
self._results_directory = results_file_base_path
self._test_results_map = test_results_map
@@ -298,9 +285,7 @@
if not self._svn_repositories:
self._svn_repositories = {}
- self._test_results_servers = test_results_server
self._test_type = test_type
- self._master_name = master_name
self._archived_results = None
@@ -319,37 +304,9 @@
file_path = self._filesystem.join(self._results_directory, self.TIMES_MS_FILENAME)
write_json(self._filesystem, times, file_path)
- def get_json(self, server_index=0):
+ def get_json(self):
"""Gets the results for the results.json file."""
- results_json = {}
-
- if not results_json:
- results_json, error = self._get_archived_json_results(server_index=server_index)
- if error:
- # If there was an error don't write a results.json
- # file at all as it would lose all the information on the
- # bot.
- _log.error("Archive directory is inaccessible. Not "
- "modifying or clobbering the results.json "
- "file: " + str(error))
- return None
-
- builder_name = self._builder_name
- if results_json and builder_name not in results_json:
- _log.debug("Builder name (%s) is not in the results.json file."
- % builder_name)
-
- self._convert_json_to_current_version(results_json)
-
- if builder_name not in results_json:
- results_json[builder_name] = (
- self._create_results_for_builder_json())
-
- results_for_builder = results_json[builder_name]
-
- if builder_name:
- self._insert_generic_metadata(results_for_builder)
-
+ results_for_builder = self._create_results_for_builder_json()
self._insert_failure_summaries(results_for_builder)
# Update the all failing tests with result type and time.
@@ -360,47 +317,13 @@
for test in all_failing_tests:
self._insert_test_time_and_result(test, tests)
- return results_json
+ return dict(
+ results=results_for_builder,
+ )
def set_archived_results(self, archived_results):
self._archived_results = archived_results
- def upload_json_files(self, json_files):
- """Uploads the given json_files to the test_results_server (if the
- test_results_server is given)."""
- if not self._test_results_servers:
- return
-
- if not self._master_name:
- _log.error("--test-results-server was set, but --master-name was not. Not uploading JSON files.")
- return
-
- _log.info("Uploading JSON files for builder: %s", self._builder_name)
- attrs = [("builder", self._builder_name),
- ("testtype", self._test_type),
- ("master", self._master_name)]
-
- files = [(file, self._filesystem.join(self._results_directory, file))
- for file in json_files]
-
- for test_results_server in self._test_results_servers:
- url = "" % test_results_server
- # Set uploading timeout in case appengine server is having problems.
- # 120 seconds are more than enough to upload test results.
- uploader = FileUploader(url, 120)
- try:
- response = uploader.upload_as_multipart_form_data(self._filesystem, files, attrs)
- if response:
- if response.code == 200:
- _log.info("JSON uploaded.")
- else:
- _log.debug("JSON upload failed, %d: '%s'" % (response.code, response.read()))
- else:
- _log.error("JSON upload failed; no response returned")
- except Exception as err:
- _log.error("Upload failed: %s" % err)
- continue
-
def _get_test_timing(self, test_name):
"""Returns test timing data (elapsed time) in second
for the given test_name."""
@@ -444,69 +367,6 @@
return self.__class__.PASS_RESULT
- def _get_svn_revision(self, in_directory):
- """Returns the svn revision for the given directory.
-
- Args:
- in_directory: The directory where svn is to be run.
- """
- scm = SCMDetector(self._filesystem, self._executive).detect_scm_system(in_directory)
- if scm:
- return scm.svn_revision(in_directory)
- return ""
-
- def _get_archived_json_results(self, server_index=0):
- """Download JSON file that only contains test
- name list from test-results server. This is for generating incremental
- JSON so the file generated has info for tests that failed before but
- pass or are skipped from current run.
-
- Returns (archived_results, error) tuple where error is None if results
- were successfully read.
- """
- results_json = {}
- old_results = None
- error = None
-
- if len(self._test_results_servers) <= server_index:
- return {}, None
-
- results_file_url = (self.URL_FOR_TEST_LIST_JSON %
- (quote(self._test_results_servers[server_index]),
- quote(self._builder_name),
- self.RESULTS_FILENAME,
- quote(self._test_type),
- quote(self._master_name)))
-
- try:
- # FIXME: We should talk to the network via a Host object.
- results_file = urlopen(results_file_url)
- info = results_file.info()
- old_results = results_file.read()
- except HTTPError as http_error:
- # A non-4xx status code means the bot is hosed for some reason
- # and we can't grab the results.json file off of it.
- if (http_error.code < 400 and http_error.code >= 500):
- error = http_error
- except URLError as url_error:
- error = url_error
-
- if old_results:
- # Strip the prefix and suffix so we can get the actual JSON object.
- old_results = strip_json_wrapper(old_results)
-
- try:
- results_json = json.loads(old_results)
- except:
- _log.debug("results.json was not valid JSON. Clobbering.")
- # The JSON file is not valid JSON. Just clobber the results.
- results_json = {}
- else:
- _log.debug('Old JSON results do not exist. Starting fresh.')
- results_json = {}
-
- return results_json, error
-
def _insert_failure_summaries(self, results_for_builder):
"""Inserts aggregate pass/failure statistics into the JSON.
This method reads self._test_results and generates
@@ -572,23 +432,6 @@
# we want the serialized form to be concise.
encoded_results.insert(0, [1, item])
- def _insert_generic_metadata(self, results_for_builder):
- """ Inserts generic metadata (such as version number, current time etc)
- into the JSON.
-
- Args:
- results_for_builder: Dictionary containing the test results for
- a single builder.
- """
- self._insert_item_into_raw_list(results_for_builder,
- self._build_number, self.BUILD_NUMBERS)
-
- # Include SVN revisions for the given repositories.
- for (name, path) in self._svn_repositories:
- self._insert_item_into_raw_list(results_for_builder, self._get_svn_revision(path), name.lower() + 'Revision')
-
- self._insert_item_into_raw_list(results_for_builder, int(time.time()), self.TIME)
-
def _insert_test_time_and_result(self, test_name, tests):
""" Insert a test item with its results to the given tests dictionary.
@@ -618,37 +461,6 @@
else:
this_test[self.TIMES] = [[1, time]]
- def _convert_json_to_current_version(self, results_json):
- """If the JSON does not match the current version, converts it to the
- current version and adds in the new version number.
- """
- if self.VERSION_KEY in results_json:
- archive_version = results_json[self.VERSION_KEY]
- if archive_version == self.VERSION:
- return
- else:
- archive_version = 3
-
- # version 3->4
- if archive_version == 3:
- num_results = len(results_json.values())
- for builder, results in iteritems(results_json):
- self._convert_tests_to_trie(results)
-
- results_json[self.VERSION_KEY] = self.VERSION
-
- def _convert_tests_to_trie(self, results):
- if not self.TESTS in results:
- return
-
- test_results = results[self.TESTS]
- test_results_trie = {}
- for test in iterkeys(test_results):
- single_test_result = test_results[test]
- add_path_to_trie(test, single_test_result, test_results_trie)
-
- results[self.TESTS] = test_results_trie
-
def _populate_results_and_times_json(self, results_and_times):
results_and_times[self.RESULTS] = []
results_and_times[self.TIMES] = []
@@ -658,51 +470,3 @@
results_for_builder = {}
results_for_builder[self.TESTS] = {}
return results_for_builder
-
- def _remove_items_over_max_number_of_builds(self, encoded_list):
- """Removes items from the run-length encoded list after the final
- item that exceeds the max number of builds to track.
-
- Args:
- encoded_results: run-length encoded results. An array of arrays, e.g.
- [[3,'A'],[1,'Q']] encodes AAAQ.
- """
- num_builds = 0
- index = 0
- for result in encoded_list:
- num_builds = num_builds + result[0]
- index = index + 1
- if num_builds > self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG:
- return encoded_list[:index]
- return encoded_list
-
- def _normalize_results_json(self, test, test_name, tests):
- """ Prune tests where all runs pass or tests that no longer exist and
- truncate all results to maxNumberOfBuilds.
-
- Args:
- test: ResultsAndTimes object for this test.
- test_name: Name of the test.
- tests: The JSON object with all the test results for this builder.
- """
- test[self.RESULTS] = self._remove_items_over_max_number_of_builds(
- test[self.RESULTS])
- test[self.TIMES] = self._remove_items_over_max_number_of_builds(
- test[self.TIMES])
-
- is_all_pass = self._is_results_all_of_type(test[self.RESULTS],
- self.PASS_RESULT)
- is_all_no_data = self._is_results_all_of_type(test[self.RESULTS],
- self.NO_DATA_RESULT)
- max_time = max([time[1] for time in test[self.TIMES]])
-
- # Remove all passes/no-data from the results to reduce noise and
- # filesize. If a test passes every run, but takes > MIN_TIME to run,
- # don't throw away the data.
- if is_all_no_data or (is_all_pass and max_time <= self.MIN_TIME):
- del tests[test_name]
-
- def _is_results_all_of_type(self, results, type):
- """Returns whether all the results are of the given type
- (e.g. all passes)."""
- return len(results) == 1 and results[0][1] == type
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py (273998 => 273999)
--- trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py 2021-03-05 21:09:42 UTC (rev 273998)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py 2021-03-05 21:11:19 UTC (rev 273999)
@@ -40,10 +40,6 @@
class JSONGeneratorTest(unittest.TestCase):
def setUp(self):
- self.builder_name = 'DUMMY_BUILDER_NAME'
- self.build_name = 'DUMMY_BUILD_NAME'
- self.build_number = 'DUMMY_BUILDER_NUMBER'
-
# For archived results.
self._json = None
self._num_runs = 0
@@ -91,8 +87,7 @@
host = MockHost()
port = Mock()
port._filesystem = host.filesystem
- generator = json_results_generator.JSONResultsGenerator(port,
- self.builder_name, self.build_name, self.build_number, '', test_results_map)
+ generator = json_results_generator.JSONResultsGenerator(port, '', test_results_map)
failed_count_map = dict([(t, 1) for t in failed_tests])
@@ -120,14 +115,9 @@
# Aliasing to a short name for better access to its constants.
JRG = json_results_generator.JSONResultsGenerator
- self.assertIn(JRG.VERSION_KEY, json)
- self.assertIn(self.builder_name, json)
-
- buildinfo = json[self.builder_name]
+ buildinfo = json['results']
self.assertIn(JRG.FIXABLE, buildinfo)
self.assertIn(JRG.TESTS, buildinfo)
- self.assertEqual(len(buildinfo[JRG.BUILD_NUMBERS]), num_runs)
- self.assertEqual(buildinfo[JRG.BUILD_NUMBERS][0], self.build_number)
if tests_set or DISABLED_count:
fixable = {}
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py (273998 => 273999)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py 2021-03-05 21:09:42 UTC (rev 273998)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py 2021-03-05 21:11:19 UTC (rev 273999)
@@ -344,25 +344,6 @@
optparse.make_option("--wptserver-doc-root", type="string", help=("Set web platform server document root, relative to LayoutTests directory")),
]))
- # FIXME: Remove this group once the old results dashboards are deprecated.
- option_group_definitions.append(("Legacy Result Options", [
- optparse.make_option("--master-name", help="The name of the buildbot master."),
- optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
- help=("The name of the builder used in its path, e.g. webkit-rel.")),
- optparse.make_option("--build-slave", default="DUMMY_BUILD_SLAVE",
- help=("The name of the worker used. e.g. apple-macpro-6.")),
- optparse.make_option("--test-results-server", action="" default=[],
- help=("If specified, upload results json files to this appengine server.")),
- optparse.make_option("--results-server-host", action="" default=[],
- help=("If specified, upload results JSON file to this results server.")),
- optparse.make_option("--additional-repository-name",
- help=("The name of an additional subversion or git checkout")),
- optparse.make_option("--additional-repository-path",
- help=("The path to an additional subversion or git checkout (requires --additional-repository-name)")),
- optparse.make_option("--allowed-host", type="string", action="" default=[],
- help=("If specified, tests are allowed to make requests to the specified hostname."))
- ]))
-
option_group_definitions.append(('Upload Options', upload_options()))
option_parser = optparse.OptionParser(usage="%prog [options] [<path>...]")
