Diff
Modified: trunk/Tools/ChangeLog (280868 => 280869)
--- trunk/Tools/ChangeLog 2021-08-10 22:23:17 UTC (rev 280868)
+++ trunk/Tools/ChangeLog 2021-08-10 22:41:13 UTC (rev 280869)
@@ -1,3 +1,35 @@
+2021-08-09 Jonathan Bedard <jbed...@apple.com>
+
+ [resultsdbpy] Add results-summary API
+ https://bugs.webkit.org/show_bug.cgi?id=226894
+ <rdar://problem/79155181>
+
+ Reviewed by Aakash Jain.
+
+ * Scripts/libraries/resultsdbpy/resultsdbpy/__init__.py: Bump version.
+ * Scripts/libraries/resultsdbpy/resultsdbpy/controller/api_routes.py:
+ (APIRoutes.__init__): Add aggregate-results endpoint.
+ * Scripts/libraries/resultsdbpy/resultsdbpy/controller/commit_controller.py:
+ (commit_for_query): Add decorator which converts a set of arguments into a single commit.
+ * Scripts/libraries/resultsdbpy/resultsdbpy/controller/test_controller.py:
+ (TestController):
+ (TestController.summarize_test_results): Given a single commit and suite/test combination, compute
+ the liklihood of each potential result.
+ * Scripts/libraries/resultsdbpy/resultsdbpy/controller/test_controller_unittest.py:
+ (TestControllerTest.test_summarize_general): Added.
+ (TestControllerTest.test_summarize_specific): Added.
+ (TestControllerTest.test_summarize_expectations): Added.
+ * Scripts/libraries/resultsdbpy/resultsdbpy/model/commit_context.py:
+ (CommitContext.find_commits_in_range): Use ascended table if user only provides lower bound.
+ * Scripts/libraries/resultsdbpy/resultsdbpy/model/commit_context_unittest.py:
+ (CommitContextTest.test_stash_commits_before): Verify upper bound.
+ (CommitContextTest.test_svn_commits_before): Ditto.
+ (CommitContextTest.test_stash_commits_after): Verify lower bound.
+ (CommitContextTest.test_svn_commits_after): Ditto.
+ * Scripts/libraries/resultsdbpy/resultsdbpy/view/templates/documentation.html: Add aggregate-results
+ documentation.
+ * Scripts/libraries/resultsdbpy/setup.py: Bump version.
+
2021-08-10 Tim Horton <timothy_hor...@apple.com>
macCatalyst: Flexible viewport tests that dump the window size fail because it doesn't match iPad
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/__init__.py (280868 => 280869)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/__init__.py 2021-08-10 22:23:17 UTC (rev 280868)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/__init__.py 2021-08-10 22:41:13 UTC (rev 280869)
@@ -44,7 +44,7 @@
"Please install webkitcorepy with `pip install webkitcorepy --extra-index-url <package index URL>`"
)
-version = Version(3, 0, 2)
+version = Version(3, 1, 0)
import webkitflaskpy
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/api_routes.py (280868 => 280869)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/api_routes.py 2021-08-10 22:23:17 UTC (rev 280868)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/api_routes.py 2021-08-10 22:41:13 UTC (rev 280869)
@@ -70,6 +70,7 @@
self.add_url_rule('/results/<path:suite>', 'suite-results', self.suite_controller.find_run_results, methods=('GET',))
self.add_url_rule('/results/<path:suite>/<path:test>', 'test-results', self.test_controller.find_test_result, methods=('GET',))
+ self.add_url_rule('/results-summary/<path:suite>/<path:test>', 'test-aggregate-results', self.test_controller.summarize_test_results, methods=('GET',))
self.add_url_rule('/failures/<path:suite>', 'suite-failures', self.failure_controller.failures, methods=('GET',))
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/commit_controller.py (280868 => 280869)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/commit_controller.py 2021-08-10 22:23:17 UTC (rev 280868)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/commit_controller.py 2021-08-10 22:41:13 UTC (rev 280869)
@@ -125,6 +125,26 @@
return decorator
+def commit_for_query():
+ def decorator(method):
+ def real_method(obj, repository_id=None, branch=None, id=None, ref=None, uuid=None, timestamp=None, **kwargs):
+ # We're making an assumption that the class using this decorator actually has a commit_context, if it does not,
+ # this decorator will fail spectacularly
+ with obj.commit_context:
+ if not branch:
+ branch = [None]
+ return method(obj, branch=branch, commit=_find_comparison(
+ obj.commit_context, repository_id=repository_id, branch=branch,
+ ref=ref or id,
+ uuid=uuid, timestamp=timestamp, priority=max,
+ ), **kwargs)
+
+ real_method.__name__ = method.__name__
+ return real_method
+
+ return decorator
+
+
class HasCommitContext(object):
def __init__(self, commit_context):
self.commit_context = commit_context
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/test_controller.py (280868 => 280869)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/test_controller.py 2021-08-10 22:23:17 UTC (rev 280868)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/test_controller.py 2021-08-10 22:41:13 UTC (rev 280869)
@@ -21,10 +21,12 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from flask import abort, jsonify
-from resultsdbpy.controller.commit_controller import uuid_range_for_query, HasCommitContext
+from heapq import merge
+from resultsdbpy.controller.commit_controller import uuid_range_for_query, commit_for_query, HasCommitContext
from resultsdbpy.controller.configuration import Configuration
from resultsdbpy.controller.configuration_controller import configuration_for_query
from resultsdbpy.controller.suite_controller import time_range_for_query
+from resultsdbpy.model.test_context import Expectations
from webkitflaskpy.util import AssertRequest, query_as_kwargs, limit_for_query, boolean_query
@@ -102,3 +104,107 @@
results=sorted(results, key=sort_function),
))
return jsonify(response)
+
+ @query_as_kwargs()
+ @commit_for_query()
+ @limit_for_query(100)
+ @configuration_for_query()
+ def summarize_test_results(
+ self, suite=None, test=None,
+ configurations=None, recent=None,
+ branch=None, commit=None,
+ limit=None, include_expectations=None, **kwargs
+ ):
+ AssertRequest.is_type(['GET'])
+ AssertRequest.query_kwargs_empty(**kwargs)
+
+ recent = boolean_query(*recent)[0] if recent else True
+ include_expectations = boolean_query(*include_expectations)[0] if include_expectations else False
+
+ if not suite:
+ abort(400, description='No suite specified')
+ if not test:
+ abort(400, description='No test specified')
+
+ limit += 1
+ before_commits = []
+ after_commits = []
+ with self.commit_context:
+ for repo_id in self.commit_context.repositories.keys():
+ if commit:
+ before_commits = sorted(list(reversed(self.commit_context.find_commits_in_range(
+ repository_id=repo_id,
+ end=commit, branch=branch[0], limit=limit,
+ ))) + before_commits)
+ after_commits = sorted(list(reversed(self.commit_context.find_commits_in_range(
+ repository_id=repo_id,
+ begin=commit, branch=branch[0], limit=limit,
+ ))) + after_commits)
+ else:
+ before_commits = list(merge(
+ self.commit_context.find_commits_in_range(repository_id=repo_id, branch=branch[0], limit=limit),
+ before_commits,
+ ))
+ before_commits.reverse()
+ after_commits.reverse()
+
+ before_commits = sorted(before_commits)
+ after_commits = sorted(after_commits)
+
+ before_commits = before_commits[-limit:]
+ after_commits = after_commits[:limit]
+ if before_commits and after_commits and before_commits[-1] == after_commits[0]:
+ del before_commits[-1]
+
+ if not before_commits and not after_commits:
+ return abort(400, description='No commits in specified range')
+
+ # Use the linear distance from the specified commit
+ scale_for_uuid = {}
+ count = limit - 1
+ for c in reversed(before_commits):
+ scale_for_uuid[c.uuid] = count
+ count -= 1
+ count = limit
+ for c in after_commits:
+ scale_for_uuid[c.uuid] = count
+ count -= 1
+
+ # A direct match to the provided commit matters most
+ if commit:
+ scale_for_uuid[commit.uuid] = limit * 2
+
+ response = {}
+ for value in Expectations.STATE_ID_TO_STRING.values():
+ response[value.lower()] = {} if include_expectations else 0
+
+ with self.test_context:
+ for config, results in self.test_context.find_by_commit(
+ suite=suite, test=test,
+ configurations=configurations, recent=recent,
+ branch=branch[0],
+ limit=limit * 4,
+ begin=(before_commits or after_commits)[0], end=(after_commits or before_commits)[-1],
+ ).items():
+ for result in results:
+ scale = scale_for_uuid.get(int(result['uuid']), 0)
+ if not scale:
+ continue
+ tag = result.get('actual', 'PASS').lower()
+ if include_expectations:
+ expected = 'expected' if not result.get('expected') or result['actual'] == result['expected'] else 'unexpected'
+ response[tag][expected] = response[tag].get(expected, 0) + scale
+ else:
+ response[tag] = response[tag] + scale
+
+ aggregate = sum([sum(value.values()) if include_expectations else value for value in response.values()])
+ if not aggregate:
+ return abort(400, description='No results for specified test and configuration in provided commit range')
+ for key in response.keys():
+ if include_expectations:
+ for expectation in response[key].keys():
+ response[key][expectation] = 100 * response[key][expectation] // (aggregate or 1)
+ else:
+ response[key] = 100 * response[key] // (aggregate or 1)
+
+ return jsonify(response)
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/test_controller_unittest.py (280868 => 280869)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/test_controller_unittest.py 2021-08-10 22:23:17 UTC (rev 280868)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/test_controller_unittest.py 2021-08-10 22:41:13 UTC (rev 280869)
@@ -136,3 +136,59 @@
response = client.get(self.URL + f'/api/results/layout-tests/fast/encoding/css-link-charset.html?platform=iOS&style=Debug&recent=False&after_time={time.time() + 1}')
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.json()), 0)
+
+ @WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
+ @FlaskTestCase.run_with_webserver()
+ def test_summarize_general(self, client, **kwargs):
+ self.maxDiff = None
+ response = client.get(
+ self.URL + f'/api/results-summary/layout-tests/fast/encoding/css-link-charset.html?limit=2')
+ self.assertEqual(response.status_code, 200)
+ self.assertDictEqual(response.json(), {
+ 'audio': 0,
+ 'crash': 0,
+ 'error': 0,
+ 'fail': 0,
+ 'image': 0,
+ 'pass': 100,
+ 'text': 0,
+ 'timeout': 0,
+ 'warning': 0,
+ })
+
+ @WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
+ @FlaskTestCase.run_with_webserver()
+ def test_summarize_specific(self, client, **kwargs):
+ self.maxDiff = None
+ response = client.get(self.URL + f'/api/results-summary/layout-tests/fast/encoding/css-link-charset.html?ref=1abe25b443e9&limit=2')
+ self.assertEqual(response.status_code, 200)
+ self.assertDictEqual(response.json(), {
+ 'audio': 0,
+ 'crash': 0,
+ 'error': 0,
+ 'fail': 0,
+ 'image': 0,
+ 'pass': 100,
+ 'text': 0,
+ 'timeout': 0,
+ 'warning': 0,
+ })
+
+ @WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
+ @FlaskTestCase.run_with_webserver()
+ def test_summarize_expectations(self, client, **kwargs):
+ self.maxDiff = None
+ response = client.get(
+ self.URL + f'/api/results-summary/layout-tests/fast/encoding/css-link-charset.html?limit=2&include_expectations=True')
+ self.assertEqual(response.status_code, 200)
+ self.assertDictEqual(response.json(), {
+ 'audio': {},
+ 'crash': {},
+ 'error': {},
+ 'fail': {},
+ 'image': {},
+ 'pass': dict(expected=100),
+ 'text': {},
+ 'timeout': {},
+ 'warning': {},
+ })
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/commit_context.py (280868 => 280869)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/commit_context.py 2021-08-10 22:23:17 UTC (rev 280868)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/commit_context.py 2021-08-10 22:41:13 UTC (rev 280869)
@@ -213,16 +213,20 @@
if branch is None:
branch = self.repositories[repository_id].default_branch
+ use_ascending = begin and not end
begin = self.convert_to_uuid(begin)
end = self.convert_to_uuid(end, self.timestamp_to_uuid())
with self:
- return [model.to_commit() for model in self.cassandra.select_from_table(
- self.CommitByUuidDescending.__table_name__, limit=limit,
- repository_id=repository_id, branch=branch,
+ result = [model.to_commit() for model in self.cassandra.select_from_table(
+ self.CommitByUuidAscending.__table_name__ if use_ascending else self.CommitByUuidDescending.__table_name__,
+ limit=limit, repository_id=repository_id, branch=branch,
uuid__gte=begin,
uuid__lte=end,
)]
+ if use_ascending:
+ result.reverse()
+ return result
def _adjacent_commit(self, commit, ascending=True):
if not isinstance(commit, Commit):
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/commit_context_unittest.py (280868 => 280869)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/commit_context_unittest.py 2021-08-10 22:23:17 UTC (rev 280868)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/commit_context_unittest.py 2021-08-10 22:41:13 UTC (rev 280869)
@@ -180,6 +180,58 @@
self.assertEqual(commits, self.database.find_commits_in_range(repository_id='webkit', branch='main', begin=commits[-1], end=commits[0]))
@WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
+ def test_stash_commits_before(self, redis=StrictRedis, cassandra=CassandraContext):
+ with MockModelFactory.safari(), MockModelFactory.webkit():
+ self.init_database(redis=redis, cassandra=cassandra)
+ self.add_all_commits_to_database()
+
+ commits = [
+ self.stash_repository.commit(ref='1abe25b443e9'),
+ self.stash_repository.commit(ref='fff83bb2d917'),
+ self.stash_repository.commit(ref='9b8311f25a77'),
+ ]
+ self.assertEqual(commits, self.database.find_commits_in_range(repository_id='safari', branch='main', end=commits[0], limit=3))
+
+ @WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
+ def test_svn_commits_before(self, redis=StrictRedis, cassandra=CassandraContext):
+ with MockModelFactory.safari(), MockModelFactory.webkit():
+ self.init_database(redis=redis, cassandra=cassandra)
+ self.add_all_commits_to_database()
+
+ commits = [
+ self.svn_repository.commit(ref=6),
+ self.svn_repository.commit(ref=4),
+ self.svn_repository.commit(ref=2),
+ ]
+ self.assertEqual(commits, self.database.find_commits_in_range(repository_id='webkit', branch='main', end=commits[0], limit=3))
+
+ @WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
+ def test_stash_commits_after(self, redis=StrictRedis, cassandra=CassandraContext):
+ with MockModelFactory.safari(), MockModelFactory.webkit():
+ self.init_database(redis=redis, cassandra=cassandra)
+ self.add_all_commits_to_database()
+
+ commits = [
+ self.stash_repository.commit(ref='1abe25b443e9'),
+ self.stash_repository.commit(ref='fff83bb2d917'),
+ self.stash_repository.commit(ref='9b8311f25a77'),
+ ]
+ self.assertEqual(commits, self.database.find_commits_in_range(repository_id='safari', branch='main', begin=commits[-1], limit=3))
+
+ @WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
+ def test_svn_commits_after(self, redis=StrictRedis, cassandra=CassandraContext):
+ with MockModelFactory.safari(), MockModelFactory.webkit():
+ self.init_database(redis=redis, cassandra=cassandra)
+ self.add_all_commits_to_database()
+
+ commits = [
+ self.svn_repository.commit(ref=6),
+ self.svn_repository.commit(ref=4),
+ self.svn_repository.commit(ref=2),
+ ]
+ self.assertEqual(commits, self.database.find_commits_in_range(repository_id='webkit', branch='main', begin=commits[-1], limit=3))
+
+ @WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_commit_from_stash_repo(self, redis=StrictRedis, cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/view/templates/documentation.html (280868 => 280869)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/view/templates/documentation.html 2021-08-10 22:23:17 UTC (rev 280868)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/view/templates/documentation.html 2021-08-10 22:41:13 UTC (rev 280869)
@@ -56,10 +56,9 @@
${queries.map((query) => {
return `<div class="badge">
<div class="text block">
- <a class="text tiny" href=""
+ <a class="text tiny" href="" '-')}">${query}</a>
</div>
</div>`;
- //return `<button class="button" _onclick_="window.location.href = '';">${query}</button>`
}).join('')}
</div>
</div>
@@ -359,6 +358,20 @@
`where <configuration-object-a> and <configuration-object-b> are both ${localLink(['Query Parameters', 'Configuration'], 'configuration objects')}' and <run-a1>, <run-a2>, <run-b1> and <run-b2> are all the afformentioned single test result dictionary.`,
],
),
+ documentEndpoint(
+ '/api/results-summary/<suite>/<test>',
+ ['GET'],
+ ['Branch', 'Configuration', 'Include Expectations', 'Limit', 'Repository', 'Ref', 'UUID'],
+ [
+ `Compute the combined results of a given test by aggregating results from a set of runs surrounding a specific commit:`,
+ codeBlock('{\n' +
+ ' "pass": 80,\n' +
+ ' "fail": 15,\n' +
+ ' "crash": 5,\n' +
+ '}'),
+ `These results are always the weighted aggregation of results for the provided configurations, with weights to be understood as percent liklihood a test will have a certain result on a given revision.`,
+ ],
+ ),
], 'Failure Analysis': [
`Results databases provide a few APIs to assist in the investigation of test failures. These analysis endpoints aggregate data from multiple test runs for consumption by both humans and automated systems.`,
documentEndpoint(
@@ -508,6 +521,10 @@
codeBlock('recent=False'),
`to your query. The downside of searching all configurations is that the default behavior of searching all recent configurations if no query parameters are provided is disabled because the results database cannot search by an unbounded number of configurations.`,
],
+ 'Include Expectations': [
+ `Some endpoints return different results if the caller requests expectations be taken into consideration. By default, this flag is disabled, but may be enabled on supporting enpoints with this query:`,
+ codeBlock('include_expectations=False'),
+ ],
'Limit': [
`The underlying architecture of the results database does not allow unlimited query sizes. Most endpoints accept a limit query which looks like this:`,
codeBlock('limit=150'),
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/setup.py (280868 => 280869)
--- trunk/Tools/Scripts/libraries/resultsdbpy/setup.py 2021-08-10 22:23:17 UTC (rev 280868)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/setup.py 2021-08-10 22:41:13 UTC (rev 280869)
@@ -30,7 +30,7 @@
setup(
name='resultsdbpy',
- version='3.0.2',
+ version='3.1.0',
description='Library for visualizing, processing and storing test results.',
long_description=readme(),
classifiers=[
