Diff
Modified: trunk/Tools/ChangeLog (274541 => 274542)
--- trunk/Tools/ChangeLog 2021-03-17 00:52:10 UTC (rev 274541)
+++ trunk/Tools/ChangeLog 2021-03-17 00:53:41 UTC (rev 274542)
@@ -1,5 +1,61 @@
2021-03-16 Jonathan Bedard <jbed...@apple.com>
+ [resultsdbpy] Save identifiers in commit table
+ https://bugs.webkit.org/show_bug.cgi?id=223101
+ <rdar://problem/75338338>
+
+ Reviewed by Dewei Zhu.
+
+ * Scripts/libraries/resultsdbpy/resultsdbpy/__init__.py: Bump version.
+ * Scripts/libraries/resultsdbpy/resultsdbpy/controller/commit_controller.py:
+ (CommitController.register): Accept uploaded new commit objects, continue to return old-style commits.
+ * Scripts/libraries/resultsdbpy/resultsdbpy/controller/upload_controller.py:
+ (UploadController.upload): Register commits using the fast path when uploading results.
+ * Scripts/libraries/resultsdbpy/resultsdbpy/controller/upload_controller_unittest.py:
+ (UploadControllerTest.test_process): Mock safari and WebKit when processing results.
+ (UploadControllerTest.test_process_commit): Ditto.
+ * Scripts/libraries/resultsdbpy/resultsdbpy/model/commit_context.py:
+ (CommitContext.CommitModelMk2): Add new style commit model base.
+ (CommitContext.CommitByRef): Sort commits by hash, revision and identifier references.
+ (CommitContext.CommitByUuidAscendingMk2): New commit style UUID table.
+ (CommitContext.CommitByUuidDescendingMk2): Ditto.
+ (CommitContext.__init__): Create new tables.
+ (CommitContext.find_commits_by_ref): Search for commits in the new ref table.
+ (CommitContext.register): Post-process commits to populate identifier field.
+ (CommitContext.register_commit): Only register new-style commits.
+ (CommitContext.register_partial_commit): Register new-style commits, allow caller to opt-out of expensive
+ Identifier computation.
+ (CommitContext.url): Accept old and new style commits.
+ (CommitContext.register_commit_with_repo_and_id): Deleted.
+ * Scripts/libraries/resultsdbpy/resultsdbpy/model/commit_context_unittest.py:
+ (CommitContextTest.add_all_commits_to_database): Update commit registration call.
+ (CommitContextTest.test_commit_from_stash_repo): Ditto.
+ (CommitContextTest.test_commit_from_svn_repo): Ditto.
+ * Scripts/libraries/resultsdbpy/resultsdbpy/model/mock_model_factory.py:
+ (MockModelFactory.create): Update commit registration call.
+ * Scripts/libraries/resultsdbpy/resultsdbpy/model/model.py:
+ (Model.__init__): commit_context may now post-process uploads.
+ * Scripts/libraries/resultsdbpy/resultsdbpy/model/repository.py:
+ (Repository.commit): Add call to return new-style commits.
+ (StashRepository.commit): Ditto.
+ (WebKitRepository.commit): Ditto.
+ * Scripts/libraries/resultsdbpy/resultsdbpy/model/repository_unittest.py:
+ (RepositoryTest.test_svn):
+ (RepositoryTest.test_ref_svn):
+ (RepositoryTest.test_stash):
+ (RepositoryTest.test_ref_stash):
+ (RepositoryTest.test_colliding_timestamps_stash):
+ (RepositoryTest.test_branch_stash):
+ (RepositoryTest.test_branch_svn): Deleted.
+ * Scripts/libraries/resultsdbpy/resultsdbpy/model/upload_context_unittest.py:
+ (UploadContextTest.test_callback):
+ (UploadContextTest.test_async_callback):
+ * Scripts/libraries/resultsdbpy/setup.py: Bump version.
+ * Scripts/libraries/webkitscmpy/webkitscmpy/commit.py:
+ (Commit): Handle single email passed to contributor.
+
+2021-03-16 Jonathan Bedard <jbed...@apple.com>
+
[resultsdbpy] Allow user to change commit representation
https://bugs.webkit.org/show_bug.cgi?id=223215
<rdar://problem/75446602>
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/__init__.py (274541 => 274542)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/__init__.py 2021-03-17 00:52:10 UTC (rev 274541)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/__init__.py 2021-03-17 00:53:41 UTC (rev 274542)
@@ -44,6 +44,6 @@
"Please install webkitcorepy with `pip install webkitcorepy --extra-index-url <package index URL>`"
)
-version = Version(2, 0, 2)
+version = Version(2, 0, 3)
name = 'resultsdbpy'
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/commit_controller.py (274541 => 274542)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/commit_controller.py 2021-03-17 00:52:10 UTC (rev 274541)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/commit_controller.py 2021-03-17 00:53:41 UTC (rev 274542)
@@ -1,4 +1,4 @@
-# Copyright (C) 2019 Apple Inc. All rights reserved.
+# Copyright (C) 2019-2021 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
@@ -26,7 +26,7 @@
from flask import abort, jsonify, request
from resultsdbpy.controller.commit import Commit
from webkitflaskpy.util import AssertRequest, query_as_kwargs, limit_for_query
-from webkitscmpy import ScmBase
+from webkitscmpy import ScmBase, Commit as ScmCommit
def _find_comparison(commit_context, repository_id, branch, id, uuid, timestamp, priority=min):
@@ -265,7 +265,7 @@
commit = self.commit_context.previous_commit(commits[0])
return jsonify(Commit.Encoder().default([commit] if commit else []))
- def register(self, commit=None):
+ def register(self, commit=None, fast=True):
is_endpoint = not bool(commit)
if is_endpoint:
AssertRequest.is_type(['POST'])
@@ -280,31 +280,48 @@
abort(400, description='Expected uploaded data to be json')
try:
- self.commit_context.register_commit(Commit.from_json(commit))
- if is_endpoint:
- return jsonify({'status': 'ok'})
- return Commit.from_json(commit)
+ candidate = ScmCommit.from_json(commit)
+
+ # Commit needs to be sufficiently defined
+ if candidate.repository_id and candidate.branch and candidate.timestamp and (candidate.revision or candidate.hash):
+ self.commit_context.register_commit(candidate)
+ if is_endpoint:
+ return jsonify({'status': 'ok'})
+ return candidate
except ValueError:
pass
- required_args = ['repository_id', 'id']
- optional_args = ['branch']
+ required_args = ['repository_id']
for arg in required_args:
if arg not in commit:
abort(400, description=f"'{arg}' required to define commit")
+ has_ref = False
+ _one_of_args_ = ['id', 'ref', 'hash', 'revision', 'identifier']
+ for arg in one_of_args:
+ if arg in commit:
+ if has_ref:
+ abort(400, description='Multiple commit references specified')
+ has_ref = True
+ if not has_ref:
+ abort(400, description='No commit reference specified')
+
+
for arg in commit.keys():
- if arg in required_args + optional_args:
+ if arg in required_args or arg in one_of_args:
continue
- if arg in ['timestamp', 'order', 'committer', 'message']:
+ if arg in ['branch', 'timestamp', 'order', 'committer', 'message']:
abort(400, description='Not enough arguments provided to define a commit, but too many to search for a commit')
abort(400, description=f"'{arg}' is not valid for defining commits")
try:
- commit = self.commit_context.register_commit_with_repo_and_id(
+ commit = self.commit_context.register_partial_commit(
repository_id=commit.get('repository_id'),
- branch=commit.get('branch'),
- commit_id=commit.get('id'),
+ ref=commit.get('id') or commit.get('ref'),
+ hash=commit.get('hash'),
+ revision=commit.get('revision'),
+ identifier=commit.get('identifier'),
+ fast=fast,
)
except (RuntimeError, ScmBase.Exception) as error:
abort(404, description=str(error))
@@ -311,4 +328,12 @@
if is_endpoint:
return jsonify({'status': 'ok'})
- return commit
+ return Commit(
+ repository_id=commit.repository_id,
+ id=commit.revision or commit.hash,
+ branch=commit.branch,
+ timestamp=commit.timestamp,
+ order=commit.order,
+ committer=commit.author.email if commit.author else None,
+ message=commit.message,
+ )
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/upload_controller.py (274541 => 274542)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/upload_controller.py 2021-03-17 00:52:10 UTC (rev 274541)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/upload_controller.py 2021-03-17 00:53:41 UTC (rev 274542)
@@ -25,7 +25,6 @@
from collections import defaultdict
from flask import abort, jsonify, request
-from resultsdbpy.controller.commit import Commit
from resultsdbpy.controller.commit_controller import uuid_range_for_query, HasCommitContext
from resultsdbpy.controller.configuration import Configuration
from resultsdbpy.controller.configuration_controller import configuration_for_query
@@ -84,7 +83,7 @@
response.append(dict(
configuration=Configuration.Encoder().default(config),
suite=suite,
- commits=Commit.Encoder().default(result['commits']),
+ commits=[commit.Encoder().default(commit) for commit in result['commits']],
timestamp=result['timestamp'],
test_results=result['test_results'],
))
@@ -113,7 +112,7 @@
if not suite:
abort(400, description='No test suite specified')
- commits = [self.commit_controller.register(commit=commit) for commit in data.get('commits', [])]
+ commits = [self.commit_controller.register(commit=commit, fast=True) for commit in data.get('commits', [])]
test_results = data.get('test_results', {})
if not test_results:
@@ -150,7 +149,7 @@
response.append(dict(
configuration=Configuration.Encoder().default(config),
suite=suite,
- commits=Commit.Encoder().default(result['commits']),
+ commits=[commit.Encoder().default(commit) for commit in result['commits']],
timestamp=result['timestamp'],
processing=processing_results,
))
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/upload_controller_unittest.py (274541 => 274542)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/upload_controller_unittest.py 2021-03-17 00:52:10 UTC (rev 274541)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/controller/upload_controller_unittest.py 2021-03-17 00:53:41 UTC (rev 274542)
@@ -49,7 +49,7 @@
def test_upload(self, client, **kwargs):
upload_dict = dict(
suite='layout-tests',
- commits=[dict(repository_id='safari', id='d8bce26fa65c6fc8f39c17927abb77f69fab82fc'), dict(repository_id='webkit', id='6')],
+ commits=[dict(repository_id='safari', id='d8bce26fa65c6fc8f39c17927abb77f69fab82fc'), dict(repository_id='webkit', id=6)],
configuration=Configuration.Encoder().default(Configuration(
platform='Mac', version='10.14.0', sdk='18A391',
is_simulator=False, architecture='x86_64',
@@ -67,7 +67,7 @@
self.assertEqual(response.status_code, 200)
retrieved = response.json()[0]
- retrieved['commits'] = [dict(repository_id=commit['repository_id'], id=commit['id']) for commit in retrieved['commits']]
+ retrieved['commits'] = [dict(repository_id=commit['repository_id'], id=commit.get('hash', commit.get('revision'))) for commit in retrieved['commits']]
self.assertEqual(retrieved, upload_dict)
@WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
@@ -180,7 +180,7 @@
'd8bce26fa65c6fc8f39c17927abb77f69fab82fc',
'bae5d1e90999d4f916a8a15810ccfa43f37a2fd6',
'1abe25b443e985f93b90d830e4a7e3731336af4d',
- ]), sorted([result['commits'][1]['id'] for result in response.json()]))
+ ]), sorted([result['commits'][1]['hash'] for result in response.json()]))
@WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
@FlaskTestCase.run_with_webserver()
@@ -188,26 +188,28 @@
response = client.get(self.URL + '/api/upload?platform=Mac&style=Release&flavor=wk2&id=d8bce26fa65c')
self.assertEqual(response.status_code, 200)
self.assertEqual(1, len(response.json()))
- self.assertEqual(['d8bce26fa65c6fc8f39c17927abb77f69fab82fc'], [result['commits'][1]['id'] for result in response.json()])
+ self.assertEqual(['d8bce26fa65c6fc8f39c17927abb77f69fab82fc'], [result['commits'][1]['hash'] for result in response.json()])
response = client.get(self.URL + '/api/upload?platform=Mac&style=Release&flavor=wk2&id=6')
self.assertEqual(response.status_code, 200)
self.assertEqual(5, len(response.json()))
- self.assertEqual(['6'] * 5, [result['commits'][0]['id'] for result in response.json()])
+ self.assertEqual([6] * 5, [result['commits'][0]['revision'] for result in response.json()])
@WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
@FlaskTestCase.run_with_webserver()
def test_process(self, client, **kwargs):
- response = client.post(self.URL + '/api/upload/process?platform=Mac&style=Release&flavor=wk2')
- self.assertEqual(response.status_code, 200)
- self.assertEqual(5, len(response.json()))
- self.assertEqual([dict(status='ok')] * 5, [element['processing']['python-tests'] for element in response.json()])
+ with MockModelFactory.safari(), MockModelFactory.webkit():
+ response = client.post(self.URL + '/api/upload/process?platform=Mac&style=Release&flavor=wk2')
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(5, len(response.json()))
+ self.assertEqual([dict(status='ok')] * 5, [element['processing']['python-tests'] for element in response.json()])
@WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
@FlaskTestCase.run_with_webserver()
def test_process_commit(self, client, **kwargs):
- response = client.post(self.URL + '/api/upload/process?platform=Mac&style=Release&flavor=wk2&id=d8bce26fa65c')
- self.assertEqual(response.status_code, 200)
- self.assertEqual(1, len(response.json()))
- self.assertEqual(['d8bce26fa65c6fc8f39c17927abb77f69fab82fc'], [result['commits'][1]['id'] for result in response.json()])
- self.assertEqual([dict(status='ok')], [element['processing']['python-tests'] for element in response.json()])
+ with MockModelFactory.safari(), MockModelFactory.webkit():
+ response = client.post(self.URL + '/api/upload/process?platform=Mac&style=Release&flavor=wk2&id=d8bce26fa65c')
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(1, len(response.json()))
+ self.assertEqual(['d8bce26fa65c6fc8f39c17927abb77f69fab82fc'], [result['commits'][1]['hash'] for result in response.json()])
+ self.assertEqual([dict(status='ok')], [element['processing']['python-tests'] for element in response.json()])
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/commit_context.py (274541 => 274542)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/commit_context.py 2021-03-17 00:52:10 UTC (rev 274541)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/commit_context.py 2021-03-17 00:53:41 UTC (rev 274542)
@@ -1,4 +1,4 @@
-# Copyright (C) 2019 Apple Inc. All rights reserved.
+# Copyright (C) 2019-2021 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
@@ -30,7 +30,7 @@
from resultsdbpy.controller.commit import Commit
from resultsdbpy.model.repository import Repository
-from webkitscmpy import ScmBase
+from webkitscmpy import ScmBase, Commit as ScmCommit, Contributor
class CommitContext(object):
@@ -65,6 +65,43 @@
uuid = columns.BigInt(primary_key=True, required=True, clustering_order='DESC')
commit_id = columns.Text(required=True)
+ class CommitModelMk2(Model):
+ repository_id = columns.Text(partition_key=True, required=True)
+ people = columns.Text(required=False)
+ message = columns.Text(required=False)
+
+ identifier = columns.Text(required=False)
+ hash = columns.Text(required=False)
+ revision = columns.Integer(required=False)
+
+ def to_commit(self):
+ return ScmCommit(
+ repository_id=self.repository_id, branch=self.branch,
+ timestamp=self.uuid // Commit.TIMESTAMP_TO_UUID_MULTIPLIER,
+ order=self.uuid % Commit.TIMESTAMP_TO_UUID_MULTIPLIER,
+ author=self.people.get('author') if self.people else None,
+ message=self.message,
+ identifier=self.identifier,
+ hash=self.hash,
+ revision=self.revision,
+ )
+
+ class CommitByRef(CommitModelMk2):
+ __table_name__ = 'commits_ref_to_object'
+ ref = columns.Text(primary_key=True, required=True)
+ uuid = columns.BigInt(required=True)
+ branch = columns.Text(required=True)
+
+ class CommitByUuidAscendingMk2(CommitModelMk2):
+ __table_name__ = 'commits_uuid_to_object_ascending'
+ branch = columns.Text(partition_key=True, required=True)
+ uuid = columns.BigInt(primary_key=True, required=True, clustering_order='ASC')
+
+ class CommitByUuidDescendingMk2(CommitModelMk2):
+ __table_name__ = 'commits_uuid_to_object_descending'
+ branch = columns.Text(partition_key=True, required=True)
+ uuid = columns.BigInt(primary_key=True, required=True, clustering_order='DESC')
+
class Branches(Model):
__table_name__ = 'commit_branches'
repository_id = columns.Text(partition_key=True, required=True)
@@ -80,13 +117,15 @@
self.cassandra = cassandra
self.repositories = {}
self.cache_timeout = cache_timeout
+ self.name = 'commit-identifiers'
with self:
- self.cassandra.create_table(self.CommitByID)
- self.cassandra.create_table(self.CommitByUuidAscending)
- self.cassandra.create_table(self.CommitByUuidDescending)
- self.cassandra.create_table(self.Branches)
+ for old in [self.CommitByID, self.CommitByUuidAscending, self.CommitByUuidDescending]:
+ self.cassandra.create_table(old)
+ for table in [self.CommitByRef, self.CommitByUuidAscendingMk2, self.CommitByUuidDescendingMk2, self.Branches]:
+ self.cassandra.create_table(table)
+
def __enter__(self):
self.cassandra.__enter__()
@@ -169,6 +208,22 @@
callback,
)
+ def find_commits_by_ref(self, repository_id, ref, limit=100):
+ # FIXME: Should use the redis cache, but the redis cache doesn't support new-style commits yet
+ with self:
+ if isinstance(ref, int) or ref.isdigit():
+ return [model.to_commit() for model in self.cassandra.select_from_table(
+ self.CommitByRef.__table_name__, limit=limit,
+ repository_id=repository_id, ref='r{}'.format(ref),
+ )]
+
+ # FIXME: SASI indecies are the canoical way to solve this problem, but require Cassandra 3.4 which
+ # hasn't been deployed to our datacenters yet. This works for commits, but is less transparent.
+ return [model.to_commit() for model in self.cassandra.select_from_table(
+ self.CommitByRef.__table_name__, limit=limit,
+ repository_id=repository_id, ref__gte=ref.lower(), ref__lte=(ref.lower() + 'g'),
+ )]
+
def find_commits_by_uuid(self, repository_id, branch, uuid, limit=100):
if branch is None:
branch = self.repositories[repository_id].default_branch
@@ -298,21 +353,77 @@
self.Branches.__table_name__, limit=limit, repository_id=repository_id,
)]
+ def register(self, configuration, commits, suite, test_results, timestamp=None):
+ try:
+ for commit in commits:
+ self.register_partial_commit(
+ repository_id=commit.repository_id,
+ ref=commit.id,
+ revision=commit.revision,
+ hash=commit.hash,
+ fast=False,
+ )
+ except Exception as e:
+ return dict(
+ status='error',
+ description=str(e),
+ )
+
+ return dict(status='ok')
+
def register_commit(self, commit):
- if not isinstance(commit, Commit):
- raise TypeError(f'Expected type {Commit}, got {type(commit)}')
+ if not isinstance(commit, ScmCommit):
+ raise TypeError(f'Expected type {ScmCommit}, got {type(commit)}')
with self:
if commit.repository_id not in self.repositories:
self.repositories[commit.repository_id] = Repository(key=commit.repository_id)
- for table in [self.CommitByID, self.CommitByUuidAscending, self.CommitByUuidDescending]:
+ for old in [self.CommitByID, self.CommitByUuidAscending, self.CommitByUuidDescending]:
self.cassandra.insert_row(
+ old.__table_name__,
+ repository_id=commit.repository_id,
+ branch=commit.branch,
+ commit_id=str(commit.revision).lower() if commit.revision else commit.hash,
+ uuid=commit.uuid,
+ committer=commit.author.email if commit.author else None,
+ message=commit.message,
+ )
+
+ for table in [self.CommitByUuidAscendingMk2, self.CommitByUuidDescendingMk2, self.Branches]:
+ self.cassandra.insert_row(
table.__table_name__,
- repository_id=commit.repository_id, branch=commit.branch,
- commit_id=str(commit.id).lower(), uuid=commit.uuid,
- committer=commit.committer, message=commit.message,
+ repository_id=commit.repository_id,
+ branch=commit.branch,
+ revision=commit.revision,
+ hash=commit.hash,
+ identifier=str(commit) if commit.identifier else None,
+ uuid=commit.uuid,
+ committer=json.dumps(commit.author, cls=Contributor.Encoder) if commit.author else None,
+ message=commit.message,
)
+
+ for key, ref_gen in (
+ ('revision', lambda commit: 'r{}'.format(commit.revision)),
+ ('hash', lambda commit: commit.hash),
+ ('identifier', str),
+ ):
+ if not getattr(commit, key):
+ continue
+
+ self.cassandra.insert_row(
+ self.CommitByRef.__table_name__,
+ ref=ref_gen(commit),
+ repository_id=commit.repository_id,
+ branch=commit.branch,
+ revision=commit.revision,
+ hash=commit.hash,
+ identifier=str(commit) if commit.identifier else None,
+ uuid=commit.uuid,
+ committer=json.dumps(commit.author, cls=Contributor.Encoder) if commit.author else None,
+ message=commit.message,
+ )
+
self.cassandra.insert_row(
self.Branches.__table_name__,
repository_id=commit.repository_id, branch=commit.branch,
@@ -319,26 +430,24 @@
)
return commit
- def register_commit_with_repo_and_id(self, repository_id, branch, commit_id):
- if branch is None:
- branch = self.repositories[repository_id].default_branch
+ def register_partial_commit(self, repository_id, ref=None, revision=None, hash=None, identifier=None, fast=True):
if repository_id not in self.repositories:
raise RuntimeError('{} is not a recognized repository')
with self:
- commits = self.find_commits_by_id(repository_id=repository_id, branch=branch, commit_id=commit_id)
+ commits = self.find_commits_by_ref(repository_id=repository_id, ref=ref or revision or hash or identifier)
if len(commits) > 1:
- raise ScmBase.Exception(f'Multiple commits with the id {commit_id} exist in {repository_id} on {branch}')
- if commits:
+ raise ScmBase.Exception(f'Multiple commits with the id {ref or revision or hash or identifier} exist in {repository_id}')
+ if commits and (fast or commits[0].identifier):
return commits[0]
- commit = self.repositories[repository_id].commit_for_id(commit_id)
+ commit = self.repositories[repository_id].commit(ref=ref, revision=revision, hash=hash, identifier=identifier, fast=fast)
return self.register_commit(commit)
def url(self, commit):
- if not isinstance(commit, Commit):
+ if not isinstance(commit, Commit) and not isinstance(commit, ScmCommit):
raise TypeError(f'Expected type {Commit}, got {type(commit)}')
repo = self.repositories.get(commit.repository_id)
if repo:
- return repo.url_for_commit(commit.id)
+ return repo.url_for_commit(commit.id if isinstance(commit, Commit) else commit.hash or commit.revision)
return None
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/commit_context_unittest.py (274541 => 274542)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/commit_context_unittest.py 2021-03-17 00:52:10 UTC (rev 274541)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/commit_context_unittest.py 2021-03-17 00:53:41 UTC (rev 274542)
@@ -53,8 +53,8 @@
for key, repository in dict(safari=safari, webkit=webkit).items():
for branch, commits in repository.commits.items():
for commit in commits:
- self.database.register_commit_with_repo_and_id(
- key, branch, commit.hash or commit.revision,
+ self.database.register_partial_commit(
+ key, ref=commit.hash or commit.revision,
)
@WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
@@ -183,7 +183,7 @@
def test_commit_from_stash_repo(self, redis=StrictRedis, cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
- self.database.register_commit_with_repo_and_id('safari', 'main', 'd8bce26fa65c')
+ self.database.register_partial_commit('safari', hash='d8bce26fa65c')
self.assertEqual(
[self.stash_repository.commit_for_id(id='d8bce26fa65c')],
self.database.find_commits_by_id(repository_id='safari', branch='main', commit_id='d8bce26fa65c'),
@@ -193,7 +193,7 @@
def test_commit_from_svn_repo(self, redis=StrictRedis, cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
- self.database.register_commit_with_repo_and_id('webkit', 'trunk', 6)
+ self.database.register_partial_commit('webkit', revision=6)
self.assertEqual(
[self.svn_repository.commit_for_id(id=6)],
self.database.find_commits_by_id(repository_id='webkit', branch='trunk', commit_id=6),
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/mock_model_factory.py (274541 => 274542)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/mock_model_factory.py 2021-03-17 00:52:10 UTC (rev 274541)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/mock_model_factory.py 2021-03-17 00:53:41 UTC (rev 274542)
@@ -1,4 +1,4 @@
-# Copyright (C) 2019 Apple Inc. All rights reserved.
+# Copyright (C) 2019-2021 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
@@ -92,8 +92,8 @@
for key, repository in dict(safari=safari, webkit=webkit).items():
for branch, commits in repository.commits.items():
for commit in commits:
- model.commit_context.register_commit_with_repo_and_id(
- key, branch, commit.hash or commit.revision,
+ model.commit_context.register_partial_commit(
+ key, hash=commit.hash, revision=commit.revision,
)
return model
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/model.py (274541 => 274542)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/model.py 2021-03-17 00:52:10 UTC (rev 274541)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/model.py 2021-03-17 00:53:41 UTC (rev 274542)
@@ -92,7 +92,7 @@
ttl_seconds=self.default_ttl_seconds,
)
- for context in [self.ci_context, self.failure_context, self.suite_context, self.test_context]:
+ for context in [self.ci_context, self.failure_context, self.suite_context, self.test_context, self.commit_context]:
self.upload_context.register_upload_callback(context.name, context.register)
self.archive_context = ArchiveContext(
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/repository.py (274541 => 274542)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/repository.py 2021-03-17 00:52:10 UTC (rev 274541)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/repository.py 2021-03-17 00:53:41 UTC (rev 274542)
@@ -33,6 +33,9 @@
def commit_for_id(self, id):
raise NotImplementedError()
+ def commit(self, ref=None, revision=None, hash=None, identifier=None, fast=True):
+ raise NotImplementedError()
+
def url_for_commit(self, commit):
return None
@@ -61,6 +64,13 @@
message=commit.message,
)
+ def commit(self, ref=None, revision=None, hash=None, identifier=None, fast=True):
+ if identifier:
+ fast = False
+ if ref:
+ return self.remote.find(ref, include_identifier=not fast)
+ return self.remote.commit(revision=revision, hash=hash, identifier=identifier, include_identifier=not fast)
+
def url_for_commit(self, commit):
return f'{self.remote.url}/commits/{commit}'
@@ -89,6 +99,15 @@
message=commit.message,
)
+ def commit(self, ref=None, revision=None, hash=None, identifier=None, fast=True):
+ if identifier:
+ fast = False
+ if ref:
+ if isinstance(ref, int) or ref.isdigit():
+ ref = 'r{}'.format(ref)
+ return self.remote.find(ref, include_identifier=not fast)
+ return self.remote.commit(revision=revision, hash=hash, identifier=identifier, include_identifier=not fast)
+
def url_for_commit(self, commit):
return f'https://commits.webkit.org/r{commit}'
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/repository_unittest.py (274541 => 274542)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/repository_unittest.py 2021-03-17 00:52:10 UTC (rev 274541)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/repository_unittest.py 2021-03-17 00:53:41 UTC (rev 274542)
@@ -1,4 +1,4 @@
-# Copyright (C) 2019 Apple Inc. All rights reserved.
+# Copyright (C) 2019-2021 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
@@ -31,15 +31,15 @@
with MockModelFactory.webkit():
svn_repo = WebKitRepository()
self.assertTrue('webkit', svn_repo.key)
- commit = svn_repo.commit_for_id(6)
+ commit = svn_repo.commit(revision=6)
self.assertEqual(commit.uuid, 160163990000)
self.assertEqual(commit.message, '6th commit')
self.assertEqual(commit.branch, 'trunk')
- def test_branch_svn(self):
+ def test_ref_svn(self):
with MockModelFactory.webkit():
svn_repo = WebKitRepository()
- commit = svn_repo.commit_for_id(7)
+ commit = svn_repo.commit(ref=7)
self.assertEqual(commit.uuid, 160164090000)
self.assertEqual(commit.message, '7th commit')
self.assertEqual(commit.branch, 'branch-a')
@@ -48,15 +48,23 @@
with MockModelFactory.safari() as safari:
git_repo = StashRepository('https://{}'.format(safari.remote))
self.assertEqual('safari', git_repo.key)
- commit = git_repo.commit_for_id('1abe25b443e9')
+ commit = git_repo.commit(hash='1abe25b443e9')
self.assertEqual(commit.uuid, 160166300000)
self.assertEqual(commit.branch, 'main')
+ def test_ref_stash(self):
+ with MockModelFactory.safari() as safari:
+ git_repo = StashRepository('https://{}'.format(safari.remote))
+ self.assertEqual('safari', git_repo.key)
+ commit = git_repo.commit(ref='1abe25b443e9')
+ self.assertEqual(commit.uuid, 160166300000)
+ self.assertEqual(commit.branch, 'main')
+
def test_colliding_timestamps_stash(self):
with MockModelFactory.safari() as safari:
git_repo = StashRepository('https://{}'.format(safari.remote))
- commit1 = git_repo.commit_for_id('bae5d1e90999')
- commit2 = git_repo.commit_for_id('d8bce26fa65c')
+ commit1 = git_repo.commit(hash='bae5d1e90999')
+ commit2 = git_repo.commit(hash='d8bce26fa65c')
self.assertEqual(commit1.timestamp, commit2.timestamp)
self.assertNotEqual(commit1.uuid, commit2.uuid)
@@ -67,6 +75,6 @@
def test_branch_stash(self):
with MockModelFactory.safari() as safari:
git_repo = StashRepository('https://{}'.format(safari.remote))
- commit = git_repo.commit_for_id('621652add7fc')
+ commit = git_repo.commit(hash='621652add7fc')
self.assertEqual(commit.uuid, 160166600000)
self.assertEqual(commit.branch, 'branch-a')
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/upload_context.py (274541 => 274542)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/upload_context.py 2021-03-17 00:52:10 UTC (rev 274541)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/upload_context.py 2021-03-17 00:53:41 UTC (rev 274542)
@@ -34,7 +34,9 @@
from resultsdbpy.model.commit_context import CommitContext
from resultsdbpy.model.configuration_context import ClusteredByConfiguration
+from webkitscmpy import Commit as ScmCommit
+
class UploadContext(object):
QUEUE_NAME = 'upload_queue'
PROCESS_TIMEOUT = 7 * 24 * 60 * 60
@@ -61,7 +63,7 @@
def unpack(self):
return dict(
- commits=[Commit.from_json(element) for element in json.loads(UploadContext.from_zip(bytearray(self.commits)))],
+ commits=[ScmCommit.from_json(element) for element in json.loads(UploadContext.from_zip(bytearray(self.commits)))],
sdk=None if self.sdk == '?' else self.sdk,
test_results=json.loads(UploadContext.from_zip(bytearray(self.test_results))),
timestamp=calendar.timegm(self.time_uploaded.timetuple()),
@@ -81,7 +83,7 @@
def unpack(self):
return dict(
- commits=[Commit.from_json(element) for element in json.loads(UploadContext.from_zip(bytearray(self.commits)))],
+ commits=[ScmCommit.from_json(element) for element in json.loads(UploadContext.from_zip(bytearray(self.commits)))],
sdk=None if self.sdk == '?' else self.sdk,
test_results=json.loads(UploadContext.from_zip(bytearray(self.test_results))),
timestamp=calendar.timegm(self.time_uploaded.timetuple()),
@@ -179,7 +181,7 @@
data = ""
self.synchronously_process_test_results(
configuration=Configuration.from_json(data['configuration']),
- commits=[Commit.from_json(commit_json) for commit_json in data['commits']],
+ commits=[ScmCommit.from_json(commit_json) for commit_json in data['commits']],
suite=data['suite'],
timestamp=data['timestamp'],
test_results=data['test_results'],
@@ -229,7 +231,7 @@
json.dumps(dict(
configuration=Configuration.Encoder().default(configuration),
suite=suite,
- commits=Commit.Encoder().default(commits),
+ commits=[commit.Encoder().default(commit) for commit in commits],
timestamp=timestamp,
test_results=test_results,
)),
@@ -241,7 +243,7 @@
if not isinstance(suite, str):
raise TypeError(f'Expected type {str}, got {type(suite)}')
for commit in commits:
- if not isinstance(commit, Commit):
+ if not isinstance(commit, ScmCommit) and not isinstance(commit, Commit):
raise TypeError(f'Expected type {Commit}, got {type(commit)}')
if len(commits) < 1:
raise ValueError('Each test result must have at least 1 associated commit')
@@ -265,7 +267,7 @@
self.configuration_context.insert_row_with_configuration(
self.UploadsByConfiguration.__table_name__, configuration=configuration,
suite=suite, branch=branch, uuid=uuid, sdk=configuration.sdk or '?', time_uploaded=timestamp,
- commits=self.to_zip(json.dumps(commits, cls=Commit.Encoder)),
+ commits=self.to_zip(json.dumps(commits, cls=commits[0].Encoder)),
test_results=self.to_zip(json.dumps(test_results)),
upload_version=version,
ttl=int((uuid // Commit.TIMESTAMP_TO_UUID_MULTIPLIER) + self.ttl_seconds - time.time()) if self.ttl_seconds else None,
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/upload_context_unittest.py (274541 => 274542)
--- trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/upload_context_unittest.py 2021-03-17 00:52:10 UTC (rev 274541)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/resultsdbpy/model/upload_context_unittest.py 2021-03-17 00:53:41 UTC (rev 274542)
@@ -135,18 +135,19 @@
self.init_database(redis=redis, cassandra=cassandra)
MockModelFactory.add_mock_results(self.model)
- configuration_to_search = Configuration(platform='iOS', version='12.0.0', is_simulator=True, style='Asan')
- configuration, uploads = next(iter(self.model.upload_context.find_test_results(configurations=[configuration_to_search], suite='layout-tests', recent=False).items()))
- self.model.upload_context.process_test_results(
- configuration=configuration,
- commits=uploads[0]['commits'],
- suite='layout-tests',
- test_results=uploads[0]['test_results'],
- timestamp=uploads[0]['timestamp'],
- )
+ with MockModelFactory.safari(), MockModelFactory.webkit():
+ configuration_to_search = Configuration(platform='iOS', version='12.0.0', is_simulator=True, style='Asan')
+ configuration, uploads = next(iter(self.model.upload_context.find_test_results(configurations=[configuration_to_search], suite='layout-tests', recent=False).items()))
+ self.model.upload_context.process_test_results(
+ configuration=configuration,
+ commits=uploads[0]['commits'],
+ suite='layout-tests',
+ test_results=uploads[0]['test_results'],
+ timestamp=uploads[0]['timestamp'],
+ )
- # Using suite results as a proxy to tell if callbacks were triggered
- self.assertEqual(1, len(self.model.suite_context.find_by_commit(configurations=[Configuration()], suite='layout-tests')))
+ # Using suite results as a proxy to tell if callbacks were triggered
+ self.assertEqual(1, len(self.model.suite_context.find_by_commit(configurations=[Configuration()], suite='layout-tests')))
@WaitForDockerTestCase.mock_if_no_docker(mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_async_callback(self, redis=StrictRedis, cassandra=CassandraContext):
@@ -153,17 +154,19 @@
self.init_database(redis=redis, cassandra=cassandra, async_processing=True)
MockModelFactory.add_mock_results(self.model)
- configuration_to_search = Configuration(platform='iOS', version='12.0.0', is_simulator=True, style='Asan')
- configuration, uploads = next(iter(self.model.upload_context.find_test_results(configurations=[configuration_to_search], suite='layout-tests', recent=False).items()))
- self.model.upload_context.process_test_results(
- configuration=configuration,
- commits=uploads[0]['commits'],
- suite='layout-tests',
- test_results=uploads[0]['test_results'],
- timestamp=uploads[0]['timestamp'],
- )
+ with MockModelFactory.safari(), MockModelFactory.webkit():
+ configuration_to_search = Configuration(platform='iOS', version='12.0.0', is_simulator=True, style='Asan')
+ configuration, uploads = next(iter(self.model.upload_context.find_test_results(configurations=[configuration_to_search], suite='layout-tests', recent=False).items()))
- # Using suite results as a proxy to tell if callbacks were triggered
- self.assertEqual(0, len(self.model.suite_context.find_by_commit(configurations=[Configuration()], suite='layout-tests')))
- self.assertTrue(self.model.upload_context.do_processing_work())
- self.assertEqual(1, len(self.model.suite_context.find_by_commit(configurations=[Configuration()], suite='layout-tests')))
+ self.model.upload_context.process_test_results(
+ configuration=configuration,
+ commits=uploads[0]['commits'],
+ suite='layout-tests',
+ test_results=uploads[0]['test_results'],
+ timestamp=uploads[0]['timestamp'],
+ )
+
+ # Using suite results as a proxy to tell if callbacks were triggered
+ self.assertEqual(0, len(self.model.suite_context.find_by_commit(configurations=[Configuration()], suite='layout-tests')))
+ self.assertTrue(self.model.upload_context.do_processing_work())
+ self.assertEqual(1, len(self.model.suite_context.find_by_commit(configurations=[Configuration()], suite='layout-tests')))
Modified: trunk/Tools/Scripts/libraries/resultsdbpy/setup.py (274541 => 274542)
--- trunk/Tools/Scripts/libraries/resultsdbpy/setup.py 2021-03-17 00:52:10 UTC (rev 274541)
+++ trunk/Tools/Scripts/libraries/resultsdbpy/setup.py 2021-03-17 00:53:41 UTC (rev 274542)
@@ -30,7 +30,7 @@
setup(
name='resultsdbpy',
- version='2.0.2',
+ version='2.0.3',
description='Library for visualizing, processing and storing test results.',
long_description=readme(),
classifiers=[
Modified: trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/commit.py (274541 => 274542)
--- trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/commit.py 2021-03-17 00:52:10 UTC (rev 274541)
+++ trunk/Tools/Scripts/libraries/webkitscmpy/webkitscmpy/commit.py 2021-03-17 00:53:41 UTC (rev 274542)
@@ -220,7 +220,7 @@
if author and isinstance(author, dict) and author.get('name'):
self.author = Contributor(author.get('name'), author.get('emails'))
elif author and isinstance(author, six.string_types) and '@' in author:
- self.author = Contributor(author, author)
+ self.author = Contributor(author, [author])
elif author and not isinstance(author, Contributor):
raise TypeError("Expected 'author' to be of type {}, got '{}'".format(Contributor, author))
else:
