This is an automated email from the ASF dual-hosted git repository. lynwee pushed a commit to branch lw-cp0223 in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 4f13b8ce1375be768e557a3a3c327a11d8879ed9 Author: Lynwee Hou <[email protected]> AuthorDate: Fri Feb 23 18:55:40 2024 +0800 fix(azuredevops): fix some fields' value --- backend/core/models/domainlayer/code/repo.go | 10 +++++----- backend/core/plugin/hub.go | 2 ++ backend/python/plugins/azuredevops/azuredevops/main.py | 8 ++++---- backend/python/plugins/azuredevops/azuredevops/migrations.py | 5 +++++ backend/python/plugins/azuredevops/azuredevops/models.py | 3 ++- .../plugins/azuredevops/azuredevops/streams/pull_requests.py | 4 ++-- backend/python/pydevlake/pydevlake/plugin.py | 2 +- 7 files changed, 21 insertions(+), 13 deletions(-) diff --git a/backend/core/models/domainlayer/code/repo.go b/backend/core/models/domainlayer/code/repo.go index 90469c1cb..f6ad9faf9 100644 --- a/backend/core/models/domainlayer/code/repo.go +++ b/backend/core/models/domainlayer/code/repo.go @@ -19,7 +19,7 @@ package code import ( "github.com/apache/incubator-devlake/core/models/domainlayer" - plugin "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/plugin" "time" ) @@ -30,11 +30,11 @@ type Repo struct { Name string `json:"name"` Url string `json:"url"` Description string `json:"description"` - OwnerId string `json:"ownerId" gorm:"type:varchar(255)"` + OwnerId string `json:"owner_id" gorm:"type:varchar(255)"` Language string `json:"language" gorm:"type:varchar(255)"` - ForkedFrom string `json:"forkedFrom"` - CreatedDate *time.Time `json:"createdDate"` - UpdatedDate *time.Time `json:"updatedDate"` + ForkedFrom string `json:"forked_from"` + CreatedDate *time.Time `json:"created_date"` + UpdatedDate *time.Time `json:"updated_date"` Deleted bool `json:"deleted"` } diff --git a/backend/core/plugin/hub.go b/backend/core/plugin/hub.go index 911bfb6ba..6788c9057 100644 --- a/backend/core/plugin/hub.go +++ b/backend/core/plugin/hub.go @@ -79,6 +79,8 @@ func InitPlugins(basicRes context.BasicRes) { if err != nil { panic(fmt.Errorf("failed to initialize plugin %v due to %w", pluginName, err)) } + } else { + basicRes.GetLogger().Info("plugin: %s doesn't implement 'PluginInit', it will be skipped.", pluginName) } } } diff --git a/backend/python/plugins/azuredevops/azuredevops/main.py b/backend/python/plugins/azuredevops/azuredevops/main.py index a58d53591..d8b7a4acb 100644 --- a/backend/python/plugins/azuredevops/azuredevops/main.py +++ b/backend/python/plugins/azuredevops/azuredevops/main.py @@ -4,7 +4,6 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at - # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software @@ -27,6 +26,7 @@ from pydevlake.domain_layer.code import Repo from pydevlake.domain_layer.devops import CicdScope from pydevlake.pipeline_tasks import gitextractor, refdiff from pydevlake.api import APIException +from pydevlake.extractor import autoextract _SUPPORTED_EXTERNAL_SOURCE_PROVIDERS = ['github', 'githubenterprise', 'bitbucket', 'git'] @@ -50,9 +50,9 @@ class AzureDevOpsPlugin(Plugin): yield Repo( name=git_repo.name, url=git_repo.url, - forked_from=git_repo.parent_repository_url + forked_from=git_repo.parent_repository_url, + updated_date=git_repo.updated_date, ) - yield CicdScope( name=git_repo.name, description=git_repo.name, @@ -88,7 +88,7 @@ class AzureDevOpsPlugin(Plugin): url = urlparse(raw_repo['remoteUrl']) url = url._replace(netloc=url.hostname) raw_repo['url'] = url.geturl() - repo = GitRepository(**raw_repo) + repo = autoextract(raw_repo, GitRepository) if not repo.default_branch: continue if "parentRepository" in raw_repo: diff --git a/backend/python/plugins/azuredevops/azuredevops/migrations.py b/backend/python/plugins/azuredevops/azuredevops/migrations.py index fd3b0d58e..ab33e8a98 100644 --- a/backend/python/plugins/azuredevops/azuredevops/migrations.py +++ b/backend/python/plugins/azuredevops/azuredevops/migrations.py @@ -192,3 +192,8 @@ def add_queue_time_field_in_tool_azuredevops_builds(b: MigrationScriptBuilder): table = '_tool_azuredevops_builds' b.execute(f'ALTER TABLE {table} ADD COLUMN queue_time timestamptz', Dialect.POSTGRESQL) b.execute(f'ALTER TABLE {table} ADD COLUMN queue_time datetime', Dialect.MYSQL) + + +@migration(20240223170000, name="add updated_date field in _tool_azuredevops_gitrepositories") +def add_updated_date_field_in_tool_azuredevops_gitrepositories(b: MigrationScriptBuilder): + b.add_column('_tool_azuredevops_gitrepositories', 'updated_date', 'datetime(3) DEFAULT NULL') diff --git a/backend/python/plugins/azuredevops/azuredevops/models.py b/backend/python/plugins/azuredevops/azuredevops/models.py index cbddd9d1d..3a295aa7c 100644 --- a/backend/python/plugins/azuredevops/azuredevops/models.py +++ b/backend/python/plugins/azuredevops/azuredevops/models.py @@ -45,8 +45,9 @@ class GitRepository(ToolScope, table=True): default_branch: Optional[str] project_id: str org_id: str - parent_repository_url: Optional[str] = Field(source='parentRepository/url') + parent_repository_url: Optional[str] = Field(source='/parentRepository/url') provider: Optional[str] + updated_date: datetime.datetime = Field(source='/project/lastUpdateTime') def is_external(self): return bool(self.provider) diff --git a/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py b/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py index 6922d1b7e..bebb7bed8 100644 --- a/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py +++ b/backend/python/plugins/azuredevops/azuredevops/streams/pull_requests.py @@ -37,7 +37,7 @@ class GitPullRequests(Stream): def convert(self, pr: GitPullRequest, ctx): repo_id = ctx.scope.domain_id() - # If the PR is from a fork, we forge a new repo ID for the base repo but it doesn't correspond to a real repo + # If the PR is from a fork, we forge a new repo ID for the base repo, but it doesn't correspond to a real repo base_repo_id = domain_id(GitRepository, ctx.connection.id, pr.fork_repo_id) if pr.fork_repo_id is not None else repo_id # Use the same status values as GitHub plugin @@ -56,7 +56,7 @@ class GitPullRequests(Stream): original_status=pr.status.value, title=pr.title, description=pr.description, - url=pr.url, + url=f"{ctx.scope.url}/pullrequest/{pr.pull_request_id}", author_name=pr.created_by_name, author_id=pr.created_by_id, pull_request_key=pr.pull_request_id, diff --git a/backend/python/pydevlake/pydevlake/plugin.py b/backend/python/pydevlake/pydevlake/plugin.py index fc368261a..e3303300e 100644 --- a/backend/python/pydevlake/pydevlake/plugin.py +++ b/backend/python/pydevlake/pydevlake/plugin.py @@ -71,7 +71,7 @@ class Plugin(ABC): @abstractmethod def test_connection(self, connection: Connection) -> msg.TestConnectionResult: """ - Test if the the connection with the datasource can be established with the given connection. + Test if the connection with the datasource can be established with the given connection. Must raise an exception if the connection can't be established. """ pass
