This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new ca89492a9 feat: add remote repository support for azure devops (#7364)
ca89492a9 is described below
commit ca89492a9ba56ad7ce1981009deb3f5e4aea6148
Author: Markus Braunbeck <[email protected]>
AuthorDate: Thu Apr 25 05:05:41 2024 +0200
feat: add remote repository support for azure devops (#7364)
It is now possible to choose remote repositories connected through 'service
connections' as a data scope. Tasks in the cicd and code domains can be
performed based on the visibility of these repositories (public or private).
---
.../azuredevops_go/api/azuredevops/client.go | 282 +++++++++++++++++++++
.../client_test.go} | 6 +-
.../azuredevops_go/api/azuredevops/models.go | 132 ++++++++++
.../api/{ => azuredevops}/testdata/test.txt | 0
.../plugins/azuredevops_go/api/blueprint_v200.go | 61 ++++-
.../azuredevops_go/api/blueprint_v200_test.go | 75 +++++-
.../plugins/azuredevops_go/api/connection_api.go | 9 +-
backend/plugins/azuredevops_go/api/remote_data.go | 65 -----
.../plugins/azuredevops_go/api/remote_helper.go | 223 +++++++++++-----
backend/plugins/azuredevops_go/api/vs_client.go | 137 ----------
backend/plugins/azuredevops_go/e2e/build_test.go | 1 +
backend/plugins/azuredevops_go/models/base.go | 5 +
.../20240413_add_remote_repo_support.go | 68 +++++
.../models/migrationscripts/register.go | 1 +
backend/plugins/azuredevops_go/models/repo.go | 13 +-
.../azuredevops_go/tasks/ci_cd_build_collector.go | 12 +-
.../azuredevops_go/tasks/ci_cd_build_converter.go | 2 +-
backend/plugins/azuredevops_go/tasks/task_data.go | 4 +-
backend/test/e2e/manual/azuredevops/models.go | 1 +
config-ui/src/routes/pipeline/components/task.tsx | 3 +
20 files changed, 806 insertions(+), 294 deletions(-)
diff --git a/backend/plugins/azuredevops_go/api/azuredevops/client.go
b/backend/plugins/azuredevops_go/api/azuredevops/client.go
new file mode 100644
index 000000000..36a9ef7c6
--- /dev/null
+++ b/backend/plugins/azuredevops_go/api/azuredevops/client.go
@@ -0,0 +1,282 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azuredevops
+
+import (
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/azuredevops_go/models"
+ "io"
+ "net/http"
+ "net/url"
+ "strconv"
+ "time"
+)
+
+const apiVersion = "7.1"
+const maxPageSize = 100
+
+type Client struct {
+ c http.Client
+
+ apiClient plugin.ApiClient
+ connection *models.AzuredevopsConnection
+ url string
+}
+
+func NewClient(con *models.AzuredevopsConnection, apiClient plugin.ApiClient,
url string) Client {
+ return Client{
+ c: http.Client{
+ Timeout: 2 * time.Second,
+ },
+ connection: con,
+ url: url,
+ apiClient: apiClient,
+ }
+}
+
+func (c *Client) GetUserProfile() (Profile, errors.Error) {
+ var p Profile
+ endpoint, err := url.JoinPath(c.url, "/_apis/profile/profiles/me")
+ if err != nil {
+ return Profile{}, errors.Internal.Wrap(err, "failed to join
user profile path")
+ }
+
+ res, err := c.doGet(endpoint)
+ if err != nil {
+ return Profile{}, errors.Internal.Wrap(err, "failed to read
user accounts")
+ }
+
+ if res.StatusCode == 302 || res.StatusCode == 401 {
+ return Profile{}, errors.Unauthorized.New("failed to read user
profile")
+ }
+
+ defer res.Body.Close()
+ resBody, err := io.ReadAll(res.Body)
+ if err != nil {
+ return Profile{}, errors.Internal.Wrap(err, "failed to read
response body")
+ }
+
+ if err := json.Unmarshal(resBody, &p); err != nil {
+ panic(err)
+ }
+ return p, nil
+}
+
+func (c *Client) GetUserAccounts(memberId string) (AccountResponse,
errors.Error) {
+ var a AccountResponse
+ endpoint := fmt.Sprintf(c.url+"/_apis/accounts?memberId=%s", memberId)
+ res, err := c.doGet(endpoint)
+ if err != nil {
+ return nil, errors.Internal.Wrap(err, "failed to read user
accounts")
+ }
+
+ if res.StatusCode == 302 || res.StatusCode == 401 {
+ return nil, errors.Unauthorized.New("failed to read user
accounts")
+ }
+
+ defer res.Body.Close()
+ resBody, err := io.ReadAll(res.Body)
+ if err != nil {
+ return nil, errors.Internal.Wrap(err, "failed to read response
body")
+ }
+
+ if err := json.Unmarshal(resBody, &a); err != nil {
+ return nil, errors.Internal.Wrap(err, "failed to read unmarshal
response body")
+ }
+ return a, nil
+}
+
+func (c *Client) doGet(url string) (*http.Response, error) {
+ req, err := http.NewRequest(http.MethodGet, url, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ if err =
c.connection.GetAccessTokenAuthenticator().SetupAuthentication(req); err != nil
{
+ return nil, errors.Internal.Wrap(err, "failed to authorize the
request using the plugin connection")
+ }
+ return http.DefaultClient.Do(req)
+}
+
+type GetProjectsArgs struct {
+ // (optional) Pagination
+ *OffsetPagination
+
+ OrgId string
+}
+
+func (c *Client) GetProjects(args GetProjectsArgs) ([]Project, errors.Error) {
+ query := url.Values{}
+ query.Set("api-version", apiVersion)
+
+ var top, skip int
+ top = maxPageSize
+ skip = 0
+ if args.OffsetPagination != nil {
+ top = args.Top
+ skip = args.Skip
+ }
+
+ var data struct {
+ Count int `json:"count"`
+ Projects []Project `json:"value"`
+ }
+
+ var projects []Project
+
+ for {
+ query.Set("$top", strconv.Itoa(top))
+ query.Set("$skip", strconv.Itoa(skip))
+
+ path := fmt.Sprintf("%s/_apis/projects", args.OrgId)
+ res, err := c.apiClient.Get(path, query, nil)
+ if err != nil {
+ return nil, err
+ }
+ err = api.UnmarshalResponse(res, &data)
+ if err != nil {
+ return nil, err
+ }
+
+ projects = append(projects, data.Projects...)
+
+ if data.Count < top {
+ return projects, nil
+ }
+
+ skip += top
+ }
+}
+
+type GetRepositoriesArgs struct {
+ OrgId string
+ ProjectId string
+}
+
+func (c *Client) GetRepositories(args GetRepositoriesArgs) ([]Repository,
errors.Error) {
+ query := url.Values{}
+ query.Set("api-version", apiVersion)
+
+ var data struct {
+ Repos []Repository `json:"value"`
+ }
+
+ path := fmt.Sprintf("%s/%s/_apis/git/repositories", args.OrgId,
args.ProjectId)
+ res, err := c.apiClient.Get(path, query, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ switch res.StatusCode {
+ case 401:
+ fallthrough
+ case 403:
+ return nil, errors.Unauthorized.New("failed to authorize the
'.../_apis/git/repositories' request using the plugin connection")
+ case 404:
+ return nil, errors.NotFound.New("failed to find requested
resource on '.../_apis/git/repositories'")
+ default:
+ }
+
+ err = api.UnmarshalResponse(res, &data)
+ if err != nil {
+ return nil, err
+ }
+
+ return data.Repos, nil
+}
+
+type GetServiceEndpointsArgs struct {
+ ProjectId string
+ OrgId string
+}
+
+func (c *Client) GetServiceEndpoints(args GetServiceEndpointsArgs)
([]ServiceEndpoint, errors.Error) {
+ query := url.Values{}
+ query.Set("api-version", apiVersion)
+
+ path := fmt.Sprintf("%s/%s/_apis/serviceendpoint/endpoints/",
args.OrgId, args.ProjectId)
+ res, err := c.apiClient.Get(path, query, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ switch res.StatusCode {
+ case 401:
+ fallthrough
+ case 403:
+ return nil, errors.Unauthorized.New("failed to authorize the
'.../serviceendpoint/endpoints' request using the plugin connection")
+ case 404:
+ return nil, errors.NotFound.New("failed to find requested
resource on '.../serviceendpoint/endpoints'")
+ default:
+ }
+
+ var data struct {
+ ServiceEndpoints []ServiceEndpoint `json:"value"`
+ }
+
+ err = api.UnmarshalResponse(res, &data)
+ if err != nil {
+ return nil, err
+ }
+ return data.ServiceEndpoints, nil
+}
+
+type GetRemoteRepositoriesArgs struct {
+ ProjectId string
+ OrgId string
+ Provider string
+ // (optional) Service Endpoint to filter for
+ ServiceEndpoint string
+}
+
+func (c *Client) GetRemoteRepositories(args GetRemoteRepositoriesArgs)
([]RemoteRepository, error) {
+ query := url.Values{}
+ query.Set("api-version", apiVersion)
+ if args.ServiceEndpoint != "" {
+ query.Set("serviceEndpointId", args.ServiceEndpoint)
+ }
+
+ var repos []RemoteRepository
+ var response struct {
+ Repository []RemoteRepository `json:"repositories"`
+ }
+
+ for {
+ path :=
fmt.Sprintf("%s/%s/_apis/sourceProviders/%s/repositories/", args.OrgId,
args.ProjectId, args.Provider)
+ res, err := c.apiClient.Get(path, query, nil)
+ if err != nil {
+ return nil, errors.Internal.Wrap(err, "failed to read
remote repositories")
+ }
+ err = api.UnmarshalResponse(res, &response)
+ if err != nil {
+ return nil, errors.Internal.Wrap(err, "failed to
unmarshal remote repositories response")
+ }
+
+ repos = append(repos, response.Repository...)
+ contToken := res.Header.Get("X-Ms-Continuationtoken")
+ if contToken == "" {
+ return repos, nil
+ }
+
+ query.Set("continuationToken", contToken)
+ }
+}
diff --git a/backend/plugins/azuredevops_go/api/vs_client_test.go
b/backend/plugins/azuredevops_go/api/azuredevops/client_test.go
similarity index 97%
rename from backend/plugins/azuredevops_go/api/vs_client_test.go
rename to backend/plugins/azuredevops_go/api/azuredevops/client_test.go
index 7aa8ed54c..6b184e8a6 100644
--- a/backend/plugins/azuredevops_go/api/vs_client_test.go
+++ b/backend/plugins/azuredevops_go/api/azuredevops/client_test.go
@@ -15,7 +15,7 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package api
+package azuredevops
import (
"bytes"
@@ -89,8 +89,8 @@ func TestRetrieveUserProfile(t *testing.T) {
},
}
- client := newVsClient(conn, ts.URL)
- p, err := client.UserProfile()
+ client := NewClient(conn, nil, ts.URL)
+ p, err := client.GetUserProfile()
if err != nil && err.GetType().GetHttpCode() != code {
t.Errorf("User Profile API Response = %d; want:
%d", err.GetType().GetHttpCode(), code)
}
diff --git a/backend/plugins/azuredevops_go/api/azuredevops/models.go
b/backend/plugins/azuredevops_go/api/azuredevops/models.go
new file mode 100644
index 000000000..388d60cfc
--- /dev/null
+++ b/backend/plugins/azuredevops_go/api/azuredevops/models.go
@@ -0,0 +1,132 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azuredevops
+
+import "time"
+
+type Profile struct {
+ DisplayName string `json:"displayName"`
+ PublicAlias string `json:"publicAlias"`
+ EmailAddress string `json:"emailAddress"`
+ CoreRevision int `json:"coreRevision"`
+ TimeStamp time.Time `json:"timeStamp"`
+ Id string `json:"id"`
+ Revision int `json:"revision"`
+}
+
+type Account struct {
+ AccountId string `json:"AccountId"`
+ NamespaceId string `json:"NamespaceId"`
+ AccountName string `json:"AccountName"`
+ OrganizationName interface{} `json:"OrganizationName"`
+ AccountType int `json:"AccountType"`
+ AccountOwner string `json:"AccountOwner"`
+ CreatedBy string `json:"CreatedBy"`
+ CreatedDate string `json:"CreatedDate"`
+ AccountStatus int `json:"AccountStatus"`
+ StatusReason interface{} `json:"StatusReason"`
+ LastUpdatedBy string `json:"LastUpdatedBy"`
+ Properties struct {
+ } `json:"Properties"`
+}
+
+type AccountResponse []Account
+
+type RemoteRepository struct {
+ Properties struct {
+ ApiUrl string `json:"apiUrl"`
+ BranchesUrl string `json:"branchesUrl"`
+ CloneUrl string `json:"cloneUrl"`
+ ConnectedServiceId string `json:"connectedServiceId"`
+ DefaultBranch string `json:"defaultBranch"`
+ FullName string `json:"fullName"`
+ HasAdminPermissions string `json:"hasAdminPermissions"`
+ IsFork string `json:"isFork"`
+ IsPrivate string `json:"isPrivate"`
+ LastUpdated time.Time `json:"lastUpdated"`
+ ManageUrl string `json:"manageUrl"`
+ NodeId string `json:"nodeId"`
+ OwnerId string `json:"ownerId"`
+ OrgName string `json:"orgName"`
+ RefsUrl string `json:"refsUrl"`
+ SafeRepository string `json:"safeRepository"`
+ ShortName string `json:"shortName"`
+ OwnerAvatarUrl string `json:"ownerAvatarUrl"`
+ Archived string `json:"archived"`
+ ExternalId string `json:"externalId"`
+ OwnerIsAUser string `json:"ownerIsAUser"`
+ } `json:"properties"`
+ Id string `json:"id"`
+ SourceProviderName string `json:"sourceProviderName"`
+ Name string `json:"name"`
+ FullName string `json:"fullName"`
+ Url string `json:"url"`
+ DefaultBranch string `json:"defaultBranch"`
+}
+
+type ServiceEndpoint struct {
+ Data interface{} `json:"data"`
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Type string `json:"type"`
+ Url string `json:"url"`
+ Description string `json:"description"`
+ IsShared bool `json:"isShared"`
+ IsOutdated bool `json:"isOutdated"`
+ IsReady bool `json:"isReady"`
+ Owner string `json:"owner"`
+}
+
+type Project struct {
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Description string `json:"description"`
+ Url string `json:"url"`
+ State string `json:"state"`
+ Revision int `json:"revision"`
+ Visibility string `json:"visibility"`
+ LastUpdateTime time.Time `json:"lastUpdateTime"`
+}
+
+type OffsetPagination struct {
+ Skip int
+ Top int
+}
+
+type Repository struct {
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Url string `json:"url"`
+ Project struct {
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Description string `json:"description"`
+ Url string `json:"url"`
+ State string `json:"state"`
+ Revision int `json:"revision"`
+ Visibility string `json:"visibility"`
+ LastUpdateTime time.Time `json:"lastUpdateTime"`
+ } `json:"project"`
+ DefaultBranch string `json:"defaultBranch"`
+ Size int `json:"size"`
+ RemoteUrl string `json:"remoteUrl"`
+ SshUrl string `json:"sshUrl"`
+ WebUrl string `json:"webUrl"`
+ IsDisabled bool `json:"isDisabled"`
+ IsInMaintenance bool `json:"isInMaintenance"`
+}
diff --git a/backend/plugins/azuredevops_go/api/testdata/test.txt
b/backend/plugins/azuredevops_go/api/azuredevops/testdata/test.txt
similarity index 100%
rename from backend/plugins/azuredevops_go/api/testdata/test.txt
rename to backend/plugins/azuredevops_go/api/azuredevops/testdata/test.txt
diff --git a/backend/plugins/azuredevops_go/api/blueprint_v200.go
b/backend/plugins/azuredevops_go/api/blueprint_v200.go
index f1df9be8c..4aa539319 100644
--- a/backend/plugins/azuredevops_go/api/blueprint_v200.go
+++ b/backend/plugins/azuredevops_go/api/blueprint_v200.go
@@ -18,6 +18,7 @@ limitations under the License.
package api
import (
+ "golang.org/x/exp/slices"
"net/url"
"github.com/apache/incubator-devlake/core/errors"
@@ -70,13 +71,15 @@ func makeScopeV200(
for _, scope := range scopeDetails {
azuredevopsRepo, scopeConfig := scope.Scope, scope.ScopeConfig
+ if azuredevopsRepo.Type != models.RepositoryTypeADO {
+ continue
+ }
id :=
didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connectionId,
azuredevopsRepo.Id)
if utils.StringsContains(scopeConfig.Entities,
plugin.DOMAIN_TYPE_CODE_REVIEW) ||
utils.StringsContains(scopeConfig.Entities,
plugin.DOMAIN_TYPE_CODE) {
// if we don't need to collect gitex, we need to add
repo to scopes here
scopeRepo := code.NewRepo(id, azuredevopsRepo.Name)
-
sc = append(sc, scopeRepo)
}
@@ -93,6 +96,26 @@ func makeScopeV200(
}
}
+ for _, scope := range scopeDetails {
+ azuredevopsRepo, scopeConfig := scope.Scope, scope.ScopeConfig
+ if azuredevopsRepo.Type == models.RepositoryTypeADO {
+ continue
+ }
+ id :=
didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connectionId,
azuredevopsRepo.Id)
+
+ // Azure DevOps Pipeline can be used with remote repositories
such as GitHub and Bitbucket
+ if utils.StringsContains(scopeConfig.Entities,
plugin.DOMAIN_TYPE_CICD) {
+ scopeCICD := devops.NewCicdScope(id,
azuredevopsRepo.Name)
+ sc = append(sc, scopeCICD)
+ }
+
+ // DOMAIN_TYPE_CODE (i.e. gitextractor, rediff) only works if
the repository is public
+ if !azuredevopsRepo.IsPrivate &&
utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) {
+ scopeRepo := code.NewRepo(id, azuredevopsRepo.Name)
+ sc = append(sc, scopeRepo)
+ }
+ }
+
return sc, nil
}
@@ -106,16 +129,32 @@ func makePipelinePlanV200(
azuredevopsRepo, scopeConfig := scope.Scope, scope.ScopeConfig
var stage coreModels.PipelineStage
var err errors.Error
- // get repo
options := make(map[string]interface{})
+ options["name"] = azuredevopsRepo.Name // this is solely for
the FE to display the repo name of a task
+
options["connectionId"] = connection.ID
options["organizationId"] = azuredevopsRepo.OrganizationId
options["projectId"] = azuredevopsRepo.ProjectId
+ options["externalId"] = azuredevopsRepo.ExternalId
options["repositoryId"] = azuredevopsRepo.Id
+ options["repositoryType"] = azuredevopsRepo.Type
// construct subtasks
- subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
scopeConfig.Entities)
+ var entities []string
+ if scope.Scope.Type == models.RepositoryTypeADO {
+ entities = append(entities, scopeConfig.Entities...)
+ } else {
+ if i := slices.Index(scopeConfig.Entities,
plugin.DOMAIN_TYPE_CICD); i >= 0 {
+ entities = append(entities,
scopeConfig.Entities[i])
+ }
+
+ if i := slices.Index(scopeConfig.Entities,
plugin.DOMAIN_TYPE_CODE); i >= 0 && !scope.Scope.IsPrivate {
+ entities = append(entities,
scopeConfig.Entities[i])
+ }
+ }
+
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
entities)
if err != nil {
return nil, err
}
@@ -127,20 +166,22 @@ func makePipelinePlanV200(
})
// collect git data by gitextractor if CODE was requested
- if utils.StringsContains(scopeConfig.Entities,
plugin.DOMAIN_TYPE_CODE) || len(scopeConfig.Entities) == 0 {
+ if utils.StringsContains(scopeConfig.Entities,
plugin.DOMAIN_TYPE_CODE) && !scope.Scope.IsPrivate || len(scopeConfig.Entities)
== 0 {
cloneUrl, err :=
errors.Convert01(url.Parse(azuredevopsRepo.RemoteUrl))
if err != nil {
return nil, err
}
- cloneUrl.User = url.UserPassword("git",
connection.Token)
+
+ if scope.Scope.Type == models.RepositoryTypeADO {
+ cloneUrl.User = url.UserPassword("git",
connection.Token)
+ }
stage = append(stage, &coreModels.PipelineTask{
Plugin: "gitextractor",
Options: map[string]interface{}{
- "url": cloneUrl.String(),
- "name": azuredevopsRepo.Name,
- "repoId":
didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connection.ID,
azuredevopsRepo.Id),
- "proxy": connection.Proxy,
- "noShallowClone": true,
+ "url": cloneUrl.String(),
+ "name": azuredevopsRepo.Name,
+ "repoId":
didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connection.ID,
azuredevopsRepo.Id),
+ "proxy": connection.Proxy,
},
})
}
diff --git a/backend/plugins/azuredevops_go/api/blueprint_v200_test.go
b/backend/plugins/azuredevops_go/api/blueprint_v200_test.go
index 8fb55c893..1c1281c62 100644
--- a/backend/plugins/azuredevops_go/api/blueprint_v200_test.go
+++ b/backend/plugins/azuredevops_go/api/blueprint_v200_test.go
@@ -19,6 +19,8 @@ package api
import (
"fmt"
+ "reflect"
+ "strings"
"testing"
coreModels "github.com/apache/incubator-devlake/core/models"
@@ -58,7 +60,8 @@ func TestMakeScopes(t *testing.T) {
Scope: common.Scope{
ConnectionId: connectionID,
},
- Id: azuredevopsRepoId,
+ Id: azuredevopsRepoId,
+ Type: models.RepositoryTypeADO,
},
ScopeConfig: &models.AzuredevopsScopeConfig{
ScopeConfig: common.ScopeConfig{
@@ -115,6 +118,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
Name: azureDevOpsProjectName,
Url: httpUrlToRepo,
RemoteUrl: httpUrlToRepo,
+ Type: models.RepositoryTypeADO,
},
ScopeConfig: &models.AzuredevopsScopeConfig{
ScopeConfig: common.ScopeConfig{
@@ -144,20 +148,22 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
tasks.ExtractApiBuildsMeta.Name,
},
Options: map[string]interface{}{
+ "name":
azureDevOpsProjectName,
"connectionId": connectionID,
"projectId":
azureDevOpsProjectName,
"repositoryId":
fmt.Sprint(azuredevopsRepoId),
"organizationId": azureDevOpsOrgName,
+ "repositoryType":
models.RepositoryTypeADO,
+ "externalId": "",
},
},
{
Plugin: "gitextractor",
Options: map[string]interface{}{
- "proxy": "",
- "repoId": expectDomainScopeId,
- "name":
azureDevOpsProjectName,
- "url":
"https://git:personal-access-token@this_is_cloneUrl";,
- "noShallowClone": true,
+ "proxy": "",
+ "repoId": expectDomainScopeId,
+ "name": azureDevOpsProjectName,
+ "url":
"https://git:personal-access-token@this_is_cloneUrl";,
},
},
},
@@ -175,3 +181,60 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
assert.Equal(t, expectPlans, actualPlans)
}
+
+func TestMakeRemoteRepoScopes(t *testing.T) {
+ mockAzuredevopsPlugin(t)
+
+ data := []struct {
+ Name string
+ Type string
+ Private bool
+ ExpectedScopes []string
+ }{
+ {Name: "Azure DevOps Repository", Type:
models.RepositoryTypeADO, Private: false, ExpectedScopes:
[]string{"*code.Repo", "*ticket.Board", "*devops.CicdScope"}},
+
+ {Name: "Public GitHub Repository", Type:
models.RepositoryTypeGithub, Private: false, ExpectedScopes:
[]string{"*code.Repo", "*devops.CicdScope"}},
+
+ {Name: "Private GitHub Repository", Type:
models.RepositoryTypeGithub, Private: true, ExpectedScopes:
[]string{"*devops.CicdScope"}},
+ }
+
+ for _, d := range data {
+
+ t.Run(d.Name, func(t *testing.T) {
+ id := strings.ToLower(d.Name)
+ id = strings.ReplaceAll(id, " ", "-")
+ actualScopes, err := makeScopeV200(
+ connectionID,
+
[]*srvhelper.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig]{
+ {
+ Scope: models.AzuredevopsRepo{
+ Scope: common.Scope{
+ ConnectionId:
connectionID,
+ },
+ Id: id,
+ Type: d.Type,
+ Name: d.Name,
+ IsPrivate: d.Private,
+ },
+ ScopeConfig:
&models.AzuredevopsScopeConfig{
+ ScopeConfig:
common.ScopeConfig{
+ Entities:
[]string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_TICKET,
+
plugin.DOMAIN_TYPE_CICD, plugin.DOMAIN_TYPE_CODE_REVIEW},
+ },
+ },
+ },
+ },
+ )
+ assert.Nil(t, err)
+ var count int
+
+ for _, s := range actualScopes {
+ xType := reflect.TypeOf(s)
+ assert.Contains(t, d.ExpectedScopes,
xType.String())
+ count++
+ }
+ assert.Equal(t, count, len(d.ExpectedScopes))
+ })
+
+ }
+}
diff --git a/backend/plugins/azuredevops_go/api/connection_api.go
b/backend/plugins/azuredevops_go/api/connection_api.go
index b096890d9..d29cac85b 100644
--- a/backend/plugins/azuredevops_go/api/connection_api.go
+++ b/backend/plugins/azuredevops_go/api/connection_api.go
@@ -21,6 +21,7 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+
"github.com/apache/incubator-devlake/plugins/azuredevops_go/api/azuredevops"
"github.com/apache/incubator-devlake/plugins/azuredevops_go/models"
"github.com/apache/incubator-devlake/server/api/shared"
"net/http"
@@ -51,9 +52,9 @@ func TestConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
},
AzuredevopsConn: conn,
}
- vsc := newVsClient(&connection, "https://app.vssps.visualstudio.com/";)
+ vsc := azuredevops.NewClient(&connection, nil,
"https://app.vssps.visualstudio.com/";)
- _, err := vsc.UserProfile()
+ _, err := vsc.GetUserProfile()
if err != nil {
return nil, err
}
@@ -79,8 +80,8 @@ func TestExistingConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResource
return nil, errors.BadInput.Wrap(err, "can't read connection
from database")
}
- vsc := newVsClient(connection, "https://app.vssps.visualstudio.com/";)
- _, err = vsc.UserProfile()
+ vsc := azuredevops.NewClient(connection, nil,
"https://app.vssps.visualstudio.com/";)
+ _, err = vsc.GetUserProfile()
if err != nil {
return nil, err
}
diff --git a/backend/plugins/azuredevops_go/api/remote_data.go
b/backend/plugins/azuredevops_go/api/remote_data.go
deleted file mode 100644
index 8596d1c84..000000000
--- a/backend/plugins/azuredevops_go/api/remote_data.go
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "github.com/apache/incubator-devlake/plugins/azuredevops_go/models"
- "time"
-)
-
-type AzuredevopsRemotePagination struct {
- Skip int
- Top int
-}
-
-type AzuredevopsApiRepo struct {
- Id string `json:"id"`
- Name string `json:"name"`
- Url string `json:"url"`
- Project struct {
- Id string `json:"id"`
- Name string `json:"name"`
- Description string `json:"description"`
- Url string `json:"url"`
- State string `json:"state"`
- Revision int `json:"revision"`
- Visibility string `json:"visibility"`
- LastUpdateTime time.Time `json:"lastUpdateTime"`
- } `json:"project"`
- DefaultBranch string `json:"defaultBranch"`
- Size int `json:"size"`
- RemoteUrl string `json:"remoteUrl"`
- SshUrl string `json:"sshUrl"`
- WebUrl string `json:"webUrl"`
- IsDisabled bool `json:"isDisabled"`
- IsInMaintenance bool `json:"isInMaintenance"`
- IsFork bool `json:"isFork"`
-}
-
-func (r AzuredevopsApiRepo) toRepoModel() models.AzuredevopsRepo {
- return models.AzuredevopsRepo{
- Id: r.Id,
- Name: r.Name,
- Url: r.WebUrl,
- AzureDevOpsPK: models.AzureDevOpsPK{
- ProjectId: r.Project.Id,
- },
- RemoteUrl: r.RemoteUrl,
- IsFork: r.IsFork,
- }
-}
diff --git a/backend/plugins/azuredevops_go/api/remote_helper.go
b/backend/plugins/azuredevops_go/api/remote_helper.go
index 9ec2c31f5..300a3fe81 100644
--- a/backend/plugins/azuredevops_go/api/remote_helper.go
+++ b/backend/plugins/azuredevops_go/api/remote_helper.go
@@ -18,21 +18,35 @@ limitations under the License.
package api
import (
+ "context"
"fmt"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
dsmodels
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/models"
+
"github.com/apache/incubator-devlake/plugins/azuredevops_go/api/azuredevops"
"github.com/apache/incubator-devlake/plugins/azuredevops_go/models"
- "net/url"
+ "golang.org/x/exp/slices"
+ "golang.org/x/sync/errgroup"
+ "strconv"
"strings"
+ "sync"
)
const (
- itemsPerPage = 100
- idSeparator = "/"
+ idSeparator = "/"
+ maxConcurrency = 10
)
+type AzuredevopsRemotePagination struct {
+ Skip int
+ Top int
+}
+
+//
https://learn.microsoft.com/en-us/azure/devops/pipelines/repos/?view=azure-devops
+//
https://learn.microsoft.com/en-us/azure/devops/pipelines/repos/multi-repo-checkout?view=azure-devops
+var supportedSourceRepositories = []string{"github", "githubenterprise",
"bitbucket", "git"}
+
func listAzuredevopsRemoteScopes(
connection *models.AzuredevopsConnection,
apiClient plugin.ApiClient,
@@ -43,102 +57,189 @@ func listAzuredevopsRemoteScopes(
nextPage *AzuredevopsRemotePagination,
err errors.Error,
) {
- if page.Top == 0 {
- page.Top = itemsPerPage
+
+ vsc := azuredevops.NewClient(connection, apiClient,
"https://app.vssps.visualstudio.com";)
+
+ if groupId == "" {
+ return listAzuredevopsProjects(vsc, page)
}
- if groupId != "" {
- id := strings.Split(groupId, idSeparator)
- return listAzuredevopsRepos(apiClient, id[0], id[1])
+ id := strings.Split(groupId, idSeparator)
+
+ if remote, err := listRemoteRepos(vsc, id[0], id[1]); err == nil {
+ children = append(children, remote...)
+ }
+
+ if remote, err := listAzuredevopsRepos(vsc, id[0], id[1]); err == nil {
+ children = append(children, remote...)
}
- return listAzuredevopsProjects(connection, apiClient, page)
+ return children, nextPage, nil
}
-func listAzuredevopsProjects(
- connection *models.AzuredevopsConnection,
- apiClient plugin.ApiClient,
- page AzuredevopsRemotePagination,
-) (
+func listAzuredevopsProjects(vsc azuredevops.Client, _
AzuredevopsRemotePagination) (
children []dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo],
nextPage *AzuredevopsRemotePagination,
err errors.Error) {
- query := url.Values{}
- query.Set("$top", fmt.Sprint(page.Top))
- query.Set("$skip", fmt.Sprint(page.Skip))
- query.Set("api-version", "7.1")
-
- vsc := newVsClient(connection, "https://app.vssps.visualstudio.com";)
-
- profile, err := vsc.UserProfile()
+ profile, err := vsc.GetUserProfile()
if err != nil {
return nil, nil, err
}
- accounts, err := vsc.UserAccounts(profile.Id)
+ accounts, err := vsc.GetUserAccounts(profile.Id)
if err != nil {
return nil, nil, err
}
- var data struct {
- Projects
[]dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsProject] `json:"value"`
- }
+ g, _ := errgroup.WithContext(context.Background())
+ g.SetLimit(maxConcurrency)
+
+ var mu sync.Mutex
for _, v := range accounts {
- res, err := apiClient.Get(fmt.Sprintf("%s/_apis/projects",
v.AccountName), query, nil)
- if err != nil {
- return nil, nil, err
- }
- err = api.UnmarshalResponse(res, &data)
- if err != nil {
+ accountName := v.AccountName
+ g.Go(func() error {
+ args := azuredevops.GetProjectsArgs{
+ OrgId: accountName,
+ }
+ projects, err := vsc.GetProjects(args)
if err != nil {
- return nil, nil, err
+ return err
}
- }
- for _, vv := range data.Projects {
- children = append(children,
dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo]{
- Id: v.AccountName + idSeparator + vv.Name,
- Type: api.RAS_ENTRY_TYPE_GROUP,
- Name: vv.Name,
- })
- }
+ var tmp
[]dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo]
+ for _, vv := range projects {
+ tmp = append(tmp,
dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo]{
+ Id: accountName + idSeparator +
vv.Name,
+ Type: api.RAS_ENTRY_TYPE_GROUP,
+ Name: vv.Name,
+ })
+ }
+ mu.Lock()
+ children = append(children, tmp...)
+ mu.Unlock()
+ return nil
+ })
}
- if len(data.Projects) >= itemsPerPage {
- nextPage = &AzuredevopsRemotePagination{
- Top: itemsPerPage,
- Skip: page.Skip + itemsPerPage,
- }
- }
+ err = errors.Convert(g.Wait())
return
}
func listAzuredevopsRepos(
- apiClient plugin.ApiClient,
+ vsc azuredevops.Client,
orgId, projectId string,
) (
children []dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo],
- nextPage *AzuredevopsRemotePagination,
err errors.Error) {
- query := url.Values{}
- query.Set("api-version", "7.1")
-
- var data struct {
- Repos []AzuredevopsApiRepo `json:"value"`
+ args := azuredevops.GetRepositoriesArgs{
+ OrgId: orgId,
+ ProjectId: projectId,
}
- res, err := apiClient.Get(fmt.Sprintf("%s/%s/_apis/git/repositories",
orgId, projectId), query, nil)
+ repos, err := vsc.GetRepositories(args)
if err != nil {
- return nil, nil, err
+ return nil, err
+ }
+
+ for _, v := range repos {
+ if v.IsDisabled {
+ continue
+ }
+
+ pID := orgId + idSeparator + projectId
+ repo := models.AzuredevopsRepo{
+ Id: v.Id,
+ Type: models.RepositoryTypeADO,
+ Name: v.Name,
+ Url: v.Url,
+ RemoteUrl: v.RemoteUrl,
+ IsFork: false,
+ }
+ repo.ProjectId = projectId
+ repo.OrganizationId = orgId
+ children = append(children,
dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo]{
+ Type: api.RAS_ENTRY_TYPE_SCOPE,
+ ParentId: &pID,
+ Id: v.Id,
+ Name: v.Name,
+ FullName: v.Name,
+ Data: &repo,
+ })
+ }
+ return
+}
+
+func listRemoteRepos(
+ vsc azuredevops.Client,
+ orgId, projectId string,
+) (
+ children []dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo],
+ err errors.Error) {
+
+ args := azuredevops.GetServiceEndpointsArgs{
+ OrgId: orgId,
+ ProjectId: projectId,
}
- err = api.UnmarshalResponse(res, &data)
+
+ endpoints, err := vsc.GetServiceEndpoints(args)
if err != nil {
- return nil, nil, err
+ return nil, err
}
- for _, v := range data.Repos {
+
+ var mu sync.Mutex
+ var remoteRepos []azuredevops.RemoteRepository
+
+ g, _ := errgroup.WithContext(context.Background())
+ g.SetLimit(maxConcurrency)
+
+ for _, v := range endpoints {
+ if !slices.Contains(supportedSourceRepositories, v.Type) {
+ continue
+ }
+
+ remoteRepoArgs := azuredevops.GetRemoteRepositoriesArgs{
+ ProjectId: projectId,
+ OrgId: orgId,
+ Provider: v.Type,
+ ServiceEndpoint: v.Id,
+ }
+
+ g.Go(func() error {
+ repos, err := vsc.GetRemoteRepositories(remoteRepoArgs)
+ mu.Lock()
+ remoteRepos = append(remoteRepos, repos...)
+ mu.Unlock()
+ return err
+ })
+ }
+
+ if err := g.Wait(); err != nil {
+ return nil, errors.Internal.Wrap(err, "failed to call
'GetRemoteRepositories', falling back to empty list")
+ }
+
+ for _, v := range remoteRepos {
pID := orgId + idSeparator + projectId
- repo := v.toRepoModel()
+ isFork, _ := strconv.ParseBool(v.Properties.IsFork)
+ isPrivate, _ := strconv.ParseBool(v.Properties.IsPrivate)
+
+ // IDs must not contain URL reserved characters (e.g., "/"), as
this breaks the routing in the scope API.
+ // Accessing
/plugins/azuredevops_go/connections/<id>/apache/incubator-devlake results in a
404 error, where
+ // "apache/incubator-devlake" is the repository ID returned by
ADOs sourceProviders API.
+ // Therefore, we are creating our own ID, by combining the
Service Connection and the External ID
+ remoteId := fmt.Sprintf("%s-%s",
v.Properties.ConnectedServiceId, v.Properties.ExternalId)
+
+ repo := models.AzuredevopsRepo{
+ Id: remoteId,
+ Type: v.SourceProviderName,
+ Name: v.SourceProviderName + idSeparator +
v.FullName,
+ Url: v.Properties.ManageUrl,
+ RemoteUrl: v.Properties.CloneUrl,
+ ExternalId: v.Id,
+ IsFork: isFork,
+ IsPrivate: isPrivate,
+ }
+
repo.ProjectId = projectId
repo.OrganizationId = orgId
children = append(children,
dsmodels.DsRemoteApiScopeListEntry[models.AzuredevopsRepo]{
@@ -146,7 +247,7 @@ func listAzuredevopsRepos(
ParentId: &pID,
Id: v.Id,
Name: v.Name,
- FullName: v.Name,
+ FullName: v.FullName,
Data: &repo,
})
}
diff --git a/backend/plugins/azuredevops_go/api/vs_client.go
b/backend/plugins/azuredevops_go/api/vs_client.go
deleted file mode 100644
index 961e761d2..000000000
--- a/backend/plugins/azuredevops_go/api/vs_client.go
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "encoding/json"
- "fmt"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/plugins/azuredevops_go/models"
- "io"
- "net/http"
- "net/url"
- "time"
-)
-
-type Profile struct {
- DisplayName string `json:"displayName"`
- PublicAlias string `json:"publicAlias"`
- EmailAddress string `json:"emailAddress"`
- CoreRevision int `json:"coreRevision"`
- TimeStamp time.Time `json:"timeStamp"`
- Id string `json:"id"`
- Revision int `json:"revision"`
-}
-
-type Account struct {
- AccountId string `json:"AccountId"`
- NamespaceId string `json:"NamespaceId"`
- AccountName string `json:"AccountName"`
- OrganizationName interface{} `json:"OrganizationName"`
- AccountType int `json:"AccountType"`
- AccountOwner string `json:"AccountOwner"`
- CreatedBy string `json:"CreatedBy"`
- CreatedDate string `json:"CreatedDate"`
- AccountStatus int `json:"AccountStatus"`
- StatusReason interface{} `json:"StatusReason"`
- LastUpdatedBy string `json:"LastUpdatedBy"`
- Properties struct {
- } `json:"Properties"`
-}
-
-type AccountResponse []Account
-
-type vsClient struct {
- c http.Client
- connection *models.AzuredevopsConnection
- url string
-}
-
-func newVsClient(con *models.AzuredevopsConnection, url string) vsClient {
- return vsClient{
- c: http.Client{
- Timeout: 2 * time.Second,
- },
- connection: con,
- url: url,
- }
-}
-
-func (vsc *vsClient) UserProfile() (Profile, errors.Error) {
- var p Profile
- endpoint, err := url.JoinPath(vsc.url, "/_apis/profile/profiles/me")
- if err != nil {
- return Profile{}, errors.Internal.Wrap(err, "failed to join
user profile path")
- }
-
- res, err := vsc.doGet(endpoint)
- if err != nil {
- return Profile{}, errors.Internal.Wrap(err, "failed to read
user accounts")
- }
-
- if res.StatusCode == 302 || res.StatusCode == 401 {
- return Profile{}, errors.Unauthorized.New("failed to read user
profile")
- }
-
- defer res.Body.Close()
- resBody, err := io.ReadAll(res.Body)
- if err != nil {
- return Profile{}, errors.Internal.Wrap(err, "failed to read
response body")
- }
-
- if err := json.Unmarshal(resBody, &p); err != nil {
- panic(err)
- }
- return p, nil
-}
-
-func (vsc *vsClient) UserAccounts(memberId string) (AccountResponse,
errors.Error) {
- var a AccountResponse
- endpoint := fmt.Sprintf(vsc.url+"/_apis/accounts?memberId=%s", memberId)
- res, err := vsc.doGet(endpoint)
- if err != nil {
- return nil, errors.Internal.Wrap(err, "failed to read user
accounts")
- }
-
- if res.StatusCode == 302 || res.StatusCode == 401 {
- return nil, errors.Unauthorized.New("failed to read user
accounts")
- }
-
- defer res.Body.Close()
- resBody, err := io.ReadAll(res.Body)
- if err != nil {
- return nil, errors.Internal.Wrap(err, "failed to read response
body")
- }
-
- if err := json.Unmarshal(resBody, &a); err != nil {
- return nil, errors.Internal.Wrap(err, "failed to read unmarshal
response body")
- }
- return a, nil
-}
-
-func (vsc *vsClient) doGet(url string) (*http.Response, error) {
- req, err := http.NewRequest(http.MethodGet, url, nil)
- if err != nil {
- return nil, err
- }
-
- if err =
vsc.connection.GetAccessTokenAuthenticator().SetupAuthentication(req); err !=
nil {
- return nil, errors.Internal.Wrap(err, "failed to authorize the
request using the plugin connection")
- }
- return http.DefaultClient.Do(req)
-}
diff --git a/backend/plugins/azuredevops_go/e2e/build_test.go
b/backend/plugins/azuredevops_go/e2e/build_test.go
index 61a776a0e..a27eebd2b 100644
--- a/backend/plugins/azuredevops_go/e2e/build_test.go
+++ b/backend/plugins/azuredevops_go/e2e/build_test.go
@@ -44,6 +44,7 @@ func TestAzuredevopsBuildDataFlow(t *testing.T) {
ProjectId: "test-project",
OrganizationId: "johndoe",
RepositoryId: "0d50ba13-f9ad-49b0-9b21-d29eda50ca33",
+ RepositoryType: models.RepositoryTypeADO,
ScopeConfig: new(models.AzuredevopsScopeConfig),
},
RegexEnricher: regexEnricher,
diff --git a/backend/plugins/azuredevops_go/models/base.go
b/backend/plugins/azuredevops_go/models/base.go
index 9c60b81a1..e30369a7d 100644
--- a/backend/plugins/azuredevops_go/models/base.go
+++ b/backend/plugins/azuredevops_go/models/base.go
@@ -21,3 +21,8 @@ type AzureDevOpsPK struct {
OrganizationId string
ProjectId string
}
+
+const (
+ RepositoryTypeADO = "TfsGit"
+ RepositoryTypeGithub = "github"
+)
diff --git
a/backend/plugins/azuredevops_go/models/migrationscripts/20240413_add_remote_repo_support.go
b/backend/plugins/azuredevops_go/models/migrationscripts/20240413_add_remote_repo_support.go
new file mode 100644
index 000000000..d9c514697
--- /dev/null
+++
b/backend/plugins/azuredevops_go/models/migrationscripts/20240413_add_remote_repo_support.go
@@ -0,0 +1,68 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+type extendRepoTable struct{}
+
+type ExtendAzuredevopsRepo struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Id string `gorm:"primaryKey;column:id"`
+
+ Type string `gorm:"type:varchar(100)"`
+ IsPrivate bool
+ ExternalId string `gorm:"type:varchar(255)"`
+}
+
+func (ExtendAzuredevopsRepo) TableName() string {
+ return "_tool_azuredevops_go_repos"
+}
+
+func (*extendRepoTable) Up(baseRes context.BasicRes) errors.Error {
+ err := migrationhelper.AutoMigrateTables(baseRes,
&ExtendAzuredevopsRepo{})
+ if err != nil {
+ return err
+ }
+
+ err = baseRes.GetDal().UpdateColumn(
+ &ExtendAzuredevopsRepo{}, "type", "TfsGit",
+ dal.Where("type IS NULL"))
+
+ if err != nil {
+ return err
+ }
+
+ return baseRes.GetDal().UpdateColumn(
+ &ExtendAzuredevopsRepo{}, "is_private", false,
+ dal.Where("is_private IS NULL"))
+
+}
+
+func (*extendRepoTable) Version() uint64 {
+ return 20240413100000
+}
+
+func (*extendRepoTable) Name() string {
+ return "add [type, is_private, external_id] to
_tool_azuredevops_go_repos in order to support remote repositories"
+}
diff --git a/backend/plugins/azuredevops_go/models/migrationscripts/register.go
b/backend/plugins/azuredevops_go/models/migrationscripts/register.go
index ec054748c..59fa7832f 100644
--- a/backend/plugins/azuredevops_go/models/migrationscripts/register.go
+++ b/backend/plugins/azuredevops_go/models/migrationscripts/register.go
@@ -25,5 +25,6 @@ import (
func All() []plugin.MigrationScript {
return []plugin.MigrationScript{
new(addInitTables),
+ new(extendRepoTable),
}
}
diff --git a/backend/plugins/azuredevops_go/models/repo.go
b/backend/plugins/azuredevops_go/models/repo.go
index e3f799a91..dd171996f 100644
--- a/backend/plugins/azuredevops_go/models/repo.go
+++ b/backend/plugins/azuredevops_go/models/repo.go
@@ -28,11 +28,14 @@ type AzuredevopsRepo struct {
common.Scope `mapstructure:",squash"`
AzureDevOpsPK `mapstructure:",squash"`
- Id string `json:"id" validate:"required" mapstructure:"id"
gorm:"primaryKey"`
- Name string `json:"name" mapstructure:"name,omitempty"`
- Url string `json:"url" mapstructure:"url,omitempty"`
- RemoteUrl string `json:"remoteUrl"`
- IsFork bool
+ Id string `json:"id" validate:"required" mapstructure:"id"
gorm:"primaryKey"`
+ Type string `json:"type" validate:"required" mapstructure:"type"`
+ Name string `json:"name" mapstructure:"name,omitempty"`
+ Url string `json:"url" mapstructure:"url,omitempty"`
+ RemoteUrl string `json:"remoteUrl"`
+ ExternalId string
+ IsFork bool
+ IsPrivate bool
}
func (repo AzuredevopsRepo) ScopeId() string {
diff --git a/backend/plugins/azuredevops_go/tasks/ci_cd_build_collector.go
b/backend/plugins/azuredevops_go/tasks/ci_cd_build_collector.go
index f70a12076..257c5fac0 100644
--- a/backend/plugins/azuredevops_go/tasks/ci_cd_build_collector.go
+++ b/backend/plugins/azuredevops_go/tasks/ci_cd_build_collector.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+ "github.com/apache/incubator-devlake/plugins/azuredevops_go/models"
"net/url"
"strconv"
"time"
@@ -46,6 +47,15 @@ var CollectBuildsMeta = plugin.SubTaskMeta{
func CollectBuilds(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RawBuildTable)
repoId := data.Options.RepositoryId
+ if data.Options.RepositoryType != models.RepositoryTypeADO {
+ repoId = data.Options.ExternalId
+ }
+ repoType := data.Options.RepositoryType
+ if repoType == "" {
+ repoType = models.RepositoryTypeADO
+ taskCtx.GetLogger().Warn(nil, "repository type for repoId: %v
not found. falling back to TfsGit", repoId)
+ }
+
collector, err :=
api.NewStatefulApiCollectorForFinalizableEntity(api.FinalizableApiCollectorArgs{
RawDataSubTaskArgs: *rawDataSubTaskArgs,
ApiClient: data.ApiClient,
@@ -56,7 +66,7 @@ func CollectBuilds(taskCtx plugin.SubTaskContext)
errors.Error {
UrlTemplate: "{{ .Params.OrganizationId }}/{{
.Params.ProjectId }}/_apis/build/builds?api-version=7.1",
Query: func(reqData *api.RequestData,
createdAfter *time.Time) (url.Values, errors.Error) {
query := url.Values{}
- query.Set("repositoryType", "tfsgit")
+ query.Set("repositoryType", repoType)
query.Set("repositoryId", repoId)
query.Set("$top",
strconv.Itoa(reqData.Pager.Size))
query.Set("queryOrder",
"queueTimeDescending")
diff --git a/backend/plugins/azuredevops_go/tasks/ci_cd_build_converter.go
b/backend/plugins/azuredevops_go/tasks/ci_cd_build_converter.go
index 505984ca7..a3d6c240b 100644
--- a/backend/plugins/azuredevops_go/tasks/ci_cd_build_converter.go
+++ b/backend/plugins/azuredevops_go/tasks/ci_cd_build_converter.go
@@ -38,7 +38,7 @@ var ConvertBuildsMeta = plugin.SubTaskMeta{
EntryPoint: ConvertBuilds,
EnabledByDefault: true,
Description: "Convert tool layer table azuredevops_builds into
domain layer table cicd_pipelines",
- DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW},
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
DependencyTables: []string{
models.AzuredevopsBuild{}.TableName(),
},
diff --git a/backend/plugins/azuredevops_go/tasks/task_data.go
b/backend/plugins/azuredevops_go/tasks/task_data.go
index 1e5baa62a..2ac479156 100644
--- a/backend/plugins/azuredevops_go/tasks/task_data.go
+++ b/backend/plugins/azuredevops_go/tasks/task_data.go
@@ -31,6 +31,8 @@ type AzuredevopsOptions struct {
ProjectId string `json:"projectId"
mapstructure:"projectId,omitempty"`
OrganizationId string `json:"organizationId"
mapstructure:"organizationId,omitempty"`
RepositoryId string `json:"repositoryId"
mapstructure:"repositoryId,omitempty"`
+ RepositoryType string `json:"repositoryType"
mapstructure:"repositoryType,omitempty"`
+ ExternalId string `json:"externalId"
mapstructure:"externalId,omitempty"`
ScopeConfigId uint64 `json:"scopeConfigId"
mapstructure:"scopeConfigId,omitempty"`
TimeAfter string `json:"timeAfter"
mapstructure:"timeAfter,omitempty"`
@@ -62,7 +64,7 @@ type AzuredevopsParams struct {
func (p *AzuredevopsOptions) GetParams() any {
return AzuredevopsParams{
OrganizationId: p.OrganizationId,
- RepositoryId: p.RepositoryId,
ProjectId: p.ProjectId,
+ RepositoryId: p.RepositoryId,
}
}
diff --git a/backend/test/e2e/manual/azuredevops/models.go
b/backend/test/e2e/manual/azuredevops/models.go
index 3ea19ffb3..e9b83dac9 100644
--- a/backend/test/e2e/manual/azuredevops/models.go
+++ b/backend/test/e2e/manual/azuredevops/models.go
@@ -46,6 +46,7 @@ type (
AzureGitRepo struct {
RawDataParams string `json:"_raw_data_params"`
Id string
+ Type string
Name string
ConnectionId uint64
Url string
diff --git a/config-ui/src/routes/pipeline/components/task.tsx
b/config-ui/src/routes/pipeline/components/task.tsx
index 3f7212042..d4a012a11 100644
--- a/config-ui/src/routes/pipeline/components/task.tsx
+++ b/config-ui/src/routes/pipeline/components/task.tsx
@@ -86,6 +86,9 @@ export const PipelineTask = ({ task }: Props) => {
case ['bamboo'].includes(config.plugin):
name = `${name}:${options.planKey}`;
break;
+ case ['azuredevops_go'].includes(config.plugin):
+ name = `ado:${options.name}`;
+ break;
}
return [config.icon, name];
