mappjzc commented on code in PR #3847:
URL:
https://github.com/apache/incubator-devlake/pull/3847#discussion_r1041059394
##########
plugins/gitlab/api/blueprint_v200.go:
##########
@@ -58,23 +77,93 @@ func MakeDataSourcePipelinePlanV200(connectionId uint64,
scopes []*core.Blueprin
sc = append(sc, &repo)
sc = append(sc, &board)
+ }
- ps = append(ps, &core.PipelineTask{
- Plugin: "gitlab",
- Options: map[string]interface{}{
- "name": scope.Name,
- },
- })
+ return sc, nil
+}
- ps = append(ps, &core.PipelineTask{
- Plugin: "gitextractor",
- Options: map[string]interface{}{
- "url": connection.Endpoint + scope.Name,
- },
- })
- }
+func makePipelinePlanV200(subtaskMetas []core.SubTaskMeta, scopes
[]*core.BlueprintScopeV200, connection *models.GitlabConnection)
(core.PipelinePlan, errors.Error) {
+ var err errors.Error
- pp = append(pp, ps)
+ plans := make(core.PipelinePlan, 0, 3*len(scopes))
+ for _, scope := range scopes {
+ var stage core.PipelineStage
+ // get repo
+ repo := &models.GitlabProject{}
+ err = BasicRes.GetDal().First(repo, dal.Where("connection_id =
? AND gitlab_id = ?", connection.ID, scope.Id))
+ if err != nil {
+ return nil, err
+ }
+
+ // get transformationRuleId
+ var transformationRules models.GitlabTransformationRule
+ transformationRuleId := repo.TransformationRuleId
+ if transformationRuleId != 0 {
+ err = BasicRes.GetDal().First(&transformationRules,
dal.Where("id = ?", transformationRuleId))
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on
get TransformationRule")
+ }
+ } else {
+ transformationRules.ID = 0
+ }
+
+ // refdiff part
+ if transformationRules.Refdiff != nil {
+ task := &core.PipelineTask{
+ Plugin: "refdiff",
+ Options: transformationRules.Refdiff,
+ }
+ stage = append(stage, task)
+ }
+
+ // get int scopeId
+ intScopeId, err1 := strconv.Atoi(scope.Id)
+ if err != nil {
+ return nil, errors.Default.Wrap(err1,
fmt.Sprintf("Failed to strconv.Atoi for scope.Id [%s]", scope.Id))
+ }
+
+ // gitlab main part
+ options := make(map[string]interface{})
+ options["connectionId"] = connection.ID
+ options["projectId"] = intScopeId
+ options["transformationRules"] = &transformationRules
+ options["transformationRuleId"] = transformationRules.ID
+ // make sure task options is valid
+ _, err := tasks.DecodeAndValidateTaskOptions(options)
+ if err != nil {
+ return nil, err
+ }
+
+ // construct subtasks
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
scope.Entities)
+ if err != nil {
+ return nil, err
+ }
+
+ stage = append(stage, &core.PipelineTask{
+ Plugin: "gitlab",
+ Subtasks: subtasks,
+ Options: options,
+ })
- return pp, sc, nil
+ // collect git data by gitextractor if CODE was requested
+ if utils.StringsContains(scope.Entities, core.DOMAIN_TYPE_CODE)
{
+ cloneUrl, err :=
errors.Convert01(url.Parse(repo.HttpUrlToRepo))
Review Comment:
In PlanV100, it gets the HttpUrlToRepo from the GitLab API to
GitlabApiProject.
And HttpUrlToRepo is set in here.
##########
plugins/gitlab/api/blueprint_V200_test.go:
##########
@@ -0,0 +1,263 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "encoding/json"
+ "strconv"
+ "testing"
+ "time"
+
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/helpers/unithelper"
+ "github.com/apache/incubator-devlake/mocks"
+ "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/ticket"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "github.com/apache/incubator-devlake/plugins/gitlab/tasks"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/go-playground/validator/v10"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+)
+
+const TestConnectionID uint64 = 1
+const TestTransformationRuleId uint64 = 2
+const TestID int = 37
+const TestGitlabEndPoint string = "https://gitlab.com/api/v4/";
+const TestHHttpUrlToRepo string = "https://this_is_cloneUrl";
+const TestToken string = "nddtf"
+const TestName string = "gitlab-test"
+const TestTransformationRuleName string = "github transformation rule"
+const TestProxy string = ""
+
+var TestGitlabProject *models.GitlabProject = &models.GitlabProject{
+ ConnectionId: TestConnectionID,
+ GitlabId: TestID,
+ Name: TestName,
+
+ TransformationRuleId: TestTransformationRuleId,
+ CreatedDate: time.Time{},
+ HttpUrlToRepo: TestHHttpUrlToRepo,
+}
+
+var TestTransformationRule *models.GitlabTransformationRule =
&models.GitlabTransformationRule{
+ Model: common.Model{
+ ID: TestTransformationRuleId,
+ },
+ Name: TestTransformationRuleName,
+ PrType: "hey,man,wasup",
+ RefdiffRule: map[string]interface{}{
+ "tagsPattern": "pattern",
+ "tagsLimit": 10,
+ "tagsOrder": "reverse semver",
+ },
+}
+
+var TestGitlabConnection *models.GitlabConnection = &models.GitlabConnection{
+ RestConnection: helper.RestConnection{
+ BaseConnection: helper.BaseConnection{
+ Name: TestName,
+ Model: common.Model{
+ ID: TestConnectionID,
+ },
+ },
+ Endpoint: TestGitlabEndPoint,
+ Proxy: TestProxy,
+ RateLimitPerHour: 0,
+ },
+ AccessToken: helper.AccessToken{
+ Token: TestToken,
+ },
+}
+
+var ExpectRepoId = "gitlab:GitlabProject:1:37"
+
+var TestSubTaskMeta = []core.SubTaskMeta{
+ tasks.CollectProjectMeta,
+ tasks.ExtractProjectMeta,
+ tasks.ConvertProjectMeta,
+ tasks.CollectApiIssuesMeta,
+ tasks.ExtractApiIssuesMeta,
+ tasks.ConvertIssuesMeta,
+ tasks.CollectApiMergeRequestsMeta,
+ tasks.ExtractApiMergeRequestsMeta,
+ tasks.ConvertApiMergeRequestsMeta,
+ tasks.CollectApiMrNotesMeta,
+ tasks.ExtractApiMrNotesMeta,
+ tasks.CollectApiMrCommitsMeta,
+ tasks.ExtractApiMrCommitsMeta,
+ tasks.ConvertApiMrCommitsMeta,
+ tasks.CollectApiPipelinesMeta,
+ tasks.ExtractApiPipelinesMeta,
+ tasks.CollectApiJobsMeta,
+ tasks.ExtractApiJobsMeta,
+ tasks.EnrichMergeRequestsMeta,
+ tasks.CollectAccountsMeta,
+ tasks.ExtractAccountsMeta,
+ tasks.ConvertAccountsMeta,
+ tasks.ConvertCommitsMeta,
+ tasks.ConvertIssueLabelsMeta,
+ tasks.ConvertJobMeta,
+ tasks.ConvertMrCommentMeta,
+ tasks.ConvertMrLabelsMeta,
+ tasks.ConvertPipelineMeta,
+ tasks.ConvertPipelineCommitMeta,
+}
+
+var ExpectPlans core.PipelinePlan = core.PipelinePlan{
+ {
+ {
+ Plugin: "refdiff",
+ SkipOnFail: false,
+ Options: map[string]interface{}{
+ "tagsLimit": 10,
+ "tagsOrder": "reverse semver",
+ "tagsPattern": "pattern",
+ },
+ },
+ {
+ Plugin: "gitlab",
+ Subtasks: []string{
+ tasks.CollectProjectMeta.Name,
+ tasks.ExtractProjectMeta.Name,
+ tasks.ConvertProjectMeta.Name,
+ tasks.CollectApiIssuesMeta.Name,
+ tasks.ExtractApiIssuesMeta.Name,
+ tasks.ConvertIssuesMeta.Name,
+ tasks.CollectApiMergeRequestsMeta.Name,
+ tasks.ExtractApiMergeRequestsMeta.Name,
+ tasks.ConvertApiMergeRequestsMeta.Name,
+ tasks.CollectApiMrNotesMeta.Name,
+ tasks.ExtractApiMrNotesMeta.Name,
+ tasks.CollectApiMrCommitsMeta.Name,
+ tasks.ExtractApiMrCommitsMeta.Name,
+ tasks.ConvertApiMrCommitsMeta.Name,
+ tasks.CollectApiPipelinesMeta.Name,
+ tasks.ExtractApiPipelinesMeta.Name,
+ tasks.CollectApiJobsMeta.Name,
+ tasks.ExtractApiJobsMeta.Name,
+ tasks.EnrichMergeRequestsMeta.Name,
+ tasks.CollectAccountsMeta.Name,
+ tasks.ExtractAccountsMeta.Name,
+ tasks.ConvertAccountsMeta.Name,
+ tasks.ConvertIssueLabelsMeta.Name,
+ tasks.ConvertJobMeta.Name,
+ tasks.ConvertMrCommentMeta.Name,
+ tasks.ConvertMrLabelsMeta.Name,
+ tasks.ConvertPipelineMeta.Name,
+ tasks.ConvertPipelineCommitMeta.Name,
+ },
+ SkipOnFail: false,
+ Options: map[string]interface{}{
+ "connectionId": uint64(1),
+ "projectId": TestID,
+ "transformationRuleId":
TestTransformationRuleId,
+ "transformationRules": TestTransformationRule,
+ },
+ },
+ {
+ Plugin: "gitextractor",
+ SkipOnFail: false,
+ Options: map[string]interface{}{
+ "proxy": "",
+ "repoId": ExpectRepoId,
+ "url": "https://git:nddtf@this_is_cloneUrl";,
+ },
+ },
+ },
+}
+
+var ExpectScopes []core.Scope = []core.Scope{
+ &code.Repo{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: ExpectRepoId,
+ },
+ Name: TestName,
+ },
+ &ticket.Board{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: ExpectRepoId,
+ },
+ Name: TestName,
+ Description: "",
+ Url: "",
+ CreatedDate: nil,
+ Type: "",
+ },
+}
+
+func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
+ var err errors.Error
+
+ bpScopes := []*core.BlueprintScopeV200{
+ {
+ Entities: []string{"CODE", "TICKET", "CODEREVIEW",
"CROSS", "CICD"},
+ Id: strconv.Itoa(TestID),
+ Name: TestName,
+ },
+ }
+ // register gitlab plugin for NewDomainIdGenerator
+ mockMeta := mocks.NewPluginMeta(t)
+
mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/gitlab")
+ err = core.RegisterPlugin("gitlab", mockMeta)
+ assert.Equal(t, err, nil)
+
+ // Refresh Global Variables and set the sql mock
+ BasicRes = unithelper.NewMockBasicRes(func(mockDal *mocks.Dal) {
+ mockDal.On("First", mock.Anything, mock.Anything).Run(func(args
mock.Arguments) {
+ dst := args.Get(0).(*models.GitlabConnection)
+ *dst = *TestGitlabConnection
+ }).Return(nil).Once()
+
+ mockDal.On("First", mock.Anything, mock.Anything).Run(func(args
mock.Arguments) {
+ dst := args.Get(0).(*models.GitlabProject)
+ *dst = *TestGitlabProject
+ }).Return(nil).Once()
+
+ mockDal.On("First", mock.Anything, mock.Anything).Run(func(args
mock.Arguments) {
+ dst := args.Get(0).(*models.GitlabTransformationRule)
+ *dst = *TestTransformationRule
+ }).Return(nil).Once()
+ })
+ connectionHelper = helper.NewConnectionHelper(
+ BasicRes,
+ validator.New(),
+ )
+
+ plans, scopes, err := MakePipelinePlanV200(TestSubTaskMeta,
TestConnectionID, bpScopes)
+ assert.Equal(t, err, nil)
+
+ expectPlansJson, err1 := json.Marshal(plans)
Review Comment:
Verify the expectedPlan will only verify the pointer address for some field,
it is not work well.
##########
helpers/unithelper/dummy_logger.go:
##########
@@ -31,7 +31,21 @@ func DummyLogger() *mocks.Logger {
logger.On("Debug", mock.Anything, mock.Anything).Maybe()
logger.On("Info", mock.Anything, mock.Anything).Maybe()
logger.On("Warn", mock.Anything, mock.Anything).Maybe()
- logger.On("Error", mock.Anything, mock.Anything).Maybe()
+ logger.On("Error", mock.Anything, mock.Anything, mock.Anything).Maybe()
logger.On("Nested", mock.Anything).Return(logger).Maybe()
return logger
}
+
+// NewMockBasicRes FIXME ...
+func NewMockBasicRes(callback func(mockDal *mocks.Dal)) *mocks.BasicRes {
Review Comment:
Do you think it is better to create a new file for this function?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
