This is an automated email from the ASF dual-hosted git repository. klesh pushed a commit to branch kw-7343-azdo-git-clone-failed in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 101271a3589c418373631c59bcc74eac2b4c78a4 Author: Klesh Wong <[email protected]> AuthorDate: Mon Apr 22 14:28:00 2024 +0800 fix: git clone failing on azuredevops --- .../plugins/azuredevops_go/api/blueprint_v200.go | 9 +++-- .../plugins/gitextractor/parser/clone_gitcli.go | 43 +++++++++++++--------- backend/plugins/gitextractor/parser/taskdata.go | 3 +- .../python/plugins/azuredevops/azuredevops/main.py | 2 +- .../python/pydevlake/pydevlake/pipeline_tasks.py | 5 ++- 5 files changed, 36 insertions(+), 26 deletions(-) diff --git a/backend/plugins/azuredevops_go/api/blueprint_v200.go b/backend/plugins/azuredevops_go/api/blueprint_v200.go index a9d178e47..f1df9be8c 100644 --- a/backend/plugins/azuredevops_go/api/blueprint_v200.go +++ b/backend/plugins/azuredevops_go/api/blueprint_v200.go @@ -136,10 +136,11 @@ func makePipelinePlanV200( stage = append(stage, &coreModels.PipelineTask{ Plugin: "gitextractor", Options: map[string]interface{}{ - "url": cloneUrl.String(), - "name": azuredevopsRepo.Name, - "repoId": didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connection.ID, azuredevopsRepo.Id), - "proxy": connection.Proxy, + "url": cloneUrl.String(), + "name": azuredevopsRepo.Name, + "repoId": didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connection.ID, azuredevopsRepo.Id), + "proxy": connection.Proxy, + "noShallowClone": true, }, }) } diff --git a/backend/plugins/gitextractor/parser/clone_gitcli.go b/backend/plugins/gitextractor/parser/clone_gitcli.go index 500a30c97..1a666ad42 100644 --- a/backend/plugins/gitextractor/parser/clone_gitcli.go +++ b/backend/plugins/gitextractor/parser/clone_gitcli.go @@ -50,38 +50,45 @@ func NewGitcliCloner(basicRes context.BasicRes) *GitcliCloner { func (g *GitcliCloner) CloneRepo(ctx plugin.SubTaskContext, localDir string) errors.Error { taskData := ctx.GetData().(*GitExtractorTaskData) - // load state - stateManager, err := api.NewCollectorStateManager( - ctx, - ctx.TaskContext().SyncPolicy(), - "gitextractor", - fmt.Sprintf( - `{"RepoId: "%s","SkipCommitStat": %v, "SkipCommitFiles": %v}`, - taskData.Options.RepoId, - *taskData.Options.SkipCommitStat, - *taskData.Options.SkipCommitFiles, - ), - ) - if err != nil { - return err + var since *time.Time + if !taskData.Options.NoShallowClone { + // load state + stateManager, err := api.NewCollectorStateManager( + ctx, + ctx.TaskContext().SyncPolicy(), + "gitextractor", + fmt.Sprintf( + `{"RepoId: "%s","SkipCommitStat": %v, "SkipCommitFiles": %v}`, + taskData.Options.RepoId, + *taskData.Options.SkipCommitStat, + *taskData.Options.SkipCommitFiles, + ), + ) + if err != nil { + return err + } + g.stateManager = stateManager + since = stateManager.GetSince() } - g.stateManager = stateManager - cmd, err := g.buildCloneCommand(ctx, localDir, g.stateManager.GetSince()) + cmd, err := g.buildCloneCommand(ctx, localDir, since) if err != nil { return err } err = g.execCloneCommand(cmd) if err != nil { // it is likely that nothing to collect on incrmental mode - if errors.Is(err, ErrShallowInfoProcessing) && stateManager.IsIncremental() { + if errors.Is(err, ErrShallowInfoProcessing) && g.stateManager != nil && g.stateManager.IsIncremental() { return ErrNoDataOnIncrementalMode } return err } // save state - return g.stateManager.Close() + if g.stateManager != nil { + return g.stateManager.Close() + } + return nil } func (g *GitcliCloner) buildCloneCommand(ctx plugin.SubTaskContext, localDir string, since *time.Time) (*exec.Cmd, errors.Error) { diff --git a/backend/plugins/gitextractor/parser/taskdata.go b/backend/plugins/gitextractor/parser/taskdata.go index b0a44c660..a2d894d02 100644 --- a/backend/plugins/gitextractor/parser/taskdata.go +++ b/backend/plugins/gitextractor/parser/taskdata.go @@ -25,7 +25,7 @@ type GitExtractorTaskData struct { Options *GitExtractorOptions ParsedURL *url.URL GitRepo RepoCollector - SkipAllSubtasks bool // skip all tasks without error if true + SkipAllSubtasks bool // siliently skip all tasks without raising error } type GitExtractorOptions struct { @@ -40,4 +40,5 @@ type GitExtractorOptions struct { UseGoGit *bool `json:"useGoGit" mapstructure:"useGoGit"` SkipCommitStat *bool `json:"skipCommitStat" mapstructure:"skipCommitStat" comment:"skip all commit stat including added/deleted lines and commit files as well"` SkipCommitFiles *bool `json:"skipCommitFiles" mapstructure:"skipCommitFiles"` + NoShallowClone bool `json:"noShallowClone" mapstructure:"noShallowClone"` } diff --git a/backend/python/plugins/azuredevops/azuredevops/main.py b/backend/python/plugins/azuredevops/azuredevops/main.py index 2560ab439..e66422522 100644 --- a/backend/python/plugins/azuredevops/azuredevops/main.py +++ b/backend/python/plugins/azuredevops/azuredevops/main.py @@ -135,7 +135,7 @@ class AzureDevOpsPlugin(Plugin): if DomainType.CODE in scope_config.domain_types and not scope.is_external(): url = urlparse(scope.remote_url) url = url._replace(netloc=f'{url.username}:{connection.token.get_secret_value()}@{url.hostname}') - yield gitextractor(url.geturl(), scope.name, scope.domain_id(), connection.proxy) + yield gitextractor(url.geturl(), scope.name, scope.domain_id(), connection.proxy, True) def extra_stages(self, scope_config_pairs: list[tuple[GitRepository, GitRepositoryConfig]], _): for scope, config in scope_config_pairs: diff --git a/backend/python/pydevlake/pydevlake/pipeline_tasks.py b/backend/python/pydevlake/pydevlake/pipeline_tasks.py index b62aaeff6..751d0069f 100644 --- a/backend/python/pydevlake/pydevlake/pipeline_tasks.py +++ b/backend/python/pydevlake/pydevlake/pipeline_tasks.py @@ -21,7 +21,7 @@ from pydantic import BaseModel from pydevlake.message import PipelineTask -def gitextractor(url: str, repo_name: str, scope_id: str, proxy: Optional[str]): +def gitextractor(url: str, repo_name: str, scope_id: str, proxy: Optional[str], no_shallow_clone: Optional[bool] = False): return PipelineTask( plugin="gitextractor", options={ @@ -29,7 +29,8 @@ def gitextractor(url: str, repo_name: str, scope_id: str, proxy: Optional[str]): "name": repo_name, "fullName": repo_name, "repoId": scope_id, - "proxy": proxy + "proxy": proxy, + "noShallowClone": no_shallow_clone, }, )
