This is an automated email from the ASF dual-hosted git repository. abeizn pushed a commit to branch release-v1.0 in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 2fdc61fd7190a7c0491a097f16e245a47a968ac0 Author: abeizn <[email protected]> AuthorDate: Tue Mar 26 11:29:31 2024 +0800 feat: add subtask logs details api (#7205) * fix: add subtask logs details api and update some subtasks name * fix: gitextractor name * fix: subtask migration add fix panic * feat: add completed rate * fix: rename some plugins subtasks name * feat: add total status field * fix: update task status logic * feat: get lastest subtask plugins --- .../20240322_add_field_to_subtasks.go | 52 ++++++++ backend/core/models/migrationscripts/register.go | 1 + backend/core/models/task.go | 72 ++++++++-- backend/core/plugin/plugin_task.go | 12 +- backend/core/runner/run_task.go | 67 +++++++++- backend/helpers/pluginhelper/api/api_collector.go | 2 +- .../plugins/bitbucket/tasks/account_convertor.go | 2 +- .../plugins/bitbucket/tasks/commit_collector.go | 2 +- .../plugins/bitbucket/tasks/commit_convertor.go | 5 +- .../plugins/bitbucket/tasks/commit_extractor.go | 5 +- .../bitbucket/tasks/deployment_collector.go | 2 +- .../bitbucket/tasks/deployment_convertor.go | 9 +- .../bitbucket/tasks/deployment_extractor.go | 5 +- backend/plugins/bitbucket/tasks/issue_collector.go | 2 +- .../bitbucket/tasks/issue_comment_collector.go | 2 +- .../bitbucket/tasks/issue_comment_convertor.go | 2 +- .../bitbucket/tasks/issue_comment_extractor.go | 2 +- backend/plugins/bitbucket/tasks/issue_convertor.go | 2 +- backend/plugins/bitbucket/tasks/issue_extractor.go | 2 +- .../plugins/bitbucket/tasks/pipeline_collector.go | 2 +- .../plugins/bitbucket/tasks/pipeline_convertor.go | 2 +- .../plugins/bitbucket/tasks/pipeline_extractor.go | 3 +- .../bitbucket/tasks/pipeline_steps_collector.go | 2 +- .../bitbucket/tasks/pipeline_steps_convertor.go | 2 +- .../bitbucket/tasks/pipeline_steps_extractor.go | 2 +- backend/plugins/bitbucket/tasks/pr_collector.go | 2 +- .../bitbucket/tasks/pr_comment_collector.go | 2 +- .../bitbucket/tasks/pr_comment_convertor.go | 2 +- .../bitbucket/tasks/pr_comment_extractor.go | 2 +- .../plugins/bitbucket/tasks/pr_commit_collector.go | 2 +- .../plugins/bitbucket/tasks/pr_commit_convertor.go | 2 +- .../plugins/bitbucket/tasks/pr_commit_extractor.go | 2 +- backend/plugins/bitbucket/tasks/pr_convertor.go | 2 +- backend/plugins/bitbucket/tasks/pr_extractor.go | 2 +- backend/plugins/bitbucket/tasks/repo_convertor.go | 2 +- backend/plugins/gitextractor/tasks/clone.go | 5 +- .../gitextractor/tasks/git_repo_collector.go | 8 +- backend/plugins/github/tasks/account_collector.go | 2 +- backend/plugins/github/tasks/account_convertor.go | 2 +- backend/plugins/github/tasks/account_extractor.go | 2 +- .../plugins/github/tasks/account_org_collector.go | 2 +- .../plugins/github/tasks/account_org_extractor.go | 2 +- backend/plugins/github/tasks/cicd_job_collector.go | 2 +- backend/plugins/github/tasks/cicd_job_convertor.go | 2 +- backend/plugins/github/tasks/cicd_job_extractor.go | 2 +- backend/plugins/github/tasks/cicd_run_collector.go | 2 +- backend/plugins/github/tasks/cicd_run_convertor.go | 7 +- backend/plugins/github/tasks/cicd_run_extractor.go | 2 +- backend/plugins/github/tasks/comment_collector.go | 2 +- backend/plugins/github/tasks/comment_extractor.go | 2 +- backend/plugins/github/tasks/commit_collector.go | 2 +- backend/plugins/github/tasks/commit_convertor.go | 2 +- backend/plugins/github/tasks/commit_extractor.go | 2 +- .../plugins/github/tasks/commit_stats_collector.go | 2 +- .../plugins/github/tasks/commit_stats_extractor.go | 2 +- .../plugins/github/tasks/deployment_convertor.go | 2 +- backend/plugins/github/tasks/event_collector.go | 2 +- backend/plugins/github/tasks/event_extractor.go | 2 +- .../github/tasks/issue_assignee_convertor.go | 2 +- backend/plugins/github/tasks/issue_collector.go | 2 +- .../github/tasks/issue_comment_convertor.go | 2 +- backend/plugins/github/tasks/issue_convertor.go | 2 +- backend/plugins/github/tasks/issue_extractor.go | 2 +- .../plugins/github/tasks/issue_label_convertor.go | 2 +- .../plugins/github/tasks/milestone_collector.go | 2 +- .../plugins/github/tasks/milestone_converter.go | 2 +- .../plugins/github/tasks/milestone_extractor.go | 2 +- backend/plugins/github/tasks/pr_collector.go | 2 +- .../plugins/github/tasks/pr_comment_convertor.go | 2 +- .../plugins/github/tasks/pr_commit_collector.go | 2 +- .../plugins/github/tasks/pr_commit_convertor.go | 2 +- .../plugins/github/tasks/pr_commit_extractor.go | 2 +- backend/plugins/github/tasks/pr_convertor.go | 2 +- backend/plugins/github/tasks/pr_extractor.go | 2 +- backend/plugins/github/tasks/pr_issue_convertor.go | 2 +- backend/plugins/github/tasks/pr_issue_enricher.go | 2 +- backend/plugins/github/tasks/pr_label_convertor.go | 2 +- .../plugins/github/tasks/pr_review_collector.go | 2 +- .../github/tasks/pr_review_comment_collector.go | 2 +- .../github/tasks/pr_review_comment_extractor.go | 2 +- .../plugins/github/tasks/pr_review_convertor.go | 2 +- .../plugins/github/tasks/pr_review_extractor.go | 2 +- backend/plugins/github/tasks/repo_convertor.go | 2 +- .../github_graphql/tasks/account_collector.go | 2 +- .../github_graphql/tasks/account_extractor.go | 2 +- .../github_graphql/tasks/deployment_collector.go | 2 +- .../github_graphql/tasks/deployment_extractor.go | 2 +- .../github_graphql/tasks/issue_collector.go | 2 +- .../github_graphql/tasks/issue_extractor.go | 2 +- .../plugins/github_graphql/tasks/job_collector.go | 2 +- .../plugins/github_graphql/tasks/job_extractor.go | 2 +- .../plugins/github_graphql/tasks/pr_collector.go | 2 +- .../plugins/github_graphql/tasks/pr_extractor.go | 2 +- backend/plugins/gitlab/tasks/account_collector.go | 2 +- backend/plugins/gitlab/tasks/account_convertor.go | 2 +- backend/plugins/gitlab/tasks/account_extractor.go | 2 +- backend/plugins/gitlab/tasks/commit_collector.go | 2 +- backend/plugins/gitlab/tasks/commit_convertor.go | 5 +- backend/plugins/gitlab/tasks/commit_extractor.go | 3 +- .../plugins/gitlab/tasks/deployment_collector.go | 2 +- .../plugins/gitlab/tasks/deployment_convertor.go | 7 +- .../plugins/gitlab/tasks/deployment_extractor.go | 2 +- .../gitlab/tasks/issue_assignee_convertor.go | 2 +- backend/plugins/gitlab/tasks/issue_collector.go | 2 +- backend/plugins/gitlab/tasks/issue_convertor.go | 2 +- backend/plugins/gitlab/tasks/issue_extractor.go | 2 +- .../plugins/gitlab/tasks/issue_label_convertor.go | 5 +- backend/plugins/gitlab/tasks/job_collector.go | 2 +- backend/plugins/gitlab/tasks/job_convertor.go | 7 +- backend/plugins/gitlab/tasks/job_extractor.go | 2 +- backend/plugins/gitlab/tasks/mr_collector.go | 2 +- .../plugins/gitlab/tasks/mr_comment_convertor.go | 2 +- .../plugins/gitlab/tasks/mr_commit_collector.go | 2 +- .../plugins/gitlab/tasks/mr_commit_convertor.go | 5 +- .../plugins/gitlab/tasks/mr_commit_extractor.go | 2 +- backend/plugins/gitlab/tasks/mr_convertor.go | 2 +- .../plugins/gitlab/tasks/mr_detail_collector.go | 2 +- .../plugins/gitlab/tasks/mr_detail_extractor.go | 2 +- backend/plugins/gitlab/tasks/mr_enricher.go | 7 +- backend/plugins/gitlab/tasks/mr_extractor.go | 2 +- backend/plugins/gitlab/tasks/mr_label_convertor.go | 5 +- backend/plugins/gitlab/tasks/mr_note_collector.go | 2 +- backend/plugins/gitlab/tasks/mr_note_extractor.go | 2 +- backend/plugins/gitlab/tasks/pipeline_collector.go | 2 +- .../gitlab/tasks/pipeline_commit_convertor.go | 2 +- .../gitlab/tasks/pipeline_detail_collector.go | 2 +- .../gitlab/tasks/pipeline_detail_convertor.go | 2 +- .../gitlab/tasks/pipeline_detail_extractor.go | 2 +- backend/plugins/gitlab/tasks/pipeline_extractor.go | 2 +- backend/plugins/gitlab/tasks/project_convertor.go | 2 +- backend/plugins/gitlab/tasks/tag_collector.go | 2 +- backend/plugins/gitlab/tasks/tag_extractor.go | 3 +- .../plugins/gitlab/tasks/trigger_job_collector.go | 2 +- .../plugins/gitlab/tasks/trigger_job_extractor.go | 5 +- backend/python/pydevlake/pydevlake/subtasks.py | 3 +- backend/server/api/router.go | 1 + backend/server/api/task/task.go | 24 ++++ backend/server/services/task.go | 145 +++++++++++++++++++++ 138 files changed, 526 insertions(+), 172 deletions(-) diff --git a/backend/core/models/migrationscripts/20240322_add_field_to_subtasks.go b/backend/core/models/migrationscripts/20240322_add_field_to_subtasks.go new file mode 100644 index 000000000..0dfb7b6fb --- /dev/null +++ b/backend/core/models/migrationscripts/20240322_add_field_to_subtasks.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*addSubtaskField)(nil) + +type subtask20240322 struct { + FinishedRecords int `json:"finishedRecords"` + Sequence int `json:"sequence"` + IsCollector bool `json:"isCollector"` + IsFailed bool `json:"isFailed"` + Message string `json:"message"` +} + +func (subtask20240322) TableName() string { + return "_devlake_subtasks" +} + +type addSubtaskField struct{} + +func (*addSubtaskField) Up(basicRes context.BasicRes) errors.Error { + return basicRes.GetDal().AutoMigrate(subtask20240322{}) +} + +func (*addSubtaskField) Version() uint64 { + return 20240322111247 +} + +func (*addSubtaskField) Name() string { + return "add some fields to _devlake_subtasks table" +} diff --git a/backend/core/models/migrationscripts/register.go b/backend/core/models/migrationscripts/register.go index 26b3ec9bb..f4641d03b 100644 --- a/backend/core/models/migrationscripts/register.go +++ b/backend/core/models/migrationscripts/register.go @@ -110,5 +110,6 @@ func All() []plugin.MigrationScript { new(addOriginalEnvironmentToCicdDeploymentsAndCicdDeploymentCommits), new(addSubtabknameToDeployment), new(addStore), + new(addSubtaskField), } } diff --git a/backend/core/models/task.go b/backend/core/models/task.go index 6fba0f760..ba6f3f4ca 100644 --- a/backend/core/models/task.go +++ b/backend/core/models/task.go @@ -36,12 +36,14 @@ const ( var PendingTaskStatus = []string{TASK_CREATED, TASK_RERUN, TASK_RUNNING} type TaskProgressDetail struct { - TotalSubTasks int `json:"totalSubTasks"` - FinishedSubTasks int `json:"finishedSubTasks"` - TotalRecords int `json:"totalRecords"` - FinishedRecords int `json:"finishedRecords"` - SubTaskName string `json:"subTaskName"` - SubTaskNumber int `json:"subTaskNumber"` + TotalSubTasks int `json:"totalSubTasks"` + FinishedSubTasks int `json:"finishedSubTasks"` + TotalRecords int `json:"totalRecords"` + FinishedRecords int `json:"finishedRecords"` + SubTaskName string `json:"subTaskName"` + SubTaskNumber int `json:"subTaskNumber"` + CollectSubtaskNumber int `json:"collectSubtaskNumber"` + OtherSubtaskNumber int `json:"otherSubtaskNumber"` } type NewTask struct { @@ -79,14 +81,60 @@ func (Task) TableName() string { type Subtask struct { common.Model - TaskID uint64 `json:"task_id" gorm:"index"` - Name string `json:"name" gorm:"index"` - Number int `json:"number"` - BeganAt *time.Time `json:"beganAt"` - FinishedAt *time.Time `json:"finishedAt" gorm:"index"` - SpentSeconds int64 `json:"spentSeconds"` + TaskID uint64 `json:"task_id" gorm:"index"` + Name string `json:"name" gorm:"index"` + Number int `json:"number"` + BeganAt *time.Time `json:"beganAt"` + FinishedAt *time.Time `json:"finishedAt" gorm:"index"` + SpentSeconds int64 `json:"spentSeconds"` + FinishedRecords int `json:"finishedRecords"` + Sequence int `json:"sequence"` + IsCollector bool `json:"isCollector"` + IsFailed bool `json:"isFailed"` + Message string `json:"message"` } func (Subtask) TableName() string { return "_devlake_subtasks" } + +type SubtaskDetails struct { + ID uint64 `json:"id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + TaskID uint64 `json:"task_id"` + Name string `json:"name"` + Number int `json:"number"` + BeganAt *time.Time `json:"began_at"` + FinishedAt *time.Time `json:"finished_at"` + SpentSeconds int64 `json:"spent_seconds"` + FinishedRecords int `json:"finished_records"` + Sequence int `json:"sequence"` + IsCollector bool `json:"is_collector"` + IsFailed bool `json:"is_failed"` + Message string `json:"message"` +} + +type SubtasksInfo struct { + ID uint64 `json:"id"` + PipelineID uint64 `json:"pipelineId"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + BeganAt *time.Time `json:"beganAt"` + FinishedAt *time.Time `json:"finishedAt"` + Plugin string `json:"plugin"` + Options any `json:"options"` + Status string `json:"status"` + FailedSubTask string `json:"failedSubTask"` + Message string `json:"message"` + ErrorName string `json:"errorName"` + SpentSeconds int `json:"spentSeconds"` + SubtaskDetails []*SubtaskDetails `json:"subtaskDetails"` +} + +type SubTasksOuput struct { + SubtasksInfo []SubtasksInfo `json:"subtasks"` + CompletionRate float64 `json:"completionRate"` + Status string `json:"status"` + Count int64 `json:"count"` +} diff --git a/backend/core/plugin/plugin_task.go b/backend/core/plugin/plugin_task.go index 3bb44b5bb..357fa2dc4 100644 --- a/backend/core/plugin/plugin_task.go +++ b/backend/core/plugin/plugin_task.go @@ -36,11 +36,13 @@ const ( ) type RunningProgress struct { - Type ProgressType - Current int - Total int - SubTaskName string - SubTaskNumber int + Type ProgressType + Current int + Total int + SubTaskName string + SubTaskNumber int + CollectSubtaskNumber int + OtherSubtaskNumber int } // nolint // ExecContext This interface define all resources that needed for task/subtask execution diff --git a/backend/core/runner/run_task.go b/backend/core/runner/run_task.go index 6ee04eccb..0f7428d66 100644 --- a/backend/core/runner/run_task.go +++ b/backend/core/runner/run_task.go @@ -246,6 +246,44 @@ func RunPluginSubTasks( taskCtx.SetSyncPolicy(syncPolicy) taskCtx.SetData(taskData) + // record subtasks sequence to DB + collectSubtaskNumber := 0 + otherSubtaskNumber := 0 + isCollector := false + subtask := []models.Subtask{} + for _, subtaskMeta := range subtaskMetas { + subtaskCtx, err := taskCtx.SubTaskContext(subtaskMeta.Name) + if err != nil { + // sth went wrong + return errors.Default.Wrap(err, fmt.Sprintf("error getting context subtask %s", subtaskMeta.Name)) + } + if subtaskCtx == nil { + // subtask was disabled + continue + } + if strings.Contains(strings.ToLower(subtaskMeta.Name), "collect") || strings.Contains(strings.ToLower(subtaskMeta.Name), "clone git repo") { + collectSubtaskNumber++ + isCollector = true + } else { + otherSubtaskNumber++ + isCollector = false + } + s := models.Subtask{ + Name: subtaskCtx.GetName(), + TaskID: task.ID, + IsCollector: isCollector, + } + if isCollector { + s.Sequence = collectSubtaskNumber + } else { + s.Sequence = otherSubtaskNumber + } + subtask = append(subtask, s) + } + if err := basicRes.GetDal().CreateOrUpdate(subtask); err != nil { + basicRes.GetLogger().Error(err, "error writing subtask list to DB") + } + // execute subtasks in order taskCtx.SetProgress(0, steps) subtaskNumber := 0 @@ -259,7 +297,6 @@ func RunPluginSubTasks( // subtask was disabled continue } - // run subtask logger.Info("executing subtask %s", subtaskMeta.Name) subtaskNumber++ @@ -274,6 +311,13 @@ func RunPluginSubTasks( if err != nil { err = errors.SubtaskErr.Wrap(err, fmt.Sprintf("subtask %s ended unexpectedly", subtaskMeta.Name), errors.WithData(&subtaskMeta)) logger.Error(err, "") + where := dal.Where("task_id = ? and name = ?", task.ID, subtaskCtx.GetName()) + if err := basicRes.GetDal().UpdateColumns(subtask, []dal.DalSet{ + {ColumnName: "is_failed", Value: 1}, + {ColumnName: "message", Value: err.Error()}, + }, where); err != nil { + basicRes.GetLogger().Error(err, "error writing subtask %v status to DB", subtaskCtx.GetName()) + } return err } taskCtx.IncProgress(1) @@ -286,6 +330,7 @@ func RunPluginSubTasks( func UpdateProgressDetail(basicRes context.BasicRes, taskId uint64, progressDetail *models.TaskProgressDetail, p *plugin.RunningProgress) { task := &models.Task{} task.ID = taskId + subtask := &models.Subtask{} switch p.Type { case plugin.TaskSetProgress: progressDetail.TotalSubTasks = p.Total @@ -307,6 +352,14 @@ func UpdateProgressDetail(basicRes context.BasicRes, taskId uint64, progressDeta progressDetail.SubTaskName = p.SubTaskName progressDetail.SubTaskNumber = p.SubTaskNumber } + // update subtask progress + where := dal.Where("task_id = ? and name = ?", taskId, progressDetail.SubTaskName) + err := basicRes.GetDal().UpdateColumns(subtask, []dal.DalSet{ + {ColumnName: "finished_records", Value: progressDetail.FinishedRecords}, + }, where) + if err != nil { + basicRes.GetLogger().Error(err, "failed to update _devlake_subtasks progress") + } } func runSubtask( @@ -323,17 +376,27 @@ func runSubtask( Number: subtaskNumber, BeganAt: &beginAt, } + recordSubtask(basicRes, subtask) + // defer to record subtask status defer func() { finishedAt := time.Now() subtask.FinishedAt = &finishedAt subtask.SpentSeconds = finishedAt.Unix() - beginAt.Unix() + recordSubtask(basicRes, subtask) }() return entryPoint(ctx) } func recordSubtask(basicRes context.BasicRes, subtask *models.Subtask) { - if err := basicRes.GetDal().Create(subtask); err != nil { + where := dal.Where("task_id = ? and name = ?", subtask.TaskID, subtask.Name) + if err := basicRes.GetDal().UpdateColumns(subtask, []dal.DalSet{ + {ColumnName: "began_at", Value: subtask.BeganAt}, + {ColumnName: "finished_at", Value: subtask.FinishedAt}, + {ColumnName: "spent_seconds", Value: subtask.SpentSeconds}, + {ColumnName: "finished_records", Value: subtask.FinishedRecords}, + {ColumnName: "number", Value: subtask.Number}, + }, where); err != nil { basicRes.GetLogger().Error(err, "error writing subtask %d status to DB: %v", subtask.ID) } } diff --git a/backend/helpers/pluginhelper/api/api_collector.go b/backend/helpers/pluginhelper/api/api_collector.go index 990610b20..f79d09148 100644 --- a/backend/helpers/pluginhelper/api/api_collector.go +++ b/backend/helpers/pluginhelper/api/api_collector.go @@ -434,7 +434,7 @@ func (collector *ApiCollector) fetchAsync(reqData *RequestData, handler func(int logger := collector.args.Ctx.GetLogger() logger.Debug("fetchAsync <<< enqueueing for %s %v", apiUrl, apiQuery) responseHandler := func(res *http.Response) errors.Error { - defer logger.Debug("fetchAsync >>> done for %s %v %v", apiUrl, apiQuery, collector.args.RequestBody) + defer logger.Debug("fetchAsync >>> done for %s %v", apiUrl, apiQuery) logger := collector.args.Ctx.GetLogger() // read body to buffer body, err := io.ReadAll(res.Body) diff --git a/backend/plugins/bitbucket/tasks/account_convertor.go b/backend/plugins/bitbucket/tasks/account_convertor.go index cd076b8c7..19f013f0e 100644 --- a/backend/plugins/bitbucket/tasks/account_convertor.go +++ b/backend/plugins/bitbucket/tasks/account_convertor.go @@ -33,7 +33,7 @@ import ( const RAW_ACCOUNT_TABLE = "bitbucket_api_accounts" var ConvertAccountsMeta = plugin.SubTaskMeta{ - Name: "convertAccounts", + Name: "Convert Users", EntryPoint: ConvertAccounts, EnabledByDefault: true, Required: false, diff --git a/backend/plugins/bitbucket/tasks/commit_collector.go b/backend/plugins/bitbucket/tasks/commit_collector.go index d7fd4f0ec..83931db38 100644 --- a/backend/plugins/bitbucket/tasks/commit_collector.go +++ b/backend/plugins/bitbucket/tasks/commit_collector.go @@ -26,7 +26,7 @@ import ( const RAW_COMMIT_TABLE = "bitbucket_api_commits" var CollectApiCommitsMeta = plugin.SubTaskMeta{ - Name: "collectApiCommits", + Name: "Collect Commits", EntryPoint: CollectApiCommits, EnabledByDefault: false, Required: false, diff --git a/backend/plugins/bitbucket/tasks/commit_convertor.go b/backend/plugins/bitbucket/tasks/commit_convertor.go index 98e57a0ee..16b2af11d 100644 --- a/backend/plugins/bitbucket/tasks/commit_convertor.go +++ b/backend/plugins/bitbucket/tasks/commit_convertor.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/code" @@ -25,11 +27,10 @@ import ( plugin "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/bitbucket/models" - "reflect" ) var ConvertCommitsMeta = plugin.SubTaskMeta{ - Name: "convertCommits", + Name: "Convert Commits", EntryPoint: ConvertCommits, EnabledByDefault: false, Required: false, diff --git a/backend/plugins/bitbucket/tasks/commit_extractor.go b/backend/plugins/bitbucket/tasks/commit_extractor.go index 00d2d48fd..0020be9d5 100644 --- a/backend/plugins/bitbucket/tasks/commit_extractor.go +++ b/backend/plugins/bitbucket/tasks/commit_extractor.go @@ -19,15 +19,16 @@ package tasks import ( "encoding/json" + "time" + "github.com/apache/incubator-devlake/core/errors" plugin "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/bitbucket/models" - "time" ) var ExtractApiCommitsMeta = plugin.SubTaskMeta{ - Name: "extractApiCommits", + Name: "Extract Commits", EntryPoint: ExtractApiCommits, EnabledByDefault: false, Required: false, diff --git a/backend/plugins/bitbucket/tasks/deployment_collector.go b/backend/plugins/bitbucket/tasks/deployment_collector.go index bf7f4f4ca..f48526d64 100644 --- a/backend/plugins/bitbucket/tasks/deployment_collector.go +++ b/backend/plugins/bitbucket/tasks/deployment_collector.go @@ -26,7 +26,7 @@ import ( const RAW_DEPLOYMENT_TABLE = "bitbucket_api_deployments" var CollectApiDeploymentsMeta = plugin.SubTaskMeta{ - Name: "collectApiDeployments", + Name: "Collect Deployments", EntryPoint: CollectApiDeployments, EnabledByDefault: true, Description: "Collect deployment data from bitbucket api", diff --git a/backend/plugins/bitbucket/tasks/deployment_convertor.go b/backend/plugins/bitbucket/tasks/deployment_convertor.go index 6c026f705..ca6ea9a4a 100644 --- a/backend/plugins/bitbucket/tasks/deployment_convertor.go +++ b/backend/plugins/bitbucket/tasks/deployment_convertor.go @@ -18,6 +18,10 @@ limitations under the License. package tasks import ( + "reflect" + "strings" + "time" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -26,13 +30,10 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/bitbucket/models" - "reflect" - "strings" - "time" ) var ConvertiDeploymentMeta = plugin.SubTaskMeta{ - Name: "convertDeployments", + Name: "Convert Deployments", EntryPoint: ConvertDeployments, EnabledByDefault: true, Description: "Convert tool layer table bitbucket_deployment into domain layer tables", diff --git a/backend/plugins/bitbucket/tasks/deployment_extractor.go b/backend/plugins/bitbucket/tasks/deployment_extractor.go index a93877378..f1705db5a 100644 --- a/backend/plugins/bitbucket/tasks/deployment_extractor.go +++ b/backend/plugins/bitbucket/tasks/deployment_extractor.go @@ -19,11 +19,12 @@ package tasks import ( "encoding/json" + "time" + "github.com/apache/incubator-devlake/core/errors" plugin "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/bitbucket/models" - "time" ) type bitbucketApiDeploymentsResponse struct { @@ -76,7 +77,7 @@ type bitbucketApiDeploymentsResponse struct { } var ExtractApiDeploymentsMeta = plugin.SubTaskMeta{ - Name: "extractApiDeployments", + Name: "Extract Deployments", EntryPoint: ExtractApiDeployments, EnabledByDefault: true, Description: "Extract raw deployments data into tool layer table BitbucketDeployment", diff --git a/backend/plugins/bitbucket/tasks/issue_collector.go b/backend/plugins/bitbucket/tasks/issue_collector.go index 3edec1bea..19bf67ef0 100644 --- a/backend/plugins/bitbucket/tasks/issue_collector.go +++ b/backend/plugins/bitbucket/tasks/issue_collector.go @@ -26,7 +26,7 @@ import ( const RAW_ISSUE_TABLE = "bitbucket_api_issues" var CollectApiIssuesMeta = plugin.SubTaskMeta{ - Name: "collectApiIssues", + Name: "Collect Issues", EntryPoint: CollectApiIssues, EnabledByDefault: true, Description: "Collect issues data from Bitbucket api", diff --git a/backend/plugins/bitbucket/tasks/issue_comment_collector.go b/backend/plugins/bitbucket/tasks/issue_comment_collector.go index cdc9c2e3d..5b1ad5a9b 100644 --- a/backend/plugins/bitbucket/tasks/issue_comment_collector.go +++ b/backend/plugins/bitbucket/tasks/issue_comment_collector.go @@ -26,7 +26,7 @@ import ( const RAW_ISSUE_COMMENTS_TABLE = "bitbucket_api_issue_comments" var CollectApiIssueCommentsMeta = plugin.SubTaskMeta{ - Name: "collectApiIssueComments", + Name: "Collect Issue Comments", EntryPoint: CollectApiIssueComments, EnabledByDefault: true, Required: false, diff --git a/backend/plugins/bitbucket/tasks/issue_comment_convertor.go b/backend/plugins/bitbucket/tasks/issue_comment_convertor.go index dcae35d10..4787dcae4 100644 --- a/backend/plugins/bitbucket/tasks/issue_comment_convertor.go +++ b/backend/plugins/bitbucket/tasks/issue_comment_convertor.go @@ -31,7 +31,7 @@ import ( ) var ConvertIssueCommentsMeta = plugin.SubTaskMeta{ - Name: "convertIssueComments", + Name: "Convert Issue Comments", EntryPoint: ConvertIssueComments, EnabledByDefault: true, Description: "ConvertIssueComments data from Bitbucket api", diff --git a/backend/plugins/bitbucket/tasks/issue_comment_extractor.go b/backend/plugins/bitbucket/tasks/issue_comment_extractor.go index 3adc0c749..86eaf7620 100644 --- a/backend/plugins/bitbucket/tasks/issue_comment_extractor.go +++ b/backend/plugins/bitbucket/tasks/issue_comment_extractor.go @@ -46,7 +46,7 @@ type BitbucketIssueCommentsResponse struct { } var ExtractApiIssueCommentsMeta = plugin.SubTaskMeta{ - Name: "extractApiIssueComments", + Name: "Extract Issue Comments", EntryPoint: ExtractApiIssueComments, EnabledByDefault: true, Required: false, diff --git a/backend/plugins/bitbucket/tasks/issue_convertor.go b/backend/plugins/bitbucket/tasks/issue_convertor.go index 3c3455fbb..c5fb4f801 100644 --- a/backend/plugins/bitbucket/tasks/issue_convertor.go +++ b/backend/plugins/bitbucket/tasks/issue_convertor.go @@ -32,7 +32,7 @@ import ( ) var ConvertIssuesMeta = plugin.SubTaskMeta{ - Name: "convertIssues", + Name: "Convert Issues", EntryPoint: ConvertIssues, EnabledByDefault: true, Description: "Convert tool layer table bitbucket_issues into domain layer table issues", diff --git a/backend/plugins/bitbucket/tasks/issue_extractor.go b/backend/plugins/bitbucket/tasks/issue_extractor.go index ee0f2c80c..ede3ad730 100644 --- a/backend/plugins/bitbucket/tasks/issue_extractor.go +++ b/backend/plugins/bitbucket/tasks/issue_extractor.go @@ -54,7 +54,7 @@ type IssuesResponse struct { } var ExtractApiIssuesMeta = plugin.SubTaskMeta{ - Name: "extractApiIssues", + Name: "Extract Issues", EntryPoint: ExtractApiIssues, EnabledByDefault: true, Description: "Extract raw Issues data into tool layer table bitbucket_issues", diff --git a/backend/plugins/bitbucket/tasks/pipeline_collector.go b/backend/plugins/bitbucket/tasks/pipeline_collector.go index 8998add3c..c19652e65 100644 --- a/backend/plugins/bitbucket/tasks/pipeline_collector.go +++ b/backend/plugins/bitbucket/tasks/pipeline_collector.go @@ -26,7 +26,7 @@ import ( const RAW_PIPELINE_TABLE = "bitbucket_api_pipelines" var CollectApiPipelinesMeta = plugin.SubTaskMeta{ - Name: "collectApiPipelines", + Name: "Collect Pipelines", EntryPoint: CollectApiPipelines, EnabledByDefault: true, Description: "Collect pipeline data from bitbucket api", diff --git a/backend/plugins/bitbucket/tasks/pipeline_convertor.go b/backend/plugins/bitbucket/tasks/pipeline_convertor.go index 442ae3086..9a8d80bdf 100644 --- a/backend/plugins/bitbucket/tasks/pipeline_convertor.go +++ b/backend/plugins/bitbucket/tasks/pipeline_convertor.go @@ -33,7 +33,7 @@ import ( ) var ConvertPipelineMeta = plugin.SubTaskMeta{ - Name: "convertPipelines", + Name: "Convert Pipelines", EntryPoint: ConvertPipelines, EnabledByDefault: true, Description: "Convert tool layer table bitbucket_pipeline into domain layer table pipeline", diff --git a/backend/plugins/bitbucket/tasks/pipeline_extractor.go b/backend/plugins/bitbucket/tasks/pipeline_extractor.go index 7cd4425f5..dd7e68880 100644 --- a/backend/plugins/bitbucket/tasks/pipeline_extractor.go +++ b/backend/plugins/bitbucket/tasks/pipeline_extractor.go @@ -19,6 +19,7 @@ package tasks import ( "encoding/json" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/models/domainlayer/devops" @@ -70,7 +71,7 @@ type BitbucketApiPipeline struct { } var ExtractApiPipelinesMeta = plugin.SubTaskMeta{ - Name: "extractApiPipelines", + Name: "Extract Pipelines", EntryPoint: ExtractApiPipelines, EnabledByDefault: true, Description: "Extract raw pipelines data into tool layer table BitbucketPipeline", diff --git a/backend/plugins/bitbucket/tasks/pipeline_steps_collector.go b/backend/plugins/bitbucket/tasks/pipeline_steps_collector.go index 8c9d70aa7..9fd99e178 100644 --- a/backend/plugins/bitbucket/tasks/pipeline_steps_collector.go +++ b/backend/plugins/bitbucket/tasks/pipeline_steps_collector.go @@ -28,7 +28,7 @@ const RAW_PIPELINE_STEPS_TABLE = "bitbucket_api_pipeline_steps" var _ plugin.SubTaskEntryPoint = CollectPipelineSteps var CollectPipelineStepsMeta = plugin.SubTaskMeta{ - Name: "CollectPipelineSteps", + Name: "Collect Pipeline Steps", EntryPoint: CollectPipelineSteps, EnabledByDefault: true, Description: "Collect PipelineSteps data from Bitbucket api", diff --git a/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go b/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go index 2276129fa..0a72041ba 100644 --- a/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go +++ b/backend/plugins/bitbucket/tasks/pipeline_steps_convertor.go @@ -32,7 +32,7 @@ import ( ) var ConvertPipelineStepMeta = plugin.SubTaskMeta{ - Name: "convertPipelineSteps", + Name: "Convert Pipeline Steps", EntryPoint: ConvertPipelineSteps, EnabledByDefault: true, Description: "Convert tool layer table bitbucket_pipeline into domain layer table pipeline", diff --git a/backend/plugins/bitbucket/tasks/pipeline_steps_extractor.go b/backend/plugins/bitbucket/tasks/pipeline_steps_extractor.go index 5ad13535f..82bd93bd4 100644 --- a/backend/plugins/bitbucket/tasks/pipeline_steps_extractor.go +++ b/backend/plugins/bitbucket/tasks/pipeline_steps_extractor.go @@ -31,7 +31,7 @@ import ( var _ plugin.SubTaskEntryPoint = ExtractPipelineSteps var ExtractPipelineStepsMeta = plugin.SubTaskMeta{ - Name: "ExtractPipelineSteps", + Name: "Extract Pipeline Steps", EntryPoint: ExtractPipelineSteps, EnabledByDefault: true, Description: "Extract raw data into tool layer table bitbucket_pipeline_steps", diff --git a/backend/plugins/bitbucket/tasks/pr_collector.go b/backend/plugins/bitbucket/tasks/pr_collector.go index 7aa9641e9..9ab2b4eac 100644 --- a/backend/plugins/bitbucket/tasks/pr_collector.go +++ b/backend/plugins/bitbucket/tasks/pr_collector.go @@ -28,7 +28,7 @@ const RAW_PULL_REQUEST_TABLE = "bitbucket_api_pull_requests" // this struct should be moved to `bitbucket_api_common.go` var CollectApiPullRequestsMeta = plugin.SubTaskMeta{ - Name: "collectApiPullRequests", + Name: "Collect Pull Requests", EntryPoint: CollectApiPullRequests, EnabledByDefault: true, Required: false, diff --git a/backend/plugins/bitbucket/tasks/pr_comment_collector.go b/backend/plugins/bitbucket/tasks/pr_comment_collector.go index ee7ccd228..30e856cd1 100644 --- a/backend/plugins/bitbucket/tasks/pr_comment_collector.go +++ b/backend/plugins/bitbucket/tasks/pr_comment_collector.go @@ -26,7 +26,7 @@ import ( const RAW_PULL_REQUEST_COMMENTS_TABLE = "bitbucket_api_pull_request_comments" var CollectApiPrCommentsMeta = plugin.SubTaskMeta{ - Name: "collectApiPullRequestsComments", + Name: "Collect PR Comments", EntryPoint: CollectApiPullRequestsComments, EnabledByDefault: true, Required: false, diff --git a/backend/plugins/bitbucket/tasks/pr_comment_convertor.go b/backend/plugins/bitbucket/tasks/pr_comment_convertor.go index 4c67ba83b..d841dd790 100644 --- a/backend/plugins/bitbucket/tasks/pr_comment_convertor.go +++ b/backend/plugins/bitbucket/tasks/pr_comment_convertor.go @@ -31,7 +31,7 @@ import ( ) var ConvertPrCommentsMeta = plugin.SubTaskMeta{ - Name: "convertPullRequestComments", + Name: "Convert PR Comments", EntryPoint: ConvertPullRequestComments, EnabledByDefault: true, Description: "ConvertPullRequestComments data from Bitbucket api", diff --git a/backend/plugins/bitbucket/tasks/pr_comment_extractor.go b/backend/plugins/bitbucket/tasks/pr_comment_extractor.go index 9b1784c72..f685897bc 100644 --- a/backend/plugins/bitbucket/tasks/pr_comment_extractor.go +++ b/backend/plugins/bitbucket/tasks/pr_comment_extractor.go @@ -28,7 +28,7 @@ import ( ) var ExtractApiPrCommentsMeta = plugin.SubTaskMeta{ - Name: "extractApiPullRequestsComments", + Name: "Extract PR Comments", EntryPoint: ExtractApiPullRequestsComments, EnabledByDefault: true, Required: false, diff --git a/backend/plugins/bitbucket/tasks/pr_commit_collector.go b/backend/plugins/bitbucket/tasks/pr_commit_collector.go index 8ed4a5a11..5260ee630 100644 --- a/backend/plugins/bitbucket/tasks/pr_commit_collector.go +++ b/backend/plugins/bitbucket/tasks/pr_commit_collector.go @@ -29,7 +29,7 @@ import ( const RAW_PULL_REQUEST_COMMITS_TABLE = "bitbucket_api_pull_request_commits" var CollectApiPrCommitsMeta = plugin.SubTaskMeta{ - Name: "collectApiPullRequestCommits", + Name: "Collect PR Commits", EntryPoint: CollectApiPullRequestCommits, EnabledByDefault: true, Description: "Collect PullRequestCommits data from Bitbucket api", diff --git a/backend/plugins/bitbucket/tasks/pr_commit_convertor.go b/backend/plugins/bitbucket/tasks/pr_commit_convertor.go index dd7d28aad..04abeb14e 100644 --- a/backend/plugins/bitbucket/tasks/pr_commit_convertor.go +++ b/backend/plugins/bitbucket/tasks/pr_commit_convertor.go @@ -31,7 +31,7 @@ import ( ) var ConvertPrCommitsMeta = plugin.SubTaskMeta{ - Name: "convertPullRequestCommits", + Name: "Convert PR Commits", EntryPoint: ConvertPullRequestCommits, EnabledByDefault: true, Description: "Convert tool layer table bitbucket_pull_request_commits into domain layer table pull_request_commits", diff --git a/backend/plugins/bitbucket/tasks/pr_commit_extractor.go b/backend/plugins/bitbucket/tasks/pr_commit_extractor.go index c8ca476a3..7862ca505 100644 --- a/backend/plugins/bitbucket/tasks/pr_commit_extractor.go +++ b/backend/plugins/bitbucket/tasks/pr_commit_extractor.go @@ -30,7 +30,7 @@ import ( ) var ExtractApiPrCommitsMeta = plugin.SubTaskMeta{ - Name: "extractApiPullRequestCommits", + Name: "Extract PR Commits", EntryPoint: ExtractApiPullRequestCommits, EnabledByDefault: true, Description: "Extract raw PullRequestCommits data into tool layer table bitbucket_commits", diff --git a/backend/plugins/bitbucket/tasks/pr_convertor.go b/backend/plugins/bitbucket/tasks/pr_convertor.go index b2fc68d56..ed12d0e12 100644 --- a/backend/plugins/bitbucket/tasks/pr_convertor.go +++ b/backend/plugins/bitbucket/tasks/pr_convertor.go @@ -31,7 +31,7 @@ import ( ) var ConvertPullRequestsMeta = plugin.SubTaskMeta{ - Name: "convertPullRequests", + Name: "Convert Pull Requests", EntryPoint: ConvertPullRequests, EnabledByDefault: true, Required: false, diff --git a/backend/plugins/bitbucket/tasks/pr_extractor.go b/backend/plugins/bitbucket/tasks/pr_extractor.go index d88a59d4f..855104c80 100644 --- a/backend/plugins/bitbucket/tasks/pr_extractor.go +++ b/backend/plugins/bitbucket/tasks/pr_extractor.go @@ -29,7 +29,7 @@ import ( ) var ExtractApiPullRequestsMeta = plugin.SubTaskMeta{ - Name: "extractApiPullRequests", + Name: "Extract Pull Requests", EntryPoint: ExtractApiPullRequests, EnabledByDefault: true, Required: false, diff --git a/backend/plugins/bitbucket/tasks/repo_convertor.go b/backend/plugins/bitbucket/tasks/repo_convertor.go index 4961462c3..798d6b252 100644 --- a/backend/plugins/bitbucket/tasks/repo_convertor.go +++ b/backend/plugins/bitbucket/tasks/repo_convertor.go @@ -40,7 +40,7 @@ import ( const RAW_REPOSITORIES_TABLE = "bitbucket_api_repositories" var ConvertRepoMeta = plugin.SubTaskMeta{ - Name: "convertRepo", + Name: "Convert Repos", EntryPoint: ConvertRepo, EnabledByDefault: true, Description: "Convert tool layer table bitbucket_repos into domain layer table repos and boards", diff --git a/backend/plugins/gitextractor/tasks/clone.go b/backend/plugins/gitextractor/tasks/clone.go index 0b88e2ff6..c27921a63 100644 --- a/backend/plugins/gitextractor/tasks/clone.go +++ b/backend/plugins/gitextractor/tasks/clone.go @@ -19,19 +19,20 @@ package tasks import ( "fmt" + "strings" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/log" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/plugins/gitextractor/models" "github.com/apache/incubator-devlake/plugins/gitextractor/parser" "github.com/apache/incubator-devlake/plugins/gitextractor/store" - "strings" ) const useGoGitInGitExtractor = "USE_GO_GIT_IN_GIT_EXTRACTOR" var CloneGitRepoMeta = plugin.SubTaskMeta{ - Name: "cloneGitRepo", + Name: "Clone Git Repo", EntryPoint: CloneGitRepo, EnabledByDefault: true, Required: true, diff --git a/backend/plugins/gitextractor/tasks/git_repo_collector.go b/backend/plugins/gitextractor/tasks/git_repo_collector.go index 5af4fe98a..00abf28cf 100644 --- a/backend/plugins/gitextractor/tasks/git_repo_collector.go +++ b/backend/plugins/gitextractor/tasks/git_repo_collector.go @@ -116,7 +116,7 @@ func getGitRepo(subTaskCtx plugin.SubTaskContext) parser.RepoCollector { } var CollectGitCommitMeta = plugin.SubTaskMeta{ - Name: "collectGitCommits", + Name: "Collect Commits", EntryPoint: CollectGitCommits, EnabledByDefault: true, Description: "collect git commits into Domain Layer Tables", @@ -125,7 +125,7 @@ var CollectGitCommitMeta = plugin.SubTaskMeta{ } var CollectGitBranchMeta = plugin.SubTaskMeta{ - Name: "collectGitBranches", + Name: "Collect Branches", EntryPoint: CollectGitBranches, EnabledByDefault: true, Description: "collect git branch into Domain Layer Tables", @@ -134,7 +134,7 @@ var CollectGitBranchMeta = plugin.SubTaskMeta{ } var CollectGitTagMeta = plugin.SubTaskMeta{ - Name: "collectGitTags", + Name: "Collect Tags", EntryPoint: CollectGitTags, EnabledByDefault: true, Description: "collect git tag into Domain Layer Tables", @@ -143,7 +143,7 @@ var CollectGitTagMeta = plugin.SubTaskMeta{ } var CollectGitDiffLineMeta = plugin.SubTaskMeta{ - Name: "collectDiffLine", + Name: "Collect DiffLine", EntryPoint: CollectGitDiffLines, EnabledByDefault: false, Description: "collect git commit diff line into Domain Layer Tables", diff --git a/backend/plugins/github/tasks/account_collector.go b/backend/plugins/github/tasks/account_collector.go index 931a7ae7c..3f2edf2e5 100644 --- a/backend/plugins/github/tasks/account_collector.go +++ b/backend/plugins/github/tasks/account_collector.go @@ -41,7 +41,7 @@ type SimpleAccount struct { } var CollectAccountsMeta = plugin.SubTaskMeta{ - Name: "collectAccounts", + Name: "Collect Users", EntryPoint: CollectAccounts, EnabledByDefault: true, Description: "Collect accounts data from Github api, does not support either timeFilter or diffSync.", diff --git a/backend/plugins/github/tasks/account_convertor.go b/backend/plugins/github/tasks/account_convertor.go index c93c80407..dfd9a929e 100644 --- a/backend/plugins/github/tasks/account_convertor.go +++ b/backend/plugins/github/tasks/account_convertor.go @@ -37,7 +37,7 @@ func init() { } var ConvertAccountsMeta = plugin.SubTaskMeta{ - Name: "convertAccounts", + Name: "Convert Users", EntryPoint: ConvertAccounts, EnabledByDefault: true, Description: "Convert tool layer table github_accounts into domain layer table accounts", diff --git a/backend/plugins/github/tasks/account_extractor.go b/backend/plugins/github/tasks/account_extractor.go index 7fa2c0f13..88e4e8efb 100644 --- a/backend/plugins/github/tasks/account_extractor.go +++ b/backend/plugins/github/tasks/account_extractor.go @@ -32,7 +32,7 @@ func init() { } var ExtractAccountsMeta = plugin.SubTaskMeta{ - Name: "extractAccounts", + Name: "Extract Users", EntryPoint: ExtractAccounts, EnabledByDefault: true, Description: "Extract raw account data into tool layer table github_accounts", diff --git a/backend/plugins/github/tasks/account_org_collector.go b/backend/plugins/github/tasks/account_org_collector.go index 590474a52..5f9cf5a29 100644 --- a/backend/plugins/github/tasks/account_org_collector.go +++ b/backend/plugins/github/tasks/account_org_collector.go @@ -42,7 +42,7 @@ type SimpleAccountWithId struct { } var CollectAccountOrgMeta = plugin.SubTaskMeta{ - Name: "collectAccountOrg", + Name: "Collect User Org", EntryPoint: CollectAccountOrg, EnabledByDefault: true, Description: "Collect accounts org data from Github api, does not support either timeFilter or diffSync.", diff --git a/backend/plugins/github/tasks/account_org_extractor.go b/backend/plugins/github/tasks/account_org_extractor.go index 60d93ce7f..8aa69b664 100644 --- a/backend/plugins/github/tasks/account_org_extractor.go +++ b/backend/plugins/github/tasks/account_org_extractor.go @@ -31,7 +31,7 @@ func init() { } var ExtractAccountOrgMeta = plugin.SubTaskMeta{ - Name: "extractAccountOrg", + Name: "Extract User Org", EntryPoint: ExtractAccountOrg, EnabledByDefault: true, Description: "Extract raw account org data into tool layer table github_account_orgs", diff --git a/backend/plugins/github/tasks/cicd_job_collector.go b/backend/plugins/github/tasks/cicd_job_collector.go index 1c9fe63e6..dda4b90c0 100644 --- a/backend/plugins/github/tasks/cicd_job_collector.go +++ b/backend/plugins/github/tasks/cicd_job_collector.go @@ -38,7 +38,7 @@ func init() { const RAW_JOB_TABLE = "github_api_jobs" var CollectJobsMeta = plugin.SubTaskMeta{ - Name: "collectJobs", + Name: "Collect Job Runs", EntryPoint: CollectJobs, EnabledByDefault: true, Description: "Collect Jobs data from Github action api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/github/tasks/cicd_job_convertor.go b/backend/plugins/github/tasks/cicd_job_convertor.go index f9134c460..e5e734d4e 100644 --- a/backend/plugins/github/tasks/cicd_job_convertor.go +++ b/backend/plugins/github/tasks/cicd_job_convertor.go @@ -36,7 +36,7 @@ func init() { } var ConvertJobsMeta = plugin.SubTaskMeta{ - Name: "convertJobs", + Name: "Convert Jobs", EntryPoint: ConvertJobs, EnabledByDefault: true, Description: "Convert tool layer table github_jobs into domain layer table cicd_tasks", diff --git a/backend/plugins/github/tasks/cicd_job_extractor.go b/backend/plugins/github/tasks/cicd_job_extractor.go index 9193b0342..5701476bc 100644 --- a/backend/plugins/github/tasks/cicd_job_extractor.go +++ b/backend/plugins/github/tasks/cicd_job_extractor.go @@ -33,7 +33,7 @@ func init() { } var ExtractJobsMeta = plugin.SubTaskMeta{ - Name: "extractJobs", + Name: "Extract Jobs", EntryPoint: ExtractJobs, EnabledByDefault: true, Description: "Extract raw run data into tool layer table github_jobs", diff --git a/backend/plugins/github/tasks/cicd_run_collector.go b/backend/plugins/github/tasks/cicd_run_collector.go index 3cd1bd5d5..85ef6108b 100644 --- a/backend/plugins/github/tasks/cicd_run_collector.go +++ b/backend/plugins/github/tasks/cicd_run_collector.go @@ -47,7 +47,7 @@ type SimpleGithubApiJob struct { } var CollectRunsMeta = plugin.SubTaskMeta{ - Name: "collectRuns", + Name: "Collect Workflow Runs", EntryPoint: CollectRuns, EnabledByDefault: true, Description: "Collect Runs data from Github action api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/github/tasks/cicd_run_convertor.go b/backend/plugins/github/tasks/cicd_run_convertor.go index b28432369..505d85fce 100644 --- a/backend/plugins/github/tasks/cicd_run_convertor.go +++ b/backend/plugins/github/tasks/cicd_run_convertor.go @@ -18,6 +18,9 @@ limitations under the License. package tasks import ( + "reflect" + "time" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -26,8 +29,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" - "reflect" - "time" ) func init() { @@ -35,7 +36,7 @@ func init() { } var ConvertRunsMeta = plugin.SubTaskMeta{ - Name: "convertRuns", + Name: "Convert Workflow Runs", EntryPoint: ConvertRuns, EnabledByDefault: true, Description: "Convert tool layer table github_runs into domain layer table cicd_pipeline", diff --git a/backend/plugins/github/tasks/cicd_run_extractor.go b/backend/plugins/github/tasks/cicd_run_extractor.go index bc052299a..1e6a3f41a 100644 --- a/backend/plugins/github/tasks/cicd_run_extractor.go +++ b/backend/plugins/github/tasks/cicd_run_extractor.go @@ -32,7 +32,7 @@ func init() { } var ExtractRunsMeta = plugin.SubTaskMeta{ - Name: "extractRuns", + Name: "Extract Workflow Runs", EntryPoint: ExtractRuns, EnabledByDefault: true, Description: "Extract raw run data into tool layer table github_runs", diff --git a/backend/plugins/github/tasks/comment_collector.go b/backend/plugins/github/tasks/comment_collector.go index 73dfad834..e06f023f4 100644 --- a/backend/plugins/github/tasks/comment_collector.go +++ b/backend/plugins/github/tasks/comment_collector.go @@ -35,7 +35,7 @@ func init() { const RAW_COMMENTS_TABLE = "github_api_comments" var CollectApiCommentsMeta = plugin.SubTaskMeta{ - Name: "collectApiComments", + Name: "Collect Comments", EntryPoint: CollectApiComments, EnabledByDefault: true, Description: "Collect comments data from Github api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/github/tasks/comment_extractor.go b/backend/plugins/github/tasks/comment_extractor.go index 4846c1d40..386babe79 100644 --- a/backend/plugins/github/tasks/comment_extractor.go +++ b/backend/plugins/github/tasks/comment_extractor.go @@ -34,7 +34,7 @@ func init() { } var ExtractApiCommentsMeta = plugin.SubTaskMeta{ - Name: "extractApiComments", + Name: "Extract Comments", EntryPoint: ExtractApiComments, EnabledByDefault: true, Description: "Extract raw comment data into tool layer table github_pull_request_comments" + diff --git a/backend/plugins/github/tasks/commit_collector.go b/backend/plugins/github/tasks/commit_collector.go index 5a9be44db..b735a508d 100644 --- a/backend/plugins/github/tasks/commit_collector.go +++ b/backend/plugins/github/tasks/commit_collector.go @@ -35,7 +35,7 @@ func init() { const RAW_COMMIT_TABLE = "github_api_commits" var CollectApiCommitsMeta = plugin.SubTaskMeta{ - Name: "collectApiCommits", + Name: "Collect Commits", EntryPoint: CollectApiCommits, EnabledByDefault: false, Description: "Collect commits data from Github api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/github/tasks/commit_convertor.go b/backend/plugins/github/tasks/commit_convertor.go index e8dcfa5e9..4dc3fa43c 100644 --- a/backend/plugins/github/tasks/commit_convertor.go +++ b/backend/plugins/github/tasks/commit_convertor.go @@ -34,7 +34,7 @@ func init() { } var ConvertCommitsMeta = plugin.SubTaskMeta{ - Name: "convertCommits", + Name: "Convert Commits", EntryPoint: ConvertCommits, EnabledByDefault: false, Description: "Convert tool layer table github_commits into domain layer table commits", diff --git a/backend/plugins/github/tasks/commit_extractor.go b/backend/plugins/github/tasks/commit_extractor.go index a81c2b5f9..a0ed8d11d 100644 --- a/backend/plugins/github/tasks/commit_extractor.go +++ b/backend/plugins/github/tasks/commit_extractor.go @@ -32,7 +32,7 @@ func init() { } var ExtractApiCommitsMeta = plugin.SubTaskMeta{ - Name: "extractApiCommits", + Name: "Extract Commits", EntryPoint: ExtractApiCommits, EnabledByDefault: false, Description: "Extract raw commit data into tool layer table github_commits", diff --git a/backend/plugins/github/tasks/commit_stats_collector.go b/backend/plugins/github/tasks/commit_stats_collector.go index 3aa71635f..0514a3096 100644 --- a/backend/plugins/github/tasks/commit_stats_collector.go +++ b/backend/plugins/github/tasks/commit_stats_collector.go @@ -39,7 +39,7 @@ func init() { const RAW_COMMIT_STATS_TABLE = "github_api_commit_stats" var CollectApiCommitStatsMeta = plugin.SubTaskMeta{ - Name: "collectApiCommitStats", + Name: "Collect Commit Stats", EntryPoint: CollectApiCommitStats, EnabledByDefault: false, Description: "Collect commitStats data from Github api, does not support either timeFilter or diffSync.", diff --git a/backend/plugins/github/tasks/commit_stats_extractor.go b/backend/plugins/github/tasks/commit_stats_extractor.go index 696305262..5f99b1ed5 100644 --- a/backend/plugins/github/tasks/commit_stats_extractor.go +++ b/backend/plugins/github/tasks/commit_stats_extractor.go @@ -33,7 +33,7 @@ func init() { } var ExtractApiCommitStatsMeta = plugin.SubTaskMeta{ - Name: "extractApiCommitStats", + Name: "Extract Commit Stats", EntryPoint: ExtractApiCommitStats, EnabledByDefault: false, Description: "Extract raw commit stats data into tool layer table github_commit_stats", diff --git a/backend/plugins/github/tasks/deployment_convertor.go b/backend/plugins/github/tasks/deployment_convertor.go index 69f17390d..0d9f89dc6 100644 --- a/backend/plugins/github/tasks/deployment_convertor.go +++ b/backend/plugins/github/tasks/deployment_convertor.go @@ -39,7 +39,7 @@ const ( ) var ConvertDeploymentsMeta = plugin.SubTaskMeta{ - Name: "ConvertDeployments", + Name: "Convert Deployments", EntryPoint: ConvertDeployment, EnabledByDefault: true, Description: "Convert tool layer table github_deployments into domain layer table deployment", diff --git a/backend/plugins/github/tasks/event_collector.go b/backend/plugins/github/tasks/event_collector.go index 491cc021c..ea192ee71 100644 --- a/backend/plugins/github/tasks/event_collector.go +++ b/backend/plugins/github/tasks/event_collector.go @@ -46,7 +46,7 @@ type SimpleGithubApiEvents struct { } var CollectApiEventsMeta = plugin.SubTaskMeta{ - Name: "collectApiEvents", + Name: "Collect Events", EntryPoint: CollectApiEvents, EnabledByDefault: true, Description: "Collect Events data from Github api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/github/tasks/event_extractor.go b/backend/plugins/github/tasks/event_extractor.go index 9e09c9920..133527d16 100644 --- a/backend/plugins/github/tasks/event_extractor.go +++ b/backend/plugins/github/tasks/event_extractor.go @@ -32,7 +32,7 @@ func init() { } var ExtractApiEventsMeta = plugin.SubTaskMeta{ - Name: "extractApiEvents", + Name: "Extract Events", EntryPoint: ExtractApiEvents, EnabledByDefault: true, Description: "Extract raw Events data into tool layer table github_issue_events", diff --git a/backend/plugins/github/tasks/issue_assignee_convertor.go b/backend/plugins/github/tasks/issue_assignee_convertor.go index 24c3c0f16..5f8c883b8 100644 --- a/backend/plugins/github/tasks/issue_assignee_convertor.go +++ b/backend/plugins/github/tasks/issue_assignee_convertor.go @@ -34,7 +34,7 @@ func init() { } var ConvertIssueAssigneeMeta = plugin.SubTaskMeta{ - Name: "convertIssueAssignee", + Name: "Convert Issue Assignees", EntryPoint: ConvertIssueAssignee, EnabledByDefault: true, Description: "Convert tool layer table _tool_github_issue_assignees into domain layer table issue_assignees", diff --git a/backend/plugins/github/tasks/issue_collector.go b/backend/plugins/github/tasks/issue_collector.go index 67d2ce506..4e9b25d21 100644 --- a/backend/plugins/github/tasks/issue_collector.go +++ b/backend/plugins/github/tasks/issue_collector.go @@ -35,7 +35,7 @@ func init() { } var CollectApiIssuesMeta = plugin.SubTaskMeta{ - Name: "collectApiIssues", + Name: "Collect Issues", EntryPoint: CollectApiIssues, EnabledByDefault: true, Description: "Collect issues data from Github api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/github/tasks/issue_comment_convertor.go b/backend/plugins/github/tasks/issue_comment_convertor.go index 860a5fc14..07ddeec65 100644 --- a/backend/plugins/github/tasks/issue_comment_convertor.go +++ b/backend/plugins/github/tasks/issue_comment_convertor.go @@ -35,7 +35,7 @@ func init() { } var ConvertIssueCommentsMeta = plugin.SubTaskMeta{ - Name: "convertIssueComments", + Name: "Convert Issue Comments", EntryPoint: ConvertIssueComments, EnabledByDefault: true, Description: "ConvertIssueComments data from Github api", diff --git a/backend/plugins/github/tasks/issue_convertor.go b/backend/plugins/github/tasks/issue_convertor.go index a1c5f9188..e50f6f22a 100644 --- a/backend/plugins/github/tasks/issue_convertor.go +++ b/backend/plugins/github/tasks/issue_convertor.go @@ -37,7 +37,7 @@ func init() { } var ConvertIssuesMeta = plugin.SubTaskMeta{ - Name: "convertIssues", + Name: "Convert Issues", EntryPoint: ConvertIssues, EnabledByDefault: true, Description: "Convert tool layer table github_issues into domain layer table issues", diff --git a/backend/plugins/github/tasks/issue_extractor.go b/backend/plugins/github/tasks/issue_extractor.go index a61c33541..101b1e5b9 100644 --- a/backend/plugins/github/tasks/issue_extractor.go +++ b/backend/plugins/github/tasks/issue_extractor.go @@ -35,7 +35,7 @@ func init() { } var ExtractApiIssuesMeta = plugin.SubTaskMeta{ - Name: "extractApiIssues", + Name: "Extract Issues", EntryPoint: ExtractApiIssues, EnabledByDefault: true, Description: "Extract raw Issues data into tool layer table github_issues", diff --git a/backend/plugins/github/tasks/issue_label_convertor.go b/backend/plugins/github/tasks/issue_label_convertor.go index 8643d48cf..bff0c0143 100644 --- a/backend/plugins/github/tasks/issue_label_convertor.go +++ b/backend/plugins/github/tasks/issue_label_convertor.go @@ -34,7 +34,7 @@ func init() { } var ConvertIssueLabelsMeta = plugin.SubTaskMeta{ - Name: "convertIssueLabels", + Name: "Convert Issue Labels", EntryPoint: ConvertIssueLabels, EnabledByDefault: true, Description: "Convert tool layer table github_issue_labels into domain layer table issue_labels", diff --git a/backend/plugins/github/tasks/milestone_collector.go b/backend/plugins/github/tasks/milestone_collector.go index 03d4ff586..9e60431ce 100644 --- a/backend/plugins/github/tasks/milestone_collector.go +++ b/backend/plugins/github/tasks/milestone_collector.go @@ -35,7 +35,7 @@ func init() { const RAW_MILESTONE_TABLE = "github_milestones" var CollectMilestonesMeta = plugin.SubTaskMeta{ - Name: "collectApiMilestones", + Name: "Collect Milestones", EntryPoint: CollectApiMilestones, EnabledByDefault: true, Description: "Collect milestone data from Github api, does not support either timeFilter or diffSync.", diff --git a/backend/plugins/github/tasks/milestone_converter.go b/backend/plugins/github/tasks/milestone_converter.go index 6d872023d..5261c7674 100644 --- a/backend/plugins/github/tasks/milestone_converter.go +++ b/backend/plugins/github/tasks/milestone_converter.go @@ -36,7 +36,7 @@ func init() { } var ConvertMilestonesMeta = plugin.SubTaskMeta{ - Name: "convertMilestones", + Name: "Cnvert Milestones", EntryPoint: ConvertMilestones, EnabledByDefault: true, Description: "Convert tool layer table github_milestones into domain layer table milestones", diff --git a/backend/plugins/github/tasks/milestone_extractor.go b/backend/plugins/github/tasks/milestone_extractor.go index 83e10b5cd..7828b2c26 100644 --- a/backend/plugins/github/tasks/milestone_extractor.go +++ b/backend/plugins/github/tasks/milestone_extractor.go @@ -32,7 +32,7 @@ func init() { } var ExtractMilestonesMeta = plugin.SubTaskMeta{ - Name: "extractMilestones", + Name: "Extract Milestones", EntryPoint: ExtractMilestones, EnabledByDefault: true, Description: "Extract raw milestone data into tool layer table github_milestones", diff --git a/backend/plugins/github/tasks/pr_collector.go b/backend/plugins/github/tasks/pr_collector.go index a83cd3085..cc6d31e3a 100644 --- a/backend/plugins/github/tasks/pr_collector.go +++ b/backend/plugins/github/tasks/pr_collector.go @@ -40,7 +40,7 @@ func init() { const RAW_PULL_REQUEST_TABLE = "github_api_pull_requests" var CollectApiPullRequestsMeta = plugin.SubTaskMeta{ - Name: "collectApiPullRequests", + Name: "Collect Pull Requests", EntryPoint: CollectApiPullRequests, EnabledByDefault: true, Description: "Collect PullRequests data from Github api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/github/tasks/pr_comment_convertor.go b/backend/plugins/github/tasks/pr_comment_convertor.go index 7df730c92..43e0e73ba 100644 --- a/backend/plugins/github/tasks/pr_comment_convertor.go +++ b/backend/plugins/github/tasks/pr_comment_convertor.go @@ -35,7 +35,7 @@ func init() { } var ConvertPullRequestCommentsMeta = plugin.SubTaskMeta{ - Name: "convertPullRequestComments", + Name: "Convert PR Comments", EntryPoint: ConvertPullRequestComments, EnabledByDefault: true, Description: "ConvertPullRequestComments data from Github api", diff --git a/backend/plugins/github/tasks/pr_commit_collector.go b/backend/plugins/github/tasks/pr_commit_collector.go index 25c5402ac..487e6ac14 100644 --- a/backend/plugins/github/tasks/pr_commit_collector.go +++ b/backend/plugins/github/tasks/pr_commit_collector.go @@ -43,7 +43,7 @@ const RAW_PR_COMMIT_TABLE = "github_api_pull_request_commits" // this struct should be moved to `gitub_api_common.go` var CollectApiPullRequestCommitsMeta = plugin.SubTaskMeta{ - Name: "collectApiPullRequestCommits", + Name: "Collect PR Commits", EntryPoint: CollectApiPullRequestCommits, EnabledByDefault: true, Description: "Collect PullRequestCommits data from Github api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/github/tasks/pr_commit_convertor.go b/backend/plugins/github/tasks/pr_commit_convertor.go index 5b6ab7042..b9cbbbbf5 100644 --- a/backend/plugins/github/tasks/pr_commit_convertor.go +++ b/backend/plugins/github/tasks/pr_commit_convertor.go @@ -34,7 +34,7 @@ func init() { } var ConvertPullRequestCommitsMeta = plugin.SubTaskMeta{ - Name: "convertPullRequestCommits", + Name: "Convert PR Commits", EntryPoint: ConvertPullRequestCommits, EnabledByDefault: true, Description: "Convert tool layer table github_pull_request_commits into domain layer table pull_request_commits", diff --git a/backend/plugins/github/tasks/pr_commit_extractor.go b/backend/plugins/github/tasks/pr_commit_extractor.go index d357e7a94..569e39aba 100644 --- a/backend/plugins/github/tasks/pr_commit_extractor.go +++ b/backend/plugins/github/tasks/pr_commit_extractor.go @@ -34,7 +34,7 @@ func init() { } var ExtractApiPullRequestCommitsMeta = plugin.SubTaskMeta{ - Name: "extractApiPullRequestCommits", + Name: "Extract PR Commits", EntryPoint: ExtractApiPullRequestCommits, EnabledByDefault: true, Description: "Extract raw PullRequestCommits data into tool layer table github_commits", diff --git a/backend/plugins/github/tasks/pr_convertor.go b/backend/plugins/github/tasks/pr_convertor.go index 672484de0..6f7250127 100644 --- a/backend/plugins/github/tasks/pr_convertor.go +++ b/backend/plugins/github/tasks/pr_convertor.go @@ -35,7 +35,7 @@ func init() { } var ConvertPullRequestsMeta = plugin.SubTaskMeta{ - Name: "convertPullRequests", + Name: "Convert Pull Requests", EntryPoint: ConvertPullRequests, EnabledByDefault: true, Description: "ConvertPullRequests data from Github api", diff --git a/backend/plugins/github/tasks/pr_extractor.go b/backend/plugins/github/tasks/pr_extractor.go index 17fec4a45..6551dedbb 100644 --- a/backend/plugins/github/tasks/pr_extractor.go +++ b/backend/plugins/github/tasks/pr_extractor.go @@ -33,7 +33,7 @@ func init() { } var ExtractApiPullRequestsMeta = plugin.SubTaskMeta{ - Name: "extractApiPullRequests", + Name: "Extract Pull Requests", EntryPoint: ExtractApiPullRequests, EnabledByDefault: true, Description: "Extract raw PullRequests data into tool layer table github_pull_requests", diff --git a/backend/plugins/github/tasks/pr_issue_convertor.go b/backend/plugins/github/tasks/pr_issue_convertor.go index 1c5906847..a0cafb1ff 100644 --- a/backend/plugins/github/tasks/pr_issue_convertor.go +++ b/backend/plugins/github/tasks/pr_issue_convertor.go @@ -34,7 +34,7 @@ func init() { } var ConvertPullRequestIssuesMeta = plugin.SubTaskMeta{ - Name: "convertPullRequestIssues", + Name: "Convert PR Issues", EntryPoint: ConvertPullRequestIssues, EnabledByDefault: true, Description: "Convert tool layer table github_pull_request_issues into domain layer table pull_request_issues", diff --git a/backend/plugins/github/tasks/pr_issue_enricher.go b/backend/plugins/github/tasks/pr_issue_enricher.go index f2b8c13fe..bf4d2837d 100644 --- a/backend/plugins/github/tasks/pr_issue_enricher.go +++ b/backend/plugins/github/tasks/pr_issue_enricher.go @@ -35,7 +35,7 @@ func init() { } var EnrichPullRequestIssuesMeta = plugin.SubTaskMeta{ - Name: "enrichPullRequestIssues", + Name: "Enrich PR Issues", EntryPoint: EnrichPullRequestIssues, EnabledByDefault: true, Description: "Create tool layer table github_pull_request_issues from github_pull_requests", diff --git a/backend/plugins/github/tasks/pr_label_convertor.go b/backend/plugins/github/tasks/pr_label_convertor.go index 9053dcd98..3351218a7 100644 --- a/backend/plugins/github/tasks/pr_label_convertor.go +++ b/backend/plugins/github/tasks/pr_label_convertor.go @@ -34,7 +34,7 @@ func init() { } var ConvertPullRequestLabelsMeta = plugin.SubTaskMeta{ - Name: "convertPullRequestLabels", + Name: "Convert PR Labels", EntryPoint: ConvertPullRequestLabels, EnabledByDefault: true, Description: "Convert tool layer table github_pull_request_labels into domain layer table pull_request_labels", diff --git a/backend/plugins/github/tasks/pr_review_collector.go b/backend/plugins/github/tasks/pr_review_collector.go index a0f6576ef..60430c583 100644 --- a/backend/plugins/github/tasks/pr_review_collector.go +++ b/backend/plugins/github/tasks/pr_review_collector.go @@ -40,7 +40,7 @@ const RAW_PR_REVIEW_TABLE = "github_api_pull_request_reviews" // this struct should be moved to `gitub_api_common.go` var CollectApiPullRequestReviewsMeta = plugin.SubTaskMeta{ - Name: "collectApiPullRequestReviews", + Name: "Collect PR Reviews", EntryPoint: CollectApiPullRequestReviews, EnabledByDefault: true, Description: "Collect PullRequestReviews data from Github api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/github/tasks/pr_review_comment_collector.go b/backend/plugins/github/tasks/pr_review_comment_collector.go index e05ae2812..d53371444 100644 --- a/backend/plugins/github/tasks/pr_review_comment_collector.go +++ b/backend/plugins/github/tasks/pr_review_comment_collector.go @@ -37,7 +37,7 @@ const RAW_PR_REVIEW_COMMENTS_TABLE = "github_api_pull_request_review_comments" // this struct should be moved to `github_api_common.go` var CollectApiPrReviewCommentsMeta = plugin.SubTaskMeta{ - Name: "collectApiPrReviewCommentsMeta", + Name: "Collect PR Review Comments", EntryPoint: CollectPrReviewComments, EnabledByDefault: true, Description: "Collect pr review comments data from Github api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/github/tasks/pr_review_comment_extractor.go b/backend/plugins/github/tasks/pr_review_comment_extractor.go index 8daeb8920..7d93f2344 100644 --- a/backend/plugins/github/tasks/pr_review_comment_extractor.go +++ b/backend/plugins/github/tasks/pr_review_comment_extractor.go @@ -36,7 +36,7 @@ func init() { } var ExtractApiPrReviewCommentsMeta = plugin.SubTaskMeta{ - Name: "extractApiPrReviewComments", + Name: "Extract PR Review Comments", EntryPoint: ExtractApiPrReviewComments, EnabledByDefault: true, Description: "Extract raw comment data into tool layer table github_pull_request_comments" + diff --git a/backend/plugins/github/tasks/pr_review_convertor.go b/backend/plugins/github/tasks/pr_review_convertor.go index 280f204a6..4ff2f3531 100644 --- a/backend/plugins/github/tasks/pr_review_convertor.go +++ b/backend/plugins/github/tasks/pr_review_convertor.go @@ -35,7 +35,7 @@ func init() { } var ConvertPullRequestReviewsMeta = plugin.SubTaskMeta{ - Name: "convertPullRequestReviews", + Name: "Convert PR Reviews", EntryPoint: ConvertPullRequestReviews, EnabledByDefault: true, Description: "ConvertPullRequestReviews data from Github api", diff --git a/backend/plugins/github/tasks/pr_review_extractor.go b/backend/plugins/github/tasks/pr_review_extractor.go index 05d11f92d..c14a179e8 100644 --- a/backend/plugins/github/tasks/pr_review_extractor.go +++ b/backend/plugins/github/tasks/pr_review_extractor.go @@ -33,7 +33,7 @@ func init() { } var ExtractApiPullRequestReviewsMeta = plugin.SubTaskMeta{ - Name: "extractApiPullRequestReviews", + Name: "Extract PR Reviews", EntryPoint: ExtractApiPullRequestReviews, EnabledByDefault: true, Description: "Extract raw PullRequestReviewers data into tool layer table github_reviewers", diff --git a/backend/plugins/github/tasks/repo_convertor.go b/backend/plugins/github/tasks/repo_convertor.go index 1866d2b7b..b9c711ad9 100644 --- a/backend/plugins/github/tasks/repo_convertor.go +++ b/backend/plugins/github/tasks/repo_convertor.go @@ -54,7 +54,7 @@ type GithubApiRepo struct { } var ConvertRepoMeta = plugin.SubTaskMeta{ - Name: "convertRepo", + Name: "Convert Repos", EntryPoint: ConvertRepo, EnabledByDefault: true, Description: "Convert tool layer table github_repos into domain layer table repos and boards", diff --git a/backend/plugins/github_graphql/tasks/account_collector.go b/backend/plugins/github_graphql/tasks/account_collector.go index 3aed8c6a9..ad16068c4 100644 --- a/backend/plugins/github_graphql/tasks/account_collector.go +++ b/backend/plugins/github_graphql/tasks/account_collector.go @@ -58,7 +58,7 @@ type GraphqlQueryAccount struct { } var CollectAccountMeta = plugin.SubTaskMeta{ - Name: "CollectAccount", + Name: "Collect Users", EntryPoint: CollectAccount, EnabledByDefault: true, Description: "Collect Account data from GithubGraphql api, does not support either timeFilter or diffSync.", diff --git a/backend/plugins/github_graphql/tasks/account_extractor.go b/backend/plugins/github_graphql/tasks/account_extractor.go index a4fff15ba..deb5877a9 100644 --- a/backend/plugins/github_graphql/tasks/account_extractor.go +++ b/backend/plugins/github_graphql/tasks/account_extractor.go @@ -30,7 +30,7 @@ import ( var _ plugin.SubTaskEntryPoint = ExtractAccounts var ExtractAccountsMeta = plugin.SubTaskMeta{ - Name: "extractAccounts", + Name: "Extract Users", EntryPoint: ExtractAccounts, EnabledByDefault: true, Description: "extract raw account data into tool layer table github_accounts", diff --git a/backend/plugins/github_graphql/tasks/deployment_collector.go b/backend/plugins/github_graphql/tasks/deployment_collector.go index 6de272f4f..437088629 100644 --- a/backend/plugins/github_graphql/tasks/deployment_collector.go +++ b/backend/plugins/github_graphql/tasks/deployment_collector.go @@ -35,7 +35,7 @@ const ( ) var CollectDeploymentsMeta = plugin.SubTaskMeta{ - Name: "CollectDeployments", + Name: "Collect Deployments", EntryPoint: CollectDeployments, EnabledByDefault: true, Description: "collect github deployments to raw and tool layer from GithubGraphql api", diff --git a/backend/plugins/github_graphql/tasks/deployment_extractor.go b/backend/plugins/github_graphql/tasks/deployment_extractor.go index 408e70ca4..b2150effd 100644 --- a/backend/plugins/github_graphql/tasks/deployment_extractor.go +++ b/backend/plugins/github_graphql/tasks/deployment_extractor.go @@ -31,7 +31,7 @@ import ( var _ plugin.SubTaskEntryPoint = ExtractDeployments var ExtractDeploymentsMeta = plugin.SubTaskMeta{ - Name: "extractDeployments", + Name: "Extract Deployments", EntryPoint: ExtractDeployments, EnabledByDefault: true, Description: "extract raw deployment data into tool layer table github_graphql_deployment", diff --git a/backend/plugins/github_graphql/tasks/issue_collector.go b/backend/plugins/github_graphql/tasks/issue_collector.go index 5bd27267f..313b7cbac 100644 --- a/backend/plugins/github_graphql/tasks/issue_collector.go +++ b/backend/plugins/github_graphql/tasks/issue_collector.go @@ -71,7 +71,7 @@ type GraphqlQueryIssue struct { } var CollectIssuesMeta = plugin.SubTaskMeta{ - Name: "CollectIssues", + Name: "Collect Issues", EntryPoint: CollectIssues, EnabledByDefault: true, Description: "Collect Issue data from GithubGraphql api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/github_graphql/tasks/issue_extractor.go b/backend/plugins/github_graphql/tasks/issue_extractor.go index 9bc318589..b5652587e 100644 --- a/backend/plugins/github_graphql/tasks/issue_extractor.go +++ b/backend/plugins/github_graphql/tasks/issue_extractor.go @@ -33,7 +33,7 @@ import ( var _ plugin.SubTaskEntryPoint = ExtractAccounts var ExtractIssuesMeta = plugin.SubTaskMeta{ - Name: "extractIssues", + Name: "Extract Issues", EntryPoint: ExtractIssues, EnabledByDefault: true, Description: "Extract raw Issues data into tool layer table github_issues", diff --git a/backend/plugins/github_graphql/tasks/job_collector.go b/backend/plugins/github_graphql/tasks/job_collector.go index 650087de3..48ec17045 100644 --- a/backend/plugins/github_graphql/tasks/job_collector.go +++ b/backend/plugins/github_graphql/tasks/job_collector.go @@ -87,7 +87,7 @@ type SimpleWorkflowRun struct { } var CollectJobsMeta = plugin.SubTaskMeta{ - Name: "CollectJobs", + Name: "Collect Job Runs", EntryPoint: CollectJobs, EnabledByDefault: true, Description: "Collect Jobs(CheckRun) data from GithubGraphql api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/github_graphql/tasks/job_extractor.go b/backend/plugins/github_graphql/tasks/job_extractor.go index f5ce0dcec..4e38cc727 100644 --- a/backend/plugins/github_graphql/tasks/job_extractor.go +++ b/backend/plugins/github_graphql/tasks/job_extractor.go @@ -32,7 +32,7 @@ import ( var _ plugin.SubTaskEntryPoint = ExtractAccounts var ExtractJobsMeta = plugin.SubTaskMeta{ - Name: "extractJobs", + Name: "Extract Jobs", EntryPoint: ExtractJobs, EnabledByDefault: true, Description: "Extract raw run data into tool layer table github_jobs", diff --git a/backend/plugins/github_graphql/tasks/pr_collector.go b/backend/plugins/github_graphql/tasks/pr_collector.go index 66e007a8b..2bc887305 100644 --- a/backend/plugins/github_graphql/tasks/pr_collector.go +++ b/backend/plugins/github_graphql/tasks/pr_collector.go @@ -118,7 +118,7 @@ type GraphqlQueryCommit struct { } var CollectPrsMeta = plugin.SubTaskMeta{ - Name: "CollectPrs", + Name: "Collect Pull Requests", EntryPoint: CollectPrs, EnabledByDefault: true, Description: "Collect Pr data from GithubGraphql api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/github_graphql/tasks/pr_extractor.go b/backend/plugins/github_graphql/tasks/pr_extractor.go index fd90d1f99..c82975fc8 100644 --- a/backend/plugins/github_graphql/tasks/pr_extractor.go +++ b/backend/plugins/github_graphql/tasks/pr_extractor.go @@ -31,7 +31,7 @@ import ( var _ plugin.SubTaskEntryPoint = ExtractPrs var ExtractPrsMeta = plugin.SubTaskMeta{ - Name: "extractPrs", + Name: "Extract Pull Requests", EntryPoint: ExtractPrs, EnabledByDefault: true, Description: "Extract raw PullRequests data into tool layer table github_pull_requests", diff --git a/backend/plugins/gitlab/tasks/account_collector.go b/backend/plugins/gitlab/tasks/account_collector.go index 4d234056c..52ea1a72d 100644 --- a/backend/plugins/gitlab/tasks/account_collector.go +++ b/backend/plugins/gitlab/tasks/account_collector.go @@ -38,7 +38,7 @@ func init() { const RAW_USER_TABLE = "gitlab_api_users" var CollectAccountsMeta = plugin.SubTaskMeta{ - Name: "collectAccounts", + Name: "Collect Users", EntryPoint: CollectAccounts, EnabledByDefault: true, Description: "collect gitlab users, does not support either timeFilter or diffSync.", diff --git a/backend/plugins/gitlab/tasks/account_convertor.go b/backend/plugins/gitlab/tasks/account_convertor.go index 9db00334c..c6bac24a1 100644 --- a/backend/plugins/gitlab/tasks/account_convertor.go +++ b/backend/plugins/gitlab/tasks/account_convertor.go @@ -35,7 +35,7 @@ func init() { } var ConvertAccountsMeta = plugin.SubTaskMeta{ - Name: "convertAccounts", + Name: "Convert Users", EntryPoint: ConvertAccounts, EnabledByDefault: true, Description: "Convert tool layer table gitlab_users into domain layer table accounts", diff --git a/backend/plugins/gitlab/tasks/account_extractor.go b/backend/plugins/gitlab/tasks/account_extractor.go index 1ab39dcb7..d2d24b48c 100644 --- a/backend/plugins/gitlab/tasks/account_extractor.go +++ b/backend/plugins/gitlab/tasks/account_extractor.go @@ -32,7 +32,7 @@ func init() { } var ExtractAccountsMeta = plugin.SubTaskMeta{ - Name: "extractAccounts", + Name: "Extract Users", EntryPoint: ExtractAccounts, EnabledByDefault: true, Description: "Extract raw workspace data into tool layer table _tool_gitlab_accounts", diff --git a/backend/plugins/gitlab/tasks/commit_collector.go b/backend/plugins/gitlab/tasks/commit_collector.go index 3649d0240..4bb2e960f 100644 --- a/backend/plugins/gitlab/tasks/commit_collector.go +++ b/backend/plugins/gitlab/tasks/commit_collector.go @@ -31,7 +31,7 @@ func init() { const RAW_COMMIT_TABLE = "gitlab_api_commit" var CollectApiCommitsMeta = plugin.SubTaskMeta{ - Name: "collectApiCommits", + Name: "Collect Commits", EntryPoint: CollectApiCommits, EnabledByDefault: false, Description: "Collect commit data from gitlab api, does not support either timeFilter or diffSync.", diff --git a/backend/plugins/gitlab/tasks/commit_convertor.go b/backend/plugins/gitlab/tasks/commit_convertor.go index c285a785e..c0fb77a79 100644 --- a/backend/plugins/gitlab/tasks/commit_convertor.go +++ b/backend/plugins/gitlab/tasks/commit_convertor.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/code" @@ -25,7 +27,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" - "reflect" ) func init() { @@ -33,7 +34,7 @@ func init() { } var ConvertCommitsMeta = plugin.SubTaskMeta{ - Name: "convertApiCommits", + Name: "Convert Commits", EntryPoint: ConvertApiCommits, EnabledByDefault: false, Description: "Update domain layer commit according to GitlabCommit", diff --git a/backend/plugins/gitlab/tasks/commit_extractor.go b/backend/plugins/gitlab/tasks/commit_extractor.go index 7cbacf997..664f8eae3 100644 --- a/backend/plugins/gitlab/tasks/commit_extractor.go +++ b/backend/plugins/gitlab/tasks/commit_extractor.go @@ -19,6 +19,7 @@ package tasks import ( "encoding/json" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" @@ -30,7 +31,7 @@ func init() { } var ExtractApiCommitsMeta = plugin.SubTaskMeta{ - Name: "extractApiCommits", + Name: "Extract Commits", EntryPoint: ExtractApiCommits, EnabledByDefault: false, Description: "Extract raw commit data into tool layer table GitlabCommit,GitlabAccount and GitlabProjectCommit", diff --git a/backend/plugins/gitlab/tasks/deployment_collector.go b/backend/plugins/gitlab/tasks/deployment_collector.go index 616f44df3..6c4c2a74a 100644 --- a/backend/plugins/gitlab/tasks/deployment_collector.go +++ b/backend/plugins/gitlab/tasks/deployment_collector.go @@ -39,7 +39,7 @@ func init() { } var CollectDeploymentMeta = plugin.SubTaskMeta{ - Name: "CollectDeployment", + Name: "Collect Deployments", EntryPoint: CollectDeployment, EnabledByDefault: true, Description: "Collect gitlab deployment from api into raw layer table", diff --git a/backend/plugins/gitlab/tasks/deployment_convertor.go b/backend/plugins/gitlab/tasks/deployment_convertor.go index b27b84d55..4db399b95 100644 --- a/backend/plugins/gitlab/tasks/deployment_convertor.go +++ b/backend/plugins/gitlab/tasks/deployment_convertor.go @@ -19,6 +19,9 @@ package tasks import ( "fmt" + "reflect" + "time" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -28,8 +31,6 @@ import ( "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" "github.com/spf13/cast" - "reflect" - "time" ) var _ plugin.SubTaskEntryPoint = ConvertDeployment @@ -39,7 +40,7 @@ func init() { } var ConvertDeploymentMeta = plugin.SubTaskMeta{ - Name: "ConvertDeployment", + Name: "Convert Deployments", EntryPoint: ConvertDeployment, EnabledByDefault: true, Description: "Convert gitlab deployment from tool layer to domain layer", diff --git a/backend/plugins/gitlab/tasks/deployment_extractor.go b/backend/plugins/gitlab/tasks/deployment_extractor.go index a84796cbf..0b439aceb 100644 --- a/backend/plugins/gitlab/tasks/deployment_extractor.go +++ b/backend/plugins/gitlab/tasks/deployment_extractor.go @@ -35,7 +35,7 @@ func init() { } var ExtractDeploymentMeta = plugin.SubTaskMeta{ - Name: "ExtractDeployment", + Name: "Extract Deployments", EntryPoint: ExtractDeployment, EnabledByDefault: true, Description: "Extract gitlab deployment from raw layer to tool layer", diff --git a/backend/plugins/gitlab/tasks/issue_assignee_convertor.go b/backend/plugins/gitlab/tasks/issue_assignee_convertor.go index 9e7366e8b..be40d9675 100644 --- a/backend/plugins/gitlab/tasks/issue_assignee_convertor.go +++ b/backend/plugins/gitlab/tasks/issue_assignee_convertor.go @@ -30,7 +30,7 @@ import ( ) var ConvertIssueAssigneeMeta = plugin.SubTaskMeta{ - Name: "convertIssueAssignee", + Name: "convert Issue Assignees", EntryPoint: ConvertIssueAssignee, EnabledByDefault: true, Description: "Convert tool layer table _tool_gitlab_issue_assignees into domain layer table issue_assignees", diff --git a/backend/plugins/gitlab/tasks/issue_collector.go b/backend/plugins/gitlab/tasks/issue_collector.go index 81c78be37..92c3fbc21 100644 --- a/backend/plugins/gitlab/tasks/issue_collector.go +++ b/backend/plugins/gitlab/tasks/issue_collector.go @@ -36,7 +36,7 @@ func init() { const RAW_ISSUE_TABLE = "gitlab_api_issues" var CollectApiIssuesMeta = plugin.SubTaskMeta{ - Name: "collectApiIssues", + Name: "Collect Issues", EntryPoint: CollectApiIssues, EnabledByDefault: true, Description: "Collect issues data from Gitlab api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/gitlab/tasks/issue_convertor.go b/backend/plugins/gitlab/tasks/issue_convertor.go index 8a09bd7d2..9be3c1d18 100644 --- a/backend/plugins/gitlab/tasks/issue_convertor.go +++ b/backend/plugins/gitlab/tasks/issue_convertor.go @@ -37,7 +37,7 @@ func init() { } var ConvertIssuesMeta = plugin.SubTaskMeta{ - Name: "convertIssues", + Name: "Convert Issues", EntryPoint: ConvertIssues, EnabledByDefault: true, Description: "Convert tool layer table gitlab_issues into domain layer table issues", diff --git a/backend/plugins/gitlab/tasks/issue_extractor.go b/backend/plugins/gitlab/tasks/issue_extractor.go index 9e76e7444..c834935b3 100644 --- a/backend/plugins/gitlab/tasks/issue_extractor.go +++ b/backend/plugins/gitlab/tasks/issue_extractor.go @@ -33,7 +33,7 @@ func init() { } var ExtractApiIssuesMeta = plugin.SubTaskMeta{ - Name: "extractApiIssues", + Name: "Extract Issues", EntryPoint: ExtractApiIssues, EnabledByDefault: true, Description: "Extract raw Issues data into tool layer table gitlab_issues", diff --git a/backend/plugins/gitlab/tasks/issue_label_convertor.go b/backend/plugins/gitlab/tasks/issue_label_convertor.go index c79ce1590..c5e8968dd 100644 --- a/backend/plugins/gitlab/tasks/issue_label_convertor.go +++ b/backend/plugins/gitlab/tasks/issue_label_convertor.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" @@ -25,7 +27,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" - "reflect" ) func init() { @@ -33,7 +34,7 @@ func init() { } var ConvertIssueLabelsMeta = plugin.SubTaskMeta{ - Name: "convertIssueLabels", + Name: "Convert Issue Labels", EntryPoint: ConvertIssueLabels, EnabledByDefault: true, Description: "Convert tool layer table gitlab_issue_labels into domain layer table issue_labels", diff --git a/backend/plugins/gitlab/tasks/job_collector.go b/backend/plugins/gitlab/tasks/job_collector.go index 755f8800b..04dda2998 100644 --- a/backend/plugins/gitlab/tasks/job_collector.go +++ b/backend/plugins/gitlab/tasks/job_collector.go @@ -42,7 +42,7 @@ type SimpleGitlabApiJob struct { } var CollectApiJobsMeta = plugin.SubTaskMeta{ - Name: "collectApiJobs", + Name: "Collect Job Runs", EntryPoint: CollectApiJobs, EnabledByDefault: true, Description: "Collect job data from gitlab api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/gitlab/tasks/job_convertor.go b/backend/plugins/gitlab/tasks/job_convertor.go index dfb70abab..58bd8f712 100644 --- a/backend/plugins/gitlab/tasks/job_convertor.go +++ b/backend/plugins/gitlab/tasks/job_convertor.go @@ -18,6 +18,9 @@ limitations under the License. package tasks import ( + "reflect" + "time" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -26,8 +29,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" gitlabModels "github.com/apache/incubator-devlake/plugins/gitlab/models" - "reflect" - "time" ) func init() { @@ -35,7 +36,7 @@ func init() { } var ConvertJobMeta = plugin.SubTaskMeta{ - Name: "convertJobs", + Name: "Convert Job Runs", EntryPoint: ConvertJobs, EnabledByDefault: true, Description: "Convert tool layer table gitlab_job into domain layer table job", diff --git a/backend/plugins/gitlab/tasks/job_extractor.go b/backend/plugins/gitlab/tasks/job_extractor.go index 7eb989770..ff5031ce6 100644 --- a/backend/plugins/gitlab/tasks/job_extractor.go +++ b/backend/plugins/gitlab/tasks/job_extractor.go @@ -51,7 +51,7 @@ type ApiJob struct { } var ExtractApiJobsMeta = plugin.SubTaskMeta{ - Name: "extractApiJobs", + Name: "Extract Job Runs", EntryPoint: ExtractApiJobs, EnabledByDefault: true, Description: "Extract raw GitlabJob data into tool layer table GitlabPipeline", diff --git a/backend/plugins/gitlab/tasks/mr_collector.go b/backend/plugins/gitlab/tasks/mr_collector.go index ce0f21b3c..029b19043 100644 --- a/backend/plugins/gitlab/tasks/mr_collector.go +++ b/backend/plugins/gitlab/tasks/mr_collector.go @@ -33,7 +33,7 @@ func init() { } var CollectApiMergeRequestsMeta = plugin.SubTaskMeta{ - Name: "collectApiMergeRequests", + Name: "Collect Merge Requests", EntryPoint: CollectApiMergeRequests, EnabledByDefault: true, Description: "Collect merge requests data from gitlab api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/gitlab/tasks/mr_comment_convertor.go b/backend/plugins/gitlab/tasks/mr_comment_convertor.go index c309792cf..3b6d1ccee 100644 --- a/backend/plugins/gitlab/tasks/mr_comment_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_comment_convertor.go @@ -35,7 +35,7 @@ func init() { } var ConvertMrCommentMeta = plugin.SubTaskMeta{ - Name: "convertMergeRequestComment", + Name: "Convert MR Comments", EntryPoint: ConvertMergeRequestComment, EnabledByDefault: true, Description: "Add domain layer Comment according to GitlabMrComment", diff --git a/backend/plugins/gitlab/tasks/mr_commit_collector.go b/backend/plugins/gitlab/tasks/mr_commit_collector.go index 2fbbdd48a..f99211218 100644 --- a/backend/plugins/gitlab/tasks/mr_commit_collector.go +++ b/backend/plugins/gitlab/tasks/mr_commit_collector.go @@ -30,7 +30,7 @@ func init() { const RAW_MERGE_REQUEST_COMMITS_TABLE = "gitlab_api_merge_request_commits" var CollectApiMrCommitsMeta = plugin.SubTaskMeta{ - Name: "collectApiMergeRequestsCommits", + Name: "Collect MR Commits", EntryPoint: CollectApiMergeRequestsCommits, EnabledByDefault: true, Description: "Collect merge requests commits data from gitlab api, supports timeFilter but not diffSync.", diff --git a/backend/plugins/gitlab/tasks/mr_commit_convertor.go b/backend/plugins/gitlab/tasks/mr_commit_convertor.go index 751e84a96..74ae0d926 100644 --- a/backend/plugins/gitlab/tasks/mr_commit_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_commit_convertor.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/code" @@ -25,7 +27,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" - "reflect" ) func init() { @@ -33,7 +34,7 @@ func init() { } var ConvertApiMrCommitsMeta = plugin.SubTaskMeta{ - Name: "convertApiMergeRequestsCommits", + Name: "Convert MR Commits", EntryPoint: ConvertApiMergeRequestsCommits, EnabledByDefault: true, Description: "Add domain layer PullRequestCommit according to GitlabMrCommit", diff --git a/backend/plugins/gitlab/tasks/mr_commit_extractor.go b/backend/plugins/gitlab/tasks/mr_commit_extractor.go index aa71305fb..5c73c8665 100644 --- a/backend/plugins/gitlab/tasks/mr_commit_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_commit_extractor.go @@ -32,7 +32,7 @@ func init() { } var ExtractApiMrCommitsMeta = plugin.SubTaskMeta{ - Name: "extractApiMergeRequestsCommits", + Name: "Extract MR Commits", EntryPoint: ExtractApiMergeRequestsCommits, EnabledByDefault: true, Description: "Extract raw merge requests commit data into tool layer table GitlabMrCommit and GitlabCommit", diff --git a/backend/plugins/gitlab/tasks/mr_convertor.go b/backend/plugins/gitlab/tasks/mr_convertor.go index 53f5ea436..478276c71 100644 --- a/backend/plugins/gitlab/tasks/mr_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_convertor.go @@ -35,7 +35,7 @@ func init() { } var ConvertApiMergeRequestsMeta = plugin.SubTaskMeta{ - Name: "convertApiMergeRequests", + Name: "Convert Merge Requests", EntryPoint: ConvertApiMergeRequests, EnabledByDefault: true, Description: "Add domain layer PullRequest according to GitlabMergeRequest", diff --git a/backend/plugins/gitlab/tasks/mr_detail_collector.go b/backend/plugins/gitlab/tasks/mr_detail_collector.go index f60871a24..eb4f8f3a6 100644 --- a/backend/plugins/gitlab/tasks/mr_detail_collector.go +++ b/backend/plugins/gitlab/tasks/mr_detail_collector.go @@ -34,7 +34,7 @@ func init() { const RAW_MERGE_REQUEST_DETAIL_TABLE = "gitlab_api_merge_request_details" var CollectApiMergeRequestDetailsMeta = plugin.SubTaskMeta{ - Name: "collectApiMergeRequestDetails", + Name: "Collect MR Details", EntryPoint: CollectApiMergeRequestDetails, EnabledByDefault: true, Description: "Collect merge request Details data from gitlab api, supports timeFilter but not diffSync.", diff --git a/backend/plugins/gitlab/tasks/mr_detail_extractor.go b/backend/plugins/gitlab/tasks/mr_detail_extractor.go index 9453a59fc..38ce2901c 100644 --- a/backend/plugins/gitlab/tasks/mr_detail_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_detail_extractor.go @@ -32,7 +32,7 @@ func init() { } var ExtractApiMergeRequestDetailsMeta = plugin.SubTaskMeta{ - Name: "extractApiMergeRequestDetails", + Name: "Extract MR Details", EntryPoint: ExtractApiMergeRequestDetails, EnabledByDefault: true, Description: "Extract raw merge request Details data into tool layer table GitlabMergeRequest and GitlabReviewer", diff --git a/backend/plugins/gitlab/tasks/mr_enricher.go b/backend/plugins/gitlab/tasks/mr_enricher.go index 256dffb7e..327765208 100644 --- a/backend/plugins/gitlab/tasks/mr_enricher.go +++ b/backend/plugins/gitlab/tasks/mr_enricher.go @@ -18,13 +18,14 @@ limitations under the License. package tasks import ( + "reflect" + "time" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" - "reflect" - "time" ) func init() { @@ -32,7 +33,7 @@ func init() { } var EnrichMergeRequestsMeta = plugin.SubTaskMeta{ - Name: "enrichMrs", + Name: "Enrich Merge Requests", EntryPoint: EnrichMergeRequests, EnabledByDefault: true, Description: "Enrich merge requests data from GitlabCommit, GitlabMrNote and GitlabMergeRequest", diff --git a/backend/plugins/gitlab/tasks/mr_extractor.go b/backend/plugins/gitlab/tasks/mr_extractor.go index 92bfccfa3..a45bc924e 100644 --- a/backend/plugins/gitlab/tasks/mr_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_extractor.go @@ -77,7 +77,7 @@ type Reviewer struct { } var ExtractApiMergeRequestsMeta = plugin.SubTaskMeta{ - Name: "extractApiMergeRequests", + Name: "Extract Merge Requests", EntryPoint: ExtractApiMergeRequests, EnabledByDefault: true, Description: "Extract raw merge requests data into tool layer table GitlabMergeRequest and GitlabReviewer", diff --git a/backend/plugins/gitlab/tasks/mr_label_convertor.go b/backend/plugins/gitlab/tasks/mr_label_convertor.go index 603332438..934085e81 100644 --- a/backend/plugins/gitlab/tasks/mr_label_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_label_convertor.go @@ -18,6 +18,8 @@ limitations under the License. package tasks import ( + "reflect" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/code" @@ -25,7 +27,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" - "reflect" ) func init() { @@ -33,7 +34,7 @@ func init() { } var ConvertMrLabelsMeta = plugin.SubTaskMeta{ - Name: "convertMrLabels", + Name: "Convert MR Labels", EntryPoint: ConvertMrLabels, EnabledByDefault: true, Description: "Convert tool layer table gitlab_mr_labels into domain layer table pull_request_labels", diff --git a/backend/plugins/gitlab/tasks/mr_note_collector.go b/backend/plugins/gitlab/tasks/mr_note_collector.go index 282135982..469648f71 100644 --- a/backend/plugins/gitlab/tasks/mr_note_collector.go +++ b/backend/plugins/gitlab/tasks/mr_note_collector.go @@ -30,7 +30,7 @@ func init() { const RAW_MERGE_REQUEST_NOTES_TABLE = "gitlab_api_merge_request_notes" var CollectApiMrNotesMeta = plugin.SubTaskMeta{ - Name: "collectApiMergeRequestsNotes", + Name: "Collect MR Notes", EntryPoint: CollectApiMergeRequestsNotes, EnabledByDefault: true, Description: "Collect merge requests notes data from gitlab api, supports timeFilter but not diffSync.", diff --git a/backend/plugins/gitlab/tasks/mr_note_extractor.go b/backend/plugins/gitlab/tasks/mr_note_extractor.go index 3b6786c52..a27d1141c 100644 --- a/backend/plugins/gitlab/tasks/mr_note_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_note_extractor.go @@ -49,7 +49,7 @@ type MergeRequestNote struct { } var ExtractApiMrNotesMeta = plugin.SubTaskMeta{ - Name: "extractApiMergeRequestsNotes", + Name: "Extract MR Notes", EntryPoint: ExtractApiMergeRequestsNotes, EnabledByDefault: true, Description: "Extract raw merge requests notes data into tool layer table GitlabMrNote", diff --git a/backend/plugins/gitlab/tasks/pipeline_collector.go b/backend/plugins/gitlab/tasks/pipeline_collector.go index 06a14c925..ceb911d68 100644 --- a/backend/plugins/gitlab/tasks/pipeline_collector.go +++ b/backend/plugins/gitlab/tasks/pipeline_collector.go @@ -34,7 +34,7 @@ func init() { const RAW_PIPELINE_TABLE = "gitlab_api_pipeline" var CollectApiPipelinesMeta = plugin.SubTaskMeta{ - Name: "collectApiPipelines", + Name: "Collect Pipelines", EntryPoint: CollectApiPipelines, EnabledByDefault: true, Description: "Collect pipeline data from gitlab api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go index 9950d39c4..a15928c4d 100644 --- a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go +++ b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go @@ -34,7 +34,7 @@ func init() { } var ConvertPipelineCommitMeta = plugin.SubTaskMeta{ - Name: "convertPipelineCommits", + Name: "Convert Pipeline Commits", EntryPoint: ConvertPipelineCommits, EnabledByDefault: true, Description: "Convert tool layer table gitlab_pipeline_project into domain layer table pipeline", diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_collector.go b/backend/plugins/gitlab/tasks/pipeline_detail_collector.go index ebafdc2c0..227d19e93 100644 --- a/backend/plugins/gitlab/tasks/pipeline_detail_collector.go +++ b/backend/plugins/gitlab/tasks/pipeline_detail_collector.go @@ -35,7 +35,7 @@ func init() { const RAW_PIPELINE_DETAILS_TABLE = "gitlab_api_pipeline_details" var CollectApiPipelineDetailsMeta = plugin.SubTaskMeta{ - Name: "collectApiPipelineDetails", + Name: "Collect Pipeline Details", EntryPoint: CollectApiPipelineDetails, EnabledByDefault: true, Description: "Collect pipeline details data from gitlab api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go index 36f9bcea6..3818a9a69 100644 --- a/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go +++ b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go @@ -37,7 +37,7 @@ func init() { } var ConvertDetailPipelineMeta = plugin.SubTaskMeta{ - Name: "convertDetailPipelines", + Name: "Convert Detail Pipelines", EntryPoint: ConvertDetailPipelines, EnabledByDefault: true, Description: "Convert tool layer table gitlab_detail_pipeline into domain layer table pipeline", diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go index 52c12c04a..83dab29c7 100644 --- a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go +++ b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go @@ -33,7 +33,7 @@ func init() { } var ExtractApiPipelineDetailsMeta = plugin.SubTaskMeta{ - Name: "extractApiPipelineDetails", + Name: "Extract Pipeline Details", EntryPoint: ExtractApiPipelineDetails, EnabledByDefault: true, Description: "Extract raw pipeline details data into tool layer table GitlabPipeline", diff --git a/backend/plugins/gitlab/tasks/pipeline_extractor.go b/backend/plugins/gitlab/tasks/pipeline_extractor.go index 3b392bf09..738109ac4 100644 --- a/backend/plugins/gitlab/tasks/pipeline_extractor.go +++ b/backend/plugins/gitlab/tasks/pipeline_extractor.go @@ -61,7 +61,7 @@ type ApiPipeline struct { } var ExtractApiPipelinesMeta = plugin.SubTaskMeta{ - Name: "extractApiPipelines", + Name: "Extract Pipelines", EntryPoint: ExtractApiPipelines, EnabledByDefault: true, Description: "Extract raw pipelines data into tool layer table GitlabPipeline", diff --git a/backend/plugins/gitlab/tasks/project_convertor.go b/backend/plugins/gitlab/tasks/project_convertor.go index 614355f30..920f1a990 100644 --- a/backend/plugins/gitlab/tasks/project_convertor.go +++ b/backend/plugins/gitlab/tasks/project_convertor.go @@ -59,7 +59,7 @@ type GitlabApiProject struct { } var ConvertProjectMeta = plugin.SubTaskMeta{ - Name: "convertApiProject", + Name: "Convert Projects", EntryPoint: ConvertApiProjects, EnabledByDefault: true, Description: "Add domain layer Repo according to GitlabProject", diff --git a/backend/plugins/gitlab/tasks/tag_collector.go b/backend/plugins/gitlab/tasks/tag_collector.go index af5e518ae..92800c5c9 100644 --- a/backend/plugins/gitlab/tasks/tag_collector.go +++ b/backend/plugins/gitlab/tasks/tag_collector.go @@ -30,7 +30,7 @@ func init() { const RAW_TAG_TABLE = "gitlab_api_tag" var CollectTagMeta = plugin.SubTaskMeta{ - Name: "collectApiTag", + Name: "Collect Tags", EntryPoint: CollectApiTag, EnabledByDefault: false, Description: "Collect tag data from gitlab api, does not support either timeFilter or diffSync.", diff --git a/backend/plugins/gitlab/tasks/tag_extractor.go b/backend/plugins/gitlab/tasks/tag_extractor.go index 9a94de337..8aba401f3 100644 --- a/backend/plugins/gitlab/tasks/tag_extractor.go +++ b/backend/plugins/gitlab/tasks/tag_extractor.go @@ -19,6 +19,7 @@ package tasks import ( "encoding/json" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" @@ -41,7 +42,7 @@ type GitlabApiTag struct { } var ExtractTagMeta = plugin.SubTaskMeta{ - Name: "extractApiTag", + Name: "Extract Tags", EntryPoint: ExtractApiTag, EnabledByDefault: false, Description: "Extract raw tag data into tool layer table GitlabTag", diff --git a/backend/plugins/gitlab/tasks/trigger_job_collector.go b/backend/plugins/gitlab/tasks/trigger_job_collector.go index 50e4d04e1..755eeb67a 100644 --- a/backend/plugins/gitlab/tasks/trigger_job_collector.go +++ b/backend/plugins/gitlab/tasks/trigger_job_collector.go @@ -34,7 +34,7 @@ func init() { const RAW_TRIGGER_JOB_TABLE = "gitlab_api_trigger_job" var CollectApiTriggerJobsMeta = plugin.SubTaskMeta{ - Name: "collectApiTriggerJobs", + Name: "Collect Trigger Jobs", EntryPoint: CollectApiTriggerJobs, EnabledByDefault: false, Description: "Collect job data from gitlab api, supports both timeFilter and diffSync.", diff --git a/backend/plugins/gitlab/tasks/trigger_job_extractor.go b/backend/plugins/gitlab/tasks/trigger_job_extractor.go index f9b64eeb4..f429f9b4d 100644 --- a/backend/plugins/gitlab/tasks/trigger_job_extractor.go +++ b/backend/plugins/gitlab/tasks/trigger_job_extractor.go @@ -19,11 +19,12 @@ package tasks import ( "encoding/json" + "time" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" - "time" ) func init() { @@ -46,7 +47,7 @@ type ApiTriggerJob struct { } var ExtractApiTriggerJobsMeta = plugin.SubTaskMeta{ - Name: "extractApiTriggerJobs", + Name: "Extract Trigger Jobs", EntryPoint: ExtractApiTriggerJobs, EnabledByDefault: true, Description: "Extract raw Gitlab trigger jobs data into tool layer table GitlabPipeline", diff --git a/backend/python/pydevlake/pydevlake/subtasks.py b/backend/python/pydevlake/pydevlake/subtasks.py index c1552f5b6..d3b549e0b 100644 --- a/backend/python/pydevlake/pydevlake/subtasks.py +++ b/backend/python/pydevlake/pydevlake/subtasks.py @@ -34,7 +34,8 @@ class Subtask: @property def name(self): - return f'{self.verb.lower()}{self.stream.plugin_name.capitalize()}{self.stream.name.capitalize()}' + #return f'{self.verb.lower()}{self.stream.plugin_name.capitalize()}{self.stream.name.capitalize()}' + return f'{self.verb.capitalize()} {self.stream.name.capitalize()}' @property def description(self): diff --git a/backend/server/api/router.go b/backend/server/api/router.go index 188c29e4c..9577b2494 100644 --- a/backend/server/api/router.go +++ b/backend/server/api/router.go @@ -48,6 +48,7 @@ func RegisterRouter(r *gin.Engine, basicRes context.BasicRes) { r.GET("/pipelines/:pipelineId", pipelines.Get) r.DELETE("/pipelines/:pipelineId", pipelines.Delete) r.GET("/pipelines/:pipelineId/tasks", task.GetTaskByPipeline) + r.GET("/pipelines/:pipelineId/subtasks", task.GetSubtaskByPipeline) r.POST("/pipelines/:pipelineId/rerun", pipelines.PostRerun) r.GET("/pipelines/:pipelineId/logging.tar.gz", pipelines.DownloadLogs) diff --git a/backend/server/api/task/task.go b/backend/server/api/task/task.go index f43e574a1..594e9d6c3 100644 --- a/backend/server/api/task/task.go +++ b/backend/server/api/task/task.go @@ -72,6 +72,30 @@ func GetTaskByPipeline(c *gin.Context) { shared.ApiOutputSuccess(c, getTaskResponse{Tasks: tasks, Count: len(tasks)}, http.StatusOK) } +// GetSubtasksByPipeline return most recent subtasks +// @Summary Get subtasks, only the most recent subtasks will be returned +// @Tags framework/tasks +// @Accept application/json +// @Param pipelineId path int true "pipelineId" +// @Success 200 {object} models.SubTasksOuput +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /pipelines/{pipelineId}/subtasks [get] +func GetSubtaskByPipeline(c *gin.Context) { + pipelineId, err := strconv.ParseUint(c.Param("pipelineId"), 10, 64) + if err != nil { + shared.ApiOutputError(c, errors.BadInput.Wrap(err, "invalid pipeline ID format")) + return + } + subTasksOuput, err := services.GetSubTasksInfo(pipelineId, true, nil) + if err != nil { + shared.ApiOutputError(c, errors.Default.Wrap(err, "error getting tasks")) + return + } + + shared.ApiOutputSuccess(c, subTasksOuput, http.StatusOK) +} + // RerunTask rerun the specified task. // @Summary rerun task // @Tags framework/tasks diff --git a/backend/server/services/task.go b/backend/server/services/task.go index 885014fa3..5640b7a6c 100644 --- a/backend/server/services/task.go +++ b/backend/server/services/task.go @@ -20,6 +20,7 @@ package services import ( "context" "fmt" + "math" "strings" "github.com/apache/incubator-devlake/core/dal" @@ -236,3 +237,147 @@ func RerunTask(taskId uint64) (*models.Task, errors.Error) { rerunTask.Options = taskOption return rerunTask, nil } + +// GetSubTasksInfo returns subtask list of the pipeline, only the most recently subtasks would be returned +func GetSubTasksInfo(pipelineId uint64, shouldSanitize bool, tx dal.Dal) (*models.SubTasksOuput, errors.Error) { + if tx == nil { + tx = db + } + var tasks []*models.Task + err := tx.All(&tasks, dal.Where("pipeline_id = ?", pipelineId)) + if err != nil { + return nil, err + } + filterTasks := filterTasks(tasks) + var subtasksInfo []models.SubtasksInfo + var totalSubtasksCount int64 + var totalFinishedSubTasksCount int64 + var count int64 + var status []string + for _, task := range filterTasks { + // skip org plugin step + if task.Plugin == "org" { + continue + } + subTaskResult := models.SubtasksInfo{ + ID: task.ID, + PipelineID: task.PipelineId, + CreatedAt: task.CreatedAt, + UpdatedAt: task.UpdatedAt, + BeganAt: task.BeganAt, + FinishedAt: task.FinishedAt, + Plugin: task.Plugin, + Status: task.Status, + Message: task.Message, + ErrorName: task.ErrorName, + SpentSeconds: task.SpentSeconds, + } + if shouldSanitize { + taskOption, err := SanitizePluginOption(task.Plugin, task.Options) + if err != nil { + return nil, errors.Convert(err) + } + subTaskResult.Options = taskOption + } + + subtasks := []*models.Subtask{} + err = tx.All(&subtasks, dal.Where("task_id = ?", task.ID)) + if err != nil { + return nil, err + } + for _, subtask := range subtasks { + t := &models.SubtaskDetails{ + ID: subtask.ID, + CreatedAt: subtask.CreatedAt, + UpdatedAt: subtask.UpdatedAt, + TaskID: subtask.TaskID, + Name: subtask.Name, + Number: subtask.Number, + BeganAt: subtask.BeganAt, + FinishedAt: subtask.FinishedAt, + SpentSeconds: subtask.SpentSeconds, + FinishedRecords: subtask.FinishedRecords, + Sequence: subtask.Sequence, + IsCollector: subtask.IsCollector, + IsFailed: subtask.IsFailed, + Message: subtask.Message, + } + subTaskResult.SubtaskDetails = append(subTaskResult.SubtaskDetails, t) + } + subtasksInfo = append(subtasksInfo, subTaskResult) + + collectSubtasksCount := errors.Must1(tx.Count(dal.From("_devlake_subtasks"), dal.Where("task_id = ?", task.ID))) + totalSubtasksCount += collectSubtasksCount + finishedSubTasksCount := errors.Must1(tx.Count(dal.From("_devlake_subtasks"), dal.Where("task_id = ? and finished_at is not null", task.ID))) + totalFinishedSubTasksCount += finishedSubTasksCount + count++ + + status = append(status, task.Status) + } + + subTasksOuput := &models.SubTasksOuput{} + subTasksOuput.SubtasksInfo = subtasksInfo + subTasksOuput.Count = totalSubtasksCount + + completionRateFloat := float64(totalFinishedSubTasksCount) / float64(totalSubtasksCount) + roundedCompletionRate := math.Round(completionRateFloat*100) / 100 + subTasksOuput.CompletionRate = roundedCompletionRate + + subTasksOuput.Status = getTaskStatus(status) + subTasksOuput.Count = count + + return subTasksOuput, nil +} + +func filterTasks(tasks []*models.Task) []*models.Task { + taskMap := make(map[string]*models.Task) + + for _, task := range tasks { + if task.Plugin == "org" { + continue + } + + if existingTask, ok := taskMap[task.Plugin]; ok { + if task.BeganAt != nil && (existingTask.BeganAt == nil || task.BeganAt.After(*existingTask.BeganAt)) { + taskMap[task.Plugin] = task + } + } else { + taskMap[task.Plugin] = task + } + } + + var filteredTasks []*models.Task + for _, task := range taskMap { + filteredTasks = append(filteredTasks, task) + } + + return filteredTasks +} + +func getTaskStatus(statuses []string) string { + var status string + if len(statuses) == 0 { + return status + } + + failedCount := 0 + completedCount := 0 + for _, s := range statuses { + if s == models.TASK_FAILED { + failedCount++ + } else if s == models.TASK_COMPLETED { + completedCount++ + } + } + if failedCount > 0 && completedCount > 0 { + status = "TASK_PARTIAL" + } else if failedCount == len(statuses) { + status = models.TASK_FAILED + } else if completedCount == len(statuses) { + status = models.TASK_COMPLETED + } else { + status = models.TASK_RUNNING + } + + return status +}
