This is an automated email from the ASF dual-hosted git repository. lynwee pushed a commit to branch 10-cp in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit d38f95dcfe54e6b9aade756a7479bc350dd602aa Author: d4x1 <[email protected]> AuthorDate: Tue Jul 16 11:22:57 2024 +0800 fix(jira): remove all usage of `tmpFromAccountId` and `tmpToAccountId` --- .../plugins/jira/tasks/apiv2models/changelog.go | 14 ++------ backend/plugins/jira/tasks/apiv2models/issue.go | 4 +-- backend/plugins/jira/tasks/epic_extractor.go | 38 +++++++++++++++++++++- .../jira/tasks/issue_changelog_convertor.go | 11 ++----- .../jira/tasks/issue_changelog_extractor.go | 6 +++- backend/plugins/jira/tasks/issue_extractor.go | 10 ++++-- 6 files changed, 56 insertions(+), 27 deletions(-) diff --git a/backend/plugins/jira/tasks/apiv2models/changelog.go b/backend/plugins/jira/tasks/apiv2models/changelog.go index 5c5c2c2a6..1ad6b9825 100644 --- a/backend/plugins/jira/tasks/apiv2models/changelog.go +++ b/backend/plugins/jira/tasks/apiv2models/changelog.go @@ -76,23 +76,15 @@ func (c ChangelogItem) ToToolLayer(connectionId, changelogId uint64) *models.Jir return item } -func (c ChangelogItem) ExtractUser(connectionId uint64) []*models.JiraAccount { +func (c ChangelogItem) ExtractUser(connectionId uint64, userFieldMaps map[string]struct{}) []*models.JiraAccount { var result []*models.JiraAccount - // if `tmpFromAccountId` or `tmpToAccountId` is not empty, then this change log item stands for changes about accounts. - if c.TmpFromAccountId != "" { - // User `from` firstly + _, ok := userFieldMaps[c.Field] + if c.Field == "assignee" || c.Field == "reporter" || ok { if c.FromValue != "" { result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.FromValue}) - } else { - result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.TmpFromAccountId}) } - } - if c.TmpToAccountId != "" { - // User `to` firstly if c.ToValue != "" { result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.ToValue}) - } else { - result = append(result, &models.JiraAccount{ConnectionId: connectionId, AccountId: c.TmpToAccountId}) } } return result diff --git a/backend/plugins/jira/tasks/apiv2models/issue.go b/backend/plugins/jira/tasks/apiv2models/issue.go index 8a4417a3d..68b5a26b6 100644 --- a/backend/plugins/jira/tasks/apiv2models/issue.go +++ b/backend/plugins/jira/tasks/apiv2models/issue.go @@ -300,7 +300,7 @@ func (i *Issue) SetAllFields(raw json.RawMessage) errors.Error { return nil } -func (i Issue) ExtractEntities(connectionId uint64) ([]uint64, *models.JiraIssue, []*models.JiraIssueComment, []*models.JiraWorklog, []*models.JiraIssueChangelogs, []*models.JiraIssueChangelogItems, []*models.JiraAccount) { +func (i Issue) ExtractEntities(connectionId uint64, userFieldMaps map[string]struct{}) ([]uint64, *models.JiraIssue, []*models.JiraIssueComment, []*models.JiraWorklog, []*models.JiraIssueChangelogs, []*models.JiraIssueChangelogItems, []*models.JiraAccount) { issue := i.toToolLayer(connectionId) var comments []*models.JiraIssueComment var worklogs []*models.JiraWorklog @@ -341,7 +341,7 @@ func (i Issue) ExtractEntities(connectionId uint64) ([]uint64, *models.JiraIssue } for _, item := range changelog.Items { changelogItems = append(changelogItems, item.ToToolLayer(connectionId, changelog.ID)) - users = append(users, item.ExtractUser(connectionId)...) + users = append(users, item.ExtractUser(connectionId, userFieldMaps)...) } } } diff --git a/backend/plugins/jira/tasks/epic_extractor.go b/backend/plugins/jira/tasks/epic_extractor.go index 8c3c8eab3..c9a981f3d 100644 --- a/backend/plugins/jira/tasks/epic_extractor.go +++ b/backend/plugins/jira/tasks/epic_extractor.go @@ -18,9 +18,12 @@ limitations under the License. package tasks import ( + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/jira/models" ) var _ plugin.SubTaskEntryPoint = ExtractEpics @@ -44,6 +47,10 @@ func ExtractEpics(taskCtx plugin.SubTaskContext) errors.Error { if err != nil { return err } + userFieldMap, err := getUserFieldMap(db, connectionId, logger) + if err != nil { + return err + } extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ RawDataSubTaskArgs: api.RawDataSubTaskArgs{ Ctx: taskCtx, @@ -54,7 +61,7 @@ func ExtractEpics(taskCtx plugin.SubTaskContext) errors.Error { Table: RAW_EPIC_TABLE, }, Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - return extractIssues(data, mappings, row) + return extractIssues(data, mappings, row, userFieldMap) }, }) if err != nil { @@ -62,3 +69,32 @@ func ExtractEpics(taskCtx plugin.SubTaskContext) errors.Error { } return extractor.Execute() } + +func getIssueFieldMap(db dal.Dal, connectionId uint64, logger log.Logger) (map[string]models.JiraIssueField, errors.Error) { + var allIssueFields []models.JiraIssueField + if err := db.All(&allIssueFields, dal.Where("connection_id = ?", connectionId)); err != nil { + return nil, err + } + issueFieldMap := make(map[string]models.JiraIssueField) + for _, v := range allIssueFields { + if _, ok := issueFieldMap[v.Name]; ok { + logger.Warn(nil, "filed name %s is duplicated", v.Name) + } + issueFieldMap[v.Name] = v + } + return issueFieldMap, nil +} + +func getUserFieldMap(db dal.Dal, connectionId uint64, logger log.Logger) (map[string]struct{}, errors.Error) { + userFieldMap := make(map[string]struct{}) + issueFieldMap, err := getIssueFieldMap(db, connectionId, logger) + if err != nil { + return nil, err + } + for filedName, issueField := range issueFieldMap { + if issueField.SchemaType == "user" { + userFieldMap[filedName] = struct{}{} + } + } + return userFieldMap, nil +} diff --git a/backend/plugins/jira/tasks/issue_changelog_convertor.go b/backend/plugins/jira/tasks/issue_changelog_convertor.go index 3ad3b7515..142b57017 100644 --- a/backend/plugins/jira/tasks/issue_changelog_convertor.go +++ b/backend/plugins/jira/tasks/issue_changelog_convertor.go @@ -68,17 +68,10 @@ func ConvertIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error { statusMap[v.ID] = v } - var allIssueFields []models.JiraIssueField - if err := db.All(&allIssueFields, dal.Where("connection_id = ?", connectionId)); err != nil { + issueFieldMap, err := getIssueFieldMap(db, connectionId, logger) + if err != nil { return err } - issueFieldMap := make(map[string]models.JiraIssueField) - for _, v := range allIssueFields { - if _, ok := issueFieldMap[v.Name]; ok { - logger.Warn(nil, "filed name %s is duplicated", v.Name) - } - issueFieldMap[v.Name] = v - } issueIdGenerator := didgen.NewDomainIdGenerator(&models.JiraIssue{}) sprintIdGenerator := didgen.NewDomainIdGenerator(&models.JiraSprint{}) diff --git a/backend/plugins/jira/tasks/issue_changelog_extractor.go b/backend/plugins/jira/tasks/issue_changelog_extractor.go index f333f82c2..a49142e3f 100644 --- a/backend/plugins/jira/tasks/issue_changelog_extractor.go +++ b/backend/plugins/jira/tasks/issue_changelog_extractor.go @@ -43,6 +43,10 @@ func ExtractIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error { return nil } connectionId := data.Options.ConnectionId + userFieldMap, err := getUserFieldMap(subtaskCtx.GetDal(), connectionId, subtaskCtx.GetLogger()) + if err != nil { + return err + } extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ SubtaskCommonArgs: &api.SubtaskCommonArgs{ SubTaskContext: subtaskCtx, @@ -77,7 +81,7 @@ func ExtractIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error { // collect changelog_items for _, item := range changelog.Items { result = append(result, item.ToToolLayer(connectionId, changelog.ID)) - extractedUsersFromChangelogItem := item.ExtractUser(connectionId) + extractedUsersFromChangelogItem := item.ExtractUser(connectionId, userFieldMap) for _, u := range extractedUsersFromChangelogItem { if u != nil && u.AccountId != "" { result = append(result, u) diff --git a/backend/plugins/jira/tasks/issue_extractor.go b/backend/plugins/jira/tasks/issue_extractor.go index 7dcb6da76..3d097d691 100644 --- a/backend/plugins/jira/tasks/issue_extractor.go +++ b/backend/plugins/jira/tasks/issue_extractor.go @@ -58,6 +58,10 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext) errors.Error { if err != nil { return err } + userFieldMap, err := getUserFieldMap(db, connectionId, logger) + if err != nil { + return err + } extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ SubtaskCommonArgs: &api.SubtaskCommonArgs{ SubTaskContext: subtaskCtx, @@ -69,7 +73,7 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext) errors.Error { SubtaskConfig: mappings, }, Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - return extractIssues(data, mappings, row) + return extractIssues(data, mappings, row, userFieldMap) }, }) if err != nil { @@ -78,7 +82,7 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext) errors.Error { return extractor.Execute() } -func extractIssues(data *JiraTaskData, mappings *typeMappings, row *api.RawData) ([]interface{}, errors.Error) { +func extractIssues(data *JiraTaskData, mappings *typeMappings, row *api.RawData, userFieldMaps map[string]struct{}) ([]interface{}, errors.Error) { var apiIssue apiv2models.Issue err := errors.Convert(json.Unmarshal(row.Data, &apiIssue)) if err != nil { @@ -93,7 +97,7 @@ func extractIssues(data *JiraTaskData, mappings *typeMappings, row *api.RawData) if apiIssue.Fields.Created == nil { return results, nil } - sprints, issue, comments, worklogs, changelogs, changelogItems, users := apiIssue.ExtractEntities(data.Options.ConnectionId) + sprints, issue, comments, worklogs, changelogs, changelogItems, users := apiIssue.ExtractEntities(data.Options.ConnectionId, userFieldMaps) for _, sprintId := range sprints { sprintIssue := &models.JiraSprintIssue{ ConnectionId: data.Options.ConnectionId,
