This is an automated email from the ASF dual-hosted git repository. narro pushed a commit to branch feat-8400 in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 7b13824c5247cec4902407dee2e65ebdb8536848 Author: narro wizard <[email protected]> AuthorDate: Wed Apr 23 16:54:55 2025 +0800 feat(customize): add account handling for imported issues - Create or update accounts for creator and assignee - Link accounts to issues in the database - Add account data to issues_output.csv - Verify account data with snapshot_tables/accounts.csv #8400 --- .../plugins/customize/e2e/import_issues_test.go | 11 +++ .../customize/e2e/raw_tables/issues_input.csv | 6 +- .../customize/e2e/snapshot_tables/accounts.csv | 5 + .../e2e/snapshot_tables/issues_output.csv | 12 +-- backend/plugins/customize/service/service.go | 101 +++++++++++++++++++-- 5 files changed, 117 insertions(+), 18 deletions(-) diff --git a/backend/plugins/customize/e2e/import_issues_test.go b/backend/plugins/customize/e2e/import_issues_test.go index aa875eb92..82de28d09 100644 --- a/backend/plugins/customize/e2e/import_issues_test.go +++ b/backend/plugins/customize/e2e/import_issues_test.go @@ -21,6 +21,7 @@ import ( "os" "testing" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" "github.com/apache/incubator-devlake/helpers/e2ehelper" "github.com/apache/incubator-devlake/plugins/customize/impl" @@ -37,6 +38,7 @@ func TestImportIssueDataFlow(t *testing.T) { dataflowTester.FlushTabler(&models.CustomizedField{}) dataflowTester.FlushTabler(&ticket.IssueLabel{}) dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.FlushTabler(&crossdomain.Account{}) svc := service.NewService(dataflowTester.Dal) err := svc.CreateField(&models.CustomizedField{ TbName: "issues", @@ -172,4 +174,13 @@ func TestImportIssueDataFlow(t *testing.T) { "board_id", "issue_id", }) + dataflowTester.VerifyTableWithRawData( + &crossdomain.Account{}, + "snapshot_tables/accounts.csv", + []string{ + "id", + "full_name", + "user_name", + }, + ) } diff --git a/backend/plugins/customize/e2e/raw_tables/issues_input.csv b/backend/plugins/customize/e2e/raw_tables/issues_input.csv index 32e72055f..1eacd5fcd 100644 --- a/backend/plugins/customize/e2e/raw_tables/issues_input.csv +++ b/backend/plugins/customize/e2e/raw_tables/issues_input.csv @@ -1,4 +1,4 @@ id,url,issue_key,title,original_type,original_status,created_date,resolution_date,story_point,priority,severity,original_estimate_minutes,time_spent_minutes,component,epic_key,creator_name,assignee_name,x_int,x_time,x_varchar,x_float,labels -csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue test,BUG,new,2022-07-17 07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,tgp,10,2022-09-15 15:27:56,world,8,NULL -csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue test007,BUG,new,2022-08-12 13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15 15:27:56,abc,24590,hello worlds -csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,11,issue test011,REQUIREMENT,new,2022-08-10 13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,,1,2022-09-15 15:27:56,NULL,0.00014,NULL +csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue test,BUG,new,2022-07-17 07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,klesh,10,2022-09-15 15:27:56,world,8,NULL +csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue test007,BUG,new,2022-08-12 13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,warren,30,2022-09-15 15:27:56,abc,24590,hello worlds +csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,11,issue test011,REQUIREMENT,new,2022-08-10 13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,abeizn,1,2022-09-15 15:27:56,NULL,0.00014,NULL diff --git a/backend/plugins/customize/e2e/snapshot_tables/accounts.csv b/backend/plugins/customize/e2e/snapshot_tables/accounts.csv new file mode 100644 index 000000000..5d48240be --- /dev/null +++ b/backend/plugins/customize/e2e/snapshot_tables/accounts.csv @@ -0,0 +1,5 @@ +id,full_name,user_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +csv:CsvAccount:0:abeizn,abeizn,abeizn,csv-board,,0, +csv:CsvAccount:0:klesh,klesh,klesh,csv-board,,0, +csv:CsvAccount:0:tgp,tgp,tgp,csv-board2,,0, +csv:CsvAccount:0:warren,warren,warren,csv-board,,0, diff --git a/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv b/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv index 43d522071..ffcffa88f 100644 --- a/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv +++ b/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv @@ -1,7 +1,7 @@ id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,parent_issue_id,priority,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,severity,component,original_project,x_varchar,x_text,x_time,x_float,x_int,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,,1,issue test,,,,BUG,,new,0,,2022-07-17T07:15:55.959+00:00,,,,major,0,0,,,tgp,,tgp,,,,world,,2022-09-15T15:27:56.000+00:00,8,10,csv-board2,,, -csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,,10,issue title edited,,,,BUG,,new,0,,2022-08-12T13:43:00.783+00:00,,,,trivial,0,0,,,tgp,,tgp,,,,abc,,2022-09-15T15:27:56.000+00:00,24590,30,csv-board2,,, -csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,,11,issue test011,,,,REQUIREMENT,,new,0,,2022-08-10T13:44:46.508+00:00,,,,major,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,, -csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,,12,issue test012,,,,REQUIREMENT,,new,0,,2022-08-11T13:44:46.508+00:00,,,,major,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,, -csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,,13,issue test013,,,,REQUIREMENT,,new,0,,2022-08-12T13:44:46.508+00:00,,,,critical,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board2,,, -csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,,14,issue test014,,,,INCIDENT,,new,0,,2022-08-12T13:45:12.810+00:00,,,,blocker,0,0,,,tgp,,tgp,,,,,,2022-09-15T15:27:56.000+00:00,,41534568464351,csv-board2,,, +csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,,1,issue test,,,,BUG,,new,0,,2022-07-17T07:15:55.959+00:00,,,,major,0,0,,csv:CsvAccount:0:tgp,tgp,csv:CsvAccount:0:tgp,tgp,,,,world,,2022-09-15T15:27:56.000+00:00,8,10,csv-board2,,, +csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,,10,issue title edited,,,,BUG,,new,0,,2022-08-12T13:43:00.783+00:00,,,,trivial,0,0,,csv:CsvAccount:0:tgp,tgp,csv:CsvAccount:0:tgp,tgp,,,,abc,,2022-09-15T15:27:56.000+00:00,24590,30,csv-board2,,, +csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,,11,issue test011,,,,REQUIREMENT,,new,0,,2022-08-10T13:44:46.508+00:00,,,,major,0,0,,csv:CsvAccount:0:tgp,tgp,csv:CsvAccount:0:abeizn,abeizn,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,, +csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,,12,issue test012,,,,REQUIREMENT,,new,0,,2022-08-11T13:44:46.508+00:00,,,,major,0,0,,csv:CsvAccount:0:tgp,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,, +csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,,13,issue test013,,,,REQUIREMENT,,new,0,,2022-08-12T13:44:46.508+00:00,,,,critical,0,0,,csv:CsvAccount:0:tgp,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board2,,, +csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,,14,issue test014,,,,INCIDENT,,new,0,,2022-08-12T13:45:12.810+00:00,,,,blocker,0,0,,csv:CsvAccount:0:tgp,tgp,csv:CsvAccount:0:tgp,tgp,,,,,,2022-09-15T15:27:56.000+00:00,,41534568464351,csv-board2,,, diff --git a/backend/plugins/customize/service/service.go b/backend/plugins/customize/service/service.go index 6586dd13a..1d0d6f147 100644 --- a/backend/plugins/customize/service/service.go +++ b/backend/plugins/customize/service/service.go @@ -156,6 +156,7 @@ func (s *Service) getCustomizedFields(table string) ([]models.CustomizedField, e // issue could exist in multiple boards, so we should only delete an old records when it doesn't belong to another board func (s *Service) ImportIssue(boardId string, file io.ReadCloser, incremental bool) errors.Error { if !incremental { + // not delete accounts data since account may be referenced by others err := s.dal.Delete( &ticket.Issue{}, dal.Where("id IN (SELECT issue_id FROM board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE board_id!=?))", boardId, boardId), @@ -262,22 +263,53 @@ func (s *Service) importCSV(file io.ReadCloser, rawDataParams string, recordHand } } +// createOrUpdateAccount creates or updates an account based on the provided name. +// It returns the account ID and an error if any occurred. +func (s *Service) createOrUpdateAccount(accountName string, rawDataParams string) (string, errors.Error) { + if accountName == "" { + return "", nil // Return empty ID if name is empty, no error needed here. + } + now := time.Now() + accountId := fmt.Sprintf("csv:CsvAccount:0:%s", accountName) + account := &crossdomain.Account{ + DomainEntity: domainlayer.DomainEntity{ + Id: accountId, + NoPKModel: common.NoPKModel{ + RawDataOrigin: common.RawDataOrigin{ + RawDataParams: rawDataParams, + }, + }, + }, + FullName: accountName, + UserName: accountName, + CreatedDate: &now, + } + err := s.dal.CreateOrUpdate(account) + if err != nil { + return "", errors.Default.Wrap(err, fmt.Sprintf("failed to create or update account for %s", accountName)) + } + return accountId, nil +} + // issueHandlerFactory returns a handler that save record into `issues`, `board_issues` and `issue_labels` table func (s *Service) issueHandlerFactory(boardId string, incremental bool) func(record map[string]interface{}) errors.Error { return func(record map[string]interface{}) errors.Error { var err errors.Error var id string - if record["id"] == nil { + idValue, ok := record["id"] + if !ok || idValue == nil { return errors.Default.New("record without id") } - id, _ = record["id"].(string) - if id == "" { - return errors.Default.New("empty id") + id, ok = idValue.(string) + if !ok || id == "" { + return errors.Default.New("invalid or empty id") } - if record["labels"] != nil { - labels, ok := record["labels"].(string) + + // Handle labels + if labelsValue, ok := record["labels"]; ok && labelsValue != nil { + labels, ok := labelsValue.(string) if !ok { - return errors.Default.New("labels is not string") + return errors.Default.New("labels field is not a string") } var issueLabels []*ticket.IssueLabel appearedLabels := make(map[string]struct{}) // record the labels that have appeared @@ -307,15 +339,66 @@ func (s *Service) issueHandlerFactory(boardId string, incremental bool) func(rec } } } - delete(record, "labels") + delete(record, "labels") // Remove labels from record map as it's handled + + // Handle creator and assignee accounts + rawDataParamsValue, ok := record["_raw_data_params"] + if !ok { + // This should ideally not happen as it's set in importCSV, but good to check + return errors.Default.New("_raw_data_params missing from record") + } + rawDataParams, ok := rawDataParamsValue.(string) + if !ok { + return errors.Default.New("_raw_data_params is not a string") + } + + // Handle creator + var creatorName string + if creatorNameValue, ok := record["creator_name"]; ok && creatorNameValue != nil { + creatorName, ok = creatorNameValue.(string) + if !ok { + return errors.Default.New("creator_name is not a string") + } + } + creatorId, err := s.createOrUpdateAccount(creatorName, rawDataParams) + if err != nil { + return err + } + if creatorId != "" { + record["creator_id"] = creatorId + } + + // Handle assignee + var assigneeName string + if assigneeNameValue, ok := record["assignee_name"]; ok && assigneeNameValue != nil { + assigneeName, ok = assigneeNameValue.(string) + if !ok { + return errors.Default.New("assignee_name is not a string") + } + } + assigneeId, err := s.createOrUpdateAccount(assigneeName, rawDataParams) + if err != nil { + return err + } + if assigneeId != "" { + record["assignee_id"] = assigneeId + } + + // Handle issues err = s.dal.CreateWithMap(&ticket.Issue{}, record) if err != nil { return err } - return s.dal.CreateOrUpdate(&ticket.BoardIssue{ + + // Handle board_issues + err = s.dal.CreateOrUpdate(&ticket.BoardIssue{ BoardId: boardId, IssueId: id, }) + if err != nil { + return err + } + return nil } }
