This is an automated email from the ASF dual-hosted git repository.
narro pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 1dfb12dfc 8216 featurecustomize add support for incremental csv upload
in the customize plugin (#8218)
1dfb12dfc is described below
commit 1dfb12dfcb7ce6aaaedfa44f1ac2694336c2623c
Author: NaRro <[email protected]>
AuthorDate: Tue Nov 26 14:30:38 2024 +0800
8216 featurecustomize add support for incremental csv upload in the
customize plugin (#8218)
* feat: support incremental import for
- /plugins/customize/csvfiles/issues.csv
- /plugins/customize/csvfiles/issue_repo_commits.csv
[Feature][Customize] Add Support for Incremental CSV Upload in the
Customize Plugin #8216
---
backend/plugins/customize/api/csv.go | 14 +++-
.../customize/e2e/import_issue_commits_test.go | 4 +-
.../e2e/import_issue_repo_commits_test.go | 31 +++++++--
.../plugins/customize/e2e/import_issues_test.go | 26 +++++--
.../customize/e2e/raw_tables/board_issues.csv | 16 +++++
.../e2e/raw_tables/issue_repo_commits.csv | 12 ----
...mits.csv => issue_repo_commits_incremental.csv} | 12 ----
.../e2e/raw_tables/issue_repo_commits_original.csv | 3 +
.../customize/e2e/raw_tables/issues_input.csv | 3 -
.../customize/e2e/raw_tables/issues_input2.csv | 4 +-
...sues_input2.csv => issues_input2_overwrite.csv} | 2 +-
...sues_input.csv => issues_input_incremental.csv} | 3 -
.../e2e/snapshot_tables/issue_commits.csv | 25 -------
.../issue_commits_from_import_issue_commit.csv | 25 +++++++
...issue_commits_from_import_issue_repo_commit.csv | 25 +++++++
.../customize/e2e/snapshot_tables/issue_labels.csv | 6 +-
.../e2e/snapshot_tables/issue_repo_commits.csv | 49 ++++++-------
.../e2e/snapshot_tables/issues_output.csv | 6 +-
backend/plugins/customize/service/service.go | 81 +++++++++++-----------
19 files changed, 205 insertions(+), 142 deletions(-)
diff --git a/backend/plugins/customize/api/csv.go
b/backend/plugins/customize/api/csv.go
index fa9ae053a..51ea626c8 100644
--- a/backend/plugins/customize/api/csv.go
+++ b/backend/plugins/customize/api/csv.go
@@ -34,6 +34,7 @@ const maxMemory = 32 << 20 // 32 MB
// @Accept multipart/form-data
// @Param boardId formData string true "the ID of the board"
// @Param boardName formData string true "the name of the board"
+// @Param incremental formData bool false "whether to import
incrementally"
// @Param file formData file true "select file to upload"
// @Produce json
// @Success 200
@@ -47,6 +48,10 @@ func (h *Handlers) ImportIssue(input
*plugin.ApiResourceInput) (*plugin.ApiResou
}
// nolint
defer file.Close()
+ incremental := false
+ if input.Request.FormValue("incremental") == "true" {
+ incremental = true
+ }
boardId := strings.TrimSpace(input.Request.FormValue("boardId"))
if boardId == "" {
return nil, errors.BadInput.New("empty boardId")
@@ -59,7 +64,7 @@ func (h *Handlers) ImportIssue(input
*plugin.ApiResourceInput) (*plugin.ApiResou
if err != nil {
return nil, err
}
- return nil, h.svc.ImportIssue(boardId, file)
+ return nil, h.svc.ImportIssue(boardId, file, incremental)
}
// ImportIssueCommit accepts a CSV file, parses and saves it to the database
@@ -94,6 +99,7 @@ func (h *Handlers) ImportIssueCommit(input
*plugin.ApiResourceInput) (*plugin.Ap
// @Tags plugins/customize
// @Accept multipart/form-data
// @Param boardId formData string true "the ID of the board"
+// @Param incremental formData bool false "whether to import
incrementally"
// @Param file formData file true "select file to upload"
// @Produce json
// @Success 200
@@ -111,7 +117,11 @@ func (h *Handlers) ImportIssueRepoCommit(input
*plugin.ApiResourceInput) (*plugi
if boardId == "" {
return nil, errors.Default.New("empty boardId")
}
- return nil, h.svc.ImportIssueRepoCommit(boardId, file)
+ incremental := false
+ if input.Request.FormValue("incremental") == "true" {
+ incremental = true
+ }
+ return nil, h.svc.ImportIssueRepoCommit(boardId, file, incremental)
}
func (h *Handlers) extractFile(input *plugin.ApiResourceInput) (io.ReadCloser,
errors.Error) {
diff --git a/backend/plugins/customize/e2e/import_issue_commits_test.go
b/backend/plugins/customize/e2e/import_issue_commits_test.go
index 3d1aeb64f..f997d81df 100644
--- a/backend/plugins/customize/e2e/import_issue_commits_test.go
+++ b/backend/plugins/customize/e2e/import_issue_commits_test.go
@@ -41,13 +41,13 @@ func TestImportIssueCommitDataFlow(t *testing.T) {
}
defer f.Close()
// import data
- err := svc.ImportIssueCommit(`{"ConnectionId":1,"BoardId":8}`, f)
+ err := svc.ImportIssueCommit(`csv-board`, f)
if err != nil {
t.Fatal(err)
}
dataflowTester.VerifyTableWithRawData(
crossdomain.IssueCommit{},
- "snapshot_tables/issue_commits.csv",
+ "snapshot_tables/issue_commits_from_import_issue_commit.csv",
[]string{
"issue_id",
"commit_sha",
diff --git a/backend/plugins/customize/e2e/import_issue_repo_commits_test.go
b/backend/plugins/customize/e2e/import_issue_repo_commits_test.go
index 2d6041b11..de1cca5ab 100644
--- a/backend/plugins/customize/e2e/import_issue_repo_commits_test.go
+++ b/backend/plugins/customize/e2e/import_issue_repo_commits_test.go
@@ -18,12 +18,14 @@ limitations under the License.
package e2e
import (
+ "os"
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/customize/impl"
"github.com/apache/incubator-devlake/plugins/customize/service"
- "os"
- "testing"
)
func TestImportIssueRepoCommitDataFlow(t *testing.T) {
@@ -33,18 +35,35 @@ func TestImportIssueRepoCommitDataFlow(t *testing.T) {
// create tables `issue_repo_commits` and `issue_commits`
dataflowTester.FlushTabler(&crossdomain.IssueRepoCommit{})
dataflowTester.FlushTabler(&crossdomain.IssueCommit{})
+ dataflowTester.FlushTabler(&ticket.BoardIssue{})
+
+
dataflowTester.ImportCsvIntoTabler("raw_tables/issue_repo_commits_original.csv",
&crossdomain.IssueRepoCommit{})
+ dataflowTester.ImportCsvIntoTabler("raw_tables/board_issues.csv",
&ticket.BoardIssue{})
+
svc := service.NewService(dataflowTester.Dal)
- f, err1 := os.Open("raw_tables/issue_repo_commits.csv")
+ issueRepoCommitsFile, err1 :=
os.Open("raw_tables/issue_repo_commits.csv")
if err1 != nil {
t.Fatal(err1)
}
- defer f.Close()
+ defer issueRepoCommitsFile.Close()
// import data
- err := svc.ImportIssueRepoCommit(`{"ConnectionId":1,"BoardId":8}`, f)
+ err := svc.ImportIssueRepoCommit("csv-board", issueRepoCommitsFile,
false)
if err != nil {
t.Fatal(err)
}
+
+ // import data incrementally
+ issueRepoCommitsIncrementalFile, err2 :=
os.Open("raw_tables/issue_repo_commits_incremental.csv")
+ if err2 != nil {
+ t.Fatal(err2)
+ }
+ defer issueRepoCommitsIncrementalFile.Close()
+ err = svc.ImportIssueRepoCommit("csv-board",
issueRepoCommitsIncrementalFile, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+
dataflowTester.VerifyTableWithRawData(
crossdomain.IssueRepoCommit{},
"snapshot_tables/issue_repo_commits.csv",
@@ -58,7 +77,7 @@ func TestImportIssueRepoCommitDataFlow(t *testing.T) {
})
dataflowTester.VerifyTableWithRawData(
crossdomain.IssueCommit{},
- "snapshot_tables/issue_commits.csv",
+
"snapshot_tables/issue_commits_from_import_issue_repo_commit.csv",
[]string{
"issue_id",
"commit_sha",
diff --git a/backend/plugins/customize/e2e/import_issues_test.go
b/backend/plugins/customize/e2e/import_issues_test.go
index 8ed878038..aa875eb92 100644
--- a/backend/plugins/customize/e2e/import_issues_test.go
+++ b/backend/plugins/customize/e2e/import_issues_test.go
@@ -89,21 +89,37 @@ func TestImportIssueDataFlow(t *testing.T) {
t.Fatal(err1)
}
defer issueFile.Close()
- err = svc.ImportIssue("csv-board", issueFile)
+ err = svc.ImportIssue("csv-board", issueFile, false)
if err != nil {
t.Fatal(err)
}
-
- issueFile2, err2 := os.Open("raw_tables/issues_input2.csv")
+ issueAppendToFile1, err2 :=
os.Open("raw_tables/issues_input_incremental.csv")
if err2 != nil {
t.Fatal(err2)
}
+ defer issueAppendToFile1.Close()
+ err = svc.ImportIssue("csv-board", issueAppendToFile1, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+ issueFile2, err3 := os.Open("raw_tables/issues_input2.csv")
+ if err3 != nil {
+ t.Fatal(err3)
+ }
defer issueFile2.Close()
- err = svc.ImportIssue("csv-board2", issueFile2)
+ err = svc.ImportIssue("csv-board2", issueFile2, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ issueToOverwriteFile2, err4 :=
os.Open("raw_tables/issues_input2_overwrite.csv")
+ if err4 != nil {
+ t.Fatal(err4)
+ }
+ defer issueToOverwriteFile2.Close()
+ err = svc.ImportIssue("csv-board2", issueToOverwriteFile2, false)
if err != nil {
t.Fatal(err)
}
-
dataflowTester.VerifyTableWithRawData(
ticket.Issue{},
"snapshot_tables/issues_output.csv",
diff --git a/backend/plugins/customize/e2e/raw_tables/board_issues.csv
b/backend/plugins/customize/e2e/raw_tables/board_issues.csv
new file mode 100644
index 000000000..c88c197ed
--- /dev/null
+++ b/backend/plugins/customize/e2e/raw_tables/board_issues.csv
@@ -0,0 +1,16 @@
+board_id,issue_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+csv-board,csv:1,,,0,
+csv-board,csv:10,,,0,
+csv-board,csv:11,,,0,
+csv-board,csv:12,,,0,
+csv-board,csv:13,,,0,
+csv-board,csv:14,,,0,
+csv-board2,jira:JiraIssue:1:10063,,,0,
+csv-board2,jira:JiraIssue:1:10064,,,0,
+csv-board2,jira:JiraIssue:1:10065,,,0,
+csv-board2,jira:JiraIssue:1:10066,,,0,
+csv-board2,jira:JiraIssue:1:10139,,,0,
+csv-board2,jira:JiraIssue:1:10145,,,0,
+csv-board2,jira:JiraIssue:1:10159,,,0,
+csv-board2,jira:JiraIssue:1:10202,,,0,
+csv-board2,jira:JiraIssue:1:10203,,,0,
diff --git a/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv
b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv
index b8a131460..477b47d7e 100644
--- a/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv
+++ b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv
@@ -11,15 +11,3 @@
csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/
csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira
csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira
csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira
-jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira
-jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake
-jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake
-jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo
-jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co
-jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira
-jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira
-jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira
-jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira
-jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira
-jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira
-jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira
diff --git a/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv
b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits_incremental.csv
similarity index 52%
copy from backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv
copy to
backend/plugins/customize/e2e/raw_tables/issue_repo_commits_incremental.csv
index b8a131460..187a87afd 100644
--- a/backend/plugins/customize/e2e/raw_tables/issue_repo_commits.csv
+++
b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits_incremental.csv
@@ -1,16 +1,4 @@
issue_id,repo_url,commit_sha,host,namespace,repo_name
-csv:1,https://example.com:8080/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira
-csv:10,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake
-csv:11,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake
-csv:12,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo
-csv:13,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co
-csv:13,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira
-csv:13,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira
-csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira
-csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira
-csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira
-csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira
-csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira
jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake
jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake
diff --git
a/backend/plugins/customize/e2e/raw_tables/issue_repo_commits_original.csv
b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits_original.csv
new file mode 100644
index 000000000..2e15ff01b
--- /dev/null
+++ b/backend/plugins/customize/e2e/raw_tables/issue_repo_commits_original.csv
@@ -0,0 +1,3 @@
+issue_id,repo_url,commit_sha,host,namespace,repo_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+csv:1,https://example.com:8080/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/commit_id_should_be_removed,commit_id_should_be_removed,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}";,,,
+jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/commit_id_should_be_remained,commit_id_should_be_remained,example.com,PROJECTNAME,ui_jira,,,,
diff --git a/backend/plugins/customize/e2e/raw_tables/issues_input.csv
b/backend/plugins/customize/e2e/raw_tables/issues_input.csv
index ac05acd76..32e72055f 100644
--- a/backend/plugins/customize/e2e/raw_tables/issues_input.csv
+++ b/backend/plugins/customize/e2e/raw_tables/issues_input.csv
@@ -2,6 +2,3 @@
id,url,issue_key,title,original_type,original_status,created_date,resolution_dat
csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue
test,BUG,new,2022-07-17
07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,tgp,10,2022-09-15
15:27:56,world,8,NULL
csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue
test007,BUG,new,2022-08-12
13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15
15:27:56,abc,24590,hello worlds
csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,11,issue
test011,REQUIREMENT,new,2022-08-10
13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,,1,2022-09-15
15:27:56,NULL,0.00014,NULL
-csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,12,issue
test012,REQUIREMENT,new,2022-08-11
13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,,1,2022-09-15
15:27:56,NULL,0.00014,NULL
-csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,13,issue
test013,REQUIREMENT,new,2022-08-12
13:44:46.508+00:00,NULL,0,critical,,0,0,,,tgp,,1,2022-09-15
15:27:56,NULL,0.00014,NULL
-csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,14,issue
test014,INCIDENT,new,2022-08-12
13:45:12.810+00:00,NULL,0,blocker,,0,0,,,tgp,tgp,41534568464351,2022-09-15
15:27:56,NULL,NULL,"label1,label2,label3"
diff --git a/backend/plugins/customize/e2e/raw_tables/issues_input2.csv
b/backend/plugins/customize/e2e/raw_tables/issues_input2.csv
index 215bc651d..b7b08fb77 100644
--- a/backend/plugins/customize/e2e/raw_tables/issues_input2.csv
+++ b/backend/plugins/customize/e2e/raw_tables/issues_input2.csv
@@ -1,3 +1,3 @@
id,url,issue_key,title,original_type,original_status,created_date,resolution_date,story_point,priority,severity,original_estimate_minutes,time_spent_minutes,component,epic_key,creator_name,assignee_name,x_int,x_time,x_varchar,x_float,labels
-csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue
test,BUG,new,2022-07-17
07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,tgp,10,2022-09-15
15:27:56,world,8,NULL
-csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue
test007,BUG,new,2022-08-12
13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15
15:27:56,abc,24590,hello worlds
+csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,13,issue
test013,REQUIREMENT,new,2022-08-12
13:44:46.508+00:00,NULL,0,critical,,0,0,,,tgp,,1,2022-09-15
15:27:56,NULL,0.00014,NULL
+csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,14,issue
test014,INCIDENT,new,2022-08-12
13:45:12.810+00:00,NULL,0,blocker,,0,0,,,tgp,tgp,41534568464351,2022-09-15
15:27:56,NULL,NULL,"label1,label2,label3"
diff --git a/backend/plugins/customize/e2e/raw_tables/issues_input2.csv
b/backend/plugins/customize/e2e/raw_tables/issues_input2_overwrite.csv
similarity index 76%
copy from backend/plugins/customize/e2e/raw_tables/issues_input2.csv
copy to backend/plugins/customize/e2e/raw_tables/issues_input2_overwrite.csv
index 215bc651d..549fefcae 100644
--- a/backend/plugins/customize/e2e/raw_tables/issues_input2.csv
+++ b/backend/plugins/customize/e2e/raw_tables/issues_input2_overwrite.csv
@@ -1,3 +1,3 @@
id,url,issue_key,title,original_type,original_status,created_date,resolution_date,story_point,priority,severity,original_estimate_minutes,time_spent_minutes,component,epic_key,creator_name,assignee_name,x_int,x_time,x_varchar,x_float,labels
csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue
test,BUG,new,2022-07-17
07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,tgp,10,2022-09-15
15:27:56,world,8,NULL
-csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue
test007,BUG,new,2022-08-12
13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15
15:27:56,abc,24590,hello worlds
+csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue
title edited,BUG,new,2022-08-12
13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15
15:27:56,abc,24590,hello worlds
diff --git a/backend/plugins/customize/e2e/raw_tables/issues_input.csv
b/backend/plugins/customize/e2e/raw_tables/issues_input_incremental.csv
similarity index 59%
copy from backend/plugins/customize/e2e/raw_tables/issues_input.csv
copy to backend/plugins/customize/e2e/raw_tables/issues_input_incremental.csv
index ac05acd76..71ca994db 100644
--- a/backend/plugins/customize/e2e/raw_tables/issues_input.csv
+++ b/backend/plugins/customize/e2e/raw_tables/issues_input_incremental.csv
@@ -1,7 +1,4 @@
id,url,issue_key,title,original_type,original_status,created_date,resolution_date,story_point,priority,severity,original_estimate_minutes,time_spent_minutes,component,epic_key,creator_name,assignee_name,x_int,x_time,x_varchar,x_float,labels
-csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,1,issue
test,BUG,new,2022-07-17
07:15:55.959+00:00,NULL,0,major,,0,0,,,tgp,tgp,10,2022-09-15
15:27:56,world,8,NULL
-csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,10,issue
test007,BUG,new,2022-08-12
13:43:00.783+00:00,NULL,0,trivial,,0,0,,,tgp,tgp,30,2022-09-15
15:27:56,abc,24590,hello worlds
-csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,11,issue
test011,REQUIREMENT,new,2022-08-10
13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,,1,2022-09-15
15:27:56,NULL,0.00014,NULL
csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,12,issue
test012,REQUIREMENT,new,2022-08-11
13:44:46.508+00:00,NULL,0,major,,0,0,,,tgp,,1,2022-09-15
15:27:56,NULL,0.00014,NULL
csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,13,issue
test013,REQUIREMENT,new,2022-08-12
13:44:46.508+00:00,NULL,0,critical,,0,0,,,tgp,,1,2022-09-15
15:27:56,NULL,0.00014,NULL
csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,14,issue
test014,INCIDENT,new,2022-08-12
13:45:12.810+00:00,NULL,0,blocker,,0,0,,,tgp,tgp,41534568464351,2022-09-15
15:27:56,NULL,NULL,"label1,label2,label3"
diff --git a/backend/plugins/customize/e2e/snapshot_tables/issue_commits.csv
b/backend/plugins/customize/e2e/snapshot_tables/issue_commits.csv
deleted file mode 100644
index 97b875af1..000000000
--- a/backend/plugins/customize/e2e/snapshot_tables/issue_commits.csv
+++ /dev/null
@@ -1,25 +0,0 @@
-issue_id,commit_sha,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-csv:1,8748a066cbaf67b15e86f2c636f9931347e987cf,"{""ConnectionId"":1,""BoardId"":8}",,,
-csv:10,abc0892edaee00dd7ee268dbee71620407a29bca,"{""ConnectionId"":1,""BoardId"":8}",,,
-csv:11,e6bde456807818c5c78d7b265964d6d48b653af6,"{""ConnectionId"":1,""BoardId"":8}",,,
-csv:12,8f91020bcf684c6ad07adfafa3d8a2f826686c42,"{""ConnectionId"":1,""BoardId"":8}",,,
-csv:13,07aa2ebed68e286dc51a7e0082031196a6135f74,"{""ConnectionId"":1,""BoardId"":8}",,,
-csv:13,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,"{""ConnectionId"":1,""BoardId"":8}",,,
-csv:13,8993c04249e9d549e8950daec86717548c53c423,"{""ConnectionId"":1,""BoardId"":8}",,,
-csv:14,0ab12c4d4064003602edceed900d1456b6209894,"{""ConnectionId"":1,""BoardId"":8}",,,
-csv:14,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,"{""ConnectionId"":1,""BoardId"":8}",,,
-csv:14,d28785ff09229ac9e3c6734f0c97466ab00eb4da,"{""ConnectionId"":1,""BoardId"":8}",,,
-csv:14,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,"{""ConnectionId"":1,""BoardId"":8}",,,
-csv:14,ef5ab26111744f65f5191b247767a473c70d6c95,"{""ConnectionId"":1,""BoardId"":8}",,,
-jira:JiraIssue:1:10063,8748a066cbaf67b15e86f2c636f9931347e987cf,"{""ConnectionId"":1,""BoardId"":8}",,,
-jira:JiraIssue:1:10064,abc0892edaee00dd7ee268dbee71620407a29bca,"{""ConnectionId"":1,""BoardId"":8}",,,
-jira:JiraIssue:1:10064,e6bde456807818c5c78d7b265964d6d48b653af6,"{""ConnectionId"":1,""BoardId"":8}",,,
-jira:JiraIssue:1:10065,8f91020bcf684c6ad07adfafa3d8a2f826686c42,"{""ConnectionId"":1,""BoardId"":8}",,,
-jira:JiraIssue:1:10066,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,"{""ConnectionId"":1,""BoardId"":8}",,,
-jira:JiraIssue:1:10139,8993c04249e9d549e8950daec86717548c53c423,"{""ConnectionId"":1,""BoardId"":8}",,,
-jira:JiraIssue:1:10145,07aa2ebed68e286dc51a7e0082031196a6135f74,"{""ConnectionId"":1,""BoardId"":8}",,,
-jira:JiraIssue:1:10145,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,"{""ConnectionId"":1,""BoardId"":8}",,,
-jira:JiraIssue:1:10145,ef5ab26111744f65f5191b247767a473c70d6c95,"{""ConnectionId"":1,""BoardId"":8}",,,
-jira:JiraIssue:1:10159,d28785ff09229ac9e3c6734f0c97466ab00eb4da,"{""ConnectionId"":1,""BoardId"":8}",,,
-jira:JiraIssue:1:10202,0ab12c4d4064003602edceed900d1456b6209894,"{""ConnectionId"":1,""BoardId"":8}",,,
-jira:JiraIssue:1:10203,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,"{""ConnectionId"":1,""BoardId"":8}",,,
diff --git
a/backend/plugins/customize/e2e/snapshot_tables/issue_commits_from_import_issue_commit.csv
b/backend/plugins/customize/e2e/snapshot_tables/issue_commits_from_import_issue_commit.csv
new file mode 100644
index 000000000..142fd78a0
--- /dev/null
+++
b/backend/plugins/customize/e2e/snapshot_tables/issue_commits_from_import_issue_commit.csv
@@ -0,0 +1,25 @@
+issue_id,commit_sha,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+csv:1,8748a066cbaf67b15e86f2c636f9931347e987cf,csv-board,,,
+csv:10,abc0892edaee00dd7ee268dbee71620407a29bca,csv-board,,,
+csv:11,e6bde456807818c5c78d7b265964d6d48b653af6,csv-board,,,
+csv:12,8f91020bcf684c6ad07adfafa3d8a2f826686c42,csv-board,,,
+csv:13,07aa2ebed68e286dc51a7e0082031196a6135f74,csv-board,,,
+csv:13,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,csv-board,,,
+csv:13,8993c04249e9d549e8950daec86717548c53c423,csv-board,,,
+csv:14,0ab12c4d4064003602edceed900d1456b6209894,csv-board,,,
+csv:14,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,csv-board,,,
+csv:14,d28785ff09229ac9e3c6734f0c97466ab00eb4da,csv-board,,,
+csv:14,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,csv-board,,,
+csv:14,ef5ab26111744f65f5191b247767a473c70d6c95,csv-board,,,
+jira:JiraIssue:1:10063,8748a066cbaf67b15e86f2c636f9931347e987cf,csv-board,,,
+jira:JiraIssue:1:10064,abc0892edaee00dd7ee268dbee71620407a29bca,csv-board,,,
+jira:JiraIssue:1:10064,e6bde456807818c5c78d7b265964d6d48b653af6,csv-board,,,
+jira:JiraIssue:1:10065,8f91020bcf684c6ad07adfafa3d8a2f826686c42,csv-board,,,
+jira:JiraIssue:1:10066,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,csv-board,,,
+jira:JiraIssue:1:10139,8993c04249e9d549e8950daec86717548c53c423,csv-board,,,
+jira:JiraIssue:1:10145,07aa2ebed68e286dc51a7e0082031196a6135f74,csv-board,,,
+jira:JiraIssue:1:10145,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,csv-board,,,
+jira:JiraIssue:1:10145,ef5ab26111744f65f5191b247767a473c70d6c95,csv-board,,,
+jira:JiraIssue:1:10159,d28785ff09229ac9e3c6734f0c97466ab00eb4da,csv-board,,,
+jira:JiraIssue:1:10202,0ab12c4d4064003602edceed900d1456b6209894,csv-board,,,
+jira:JiraIssue:1:10203,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,csv-board,,,
diff --git
a/backend/plugins/customize/e2e/snapshot_tables/issue_commits_from_import_issue_repo_commit.csv
b/backend/plugins/customize/e2e/snapshot_tables/issue_commits_from_import_issue_repo_commit.csv
new file mode 100644
index 000000000..142fd78a0
--- /dev/null
+++
b/backend/plugins/customize/e2e/snapshot_tables/issue_commits_from_import_issue_repo_commit.csv
@@ -0,0 +1,25 @@
+issue_id,commit_sha,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+csv:1,8748a066cbaf67b15e86f2c636f9931347e987cf,csv-board,,,
+csv:10,abc0892edaee00dd7ee268dbee71620407a29bca,csv-board,,,
+csv:11,e6bde456807818c5c78d7b265964d6d48b653af6,csv-board,,,
+csv:12,8f91020bcf684c6ad07adfafa3d8a2f826686c42,csv-board,,,
+csv:13,07aa2ebed68e286dc51a7e0082031196a6135f74,csv-board,,,
+csv:13,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,csv-board,,,
+csv:13,8993c04249e9d549e8950daec86717548c53c423,csv-board,,,
+csv:14,0ab12c4d4064003602edceed900d1456b6209894,csv-board,,,
+csv:14,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,csv-board,,,
+csv:14,d28785ff09229ac9e3c6734f0c97466ab00eb4da,csv-board,,,
+csv:14,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,csv-board,,,
+csv:14,ef5ab26111744f65f5191b247767a473c70d6c95,csv-board,,,
+jira:JiraIssue:1:10063,8748a066cbaf67b15e86f2c636f9931347e987cf,csv-board,,,
+jira:JiraIssue:1:10064,abc0892edaee00dd7ee268dbee71620407a29bca,csv-board,,,
+jira:JiraIssue:1:10064,e6bde456807818c5c78d7b265964d6d48b653af6,csv-board,,,
+jira:JiraIssue:1:10065,8f91020bcf684c6ad07adfafa3d8a2f826686c42,csv-board,,,
+jira:JiraIssue:1:10066,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,csv-board,,,
+jira:JiraIssue:1:10139,8993c04249e9d549e8950daec86717548c53c423,csv-board,,,
+jira:JiraIssue:1:10145,07aa2ebed68e286dc51a7e0082031196a6135f74,csv-board,,,
+jira:JiraIssue:1:10145,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,csv-board,,,
+jira:JiraIssue:1:10145,ef5ab26111744f65f5191b247767a473c70d6c95,csv-board,,,
+jira:JiraIssue:1:10159,d28785ff09229ac9e3c6734f0c97466ab00eb4da,csv-board,,,
+jira:JiraIssue:1:10202,0ab12c4d4064003602edceed900d1456b6209894,csv-board,,,
+jira:JiraIssue:1:10203,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,csv-board,,,
diff --git a/backend/plugins/customize/e2e/snapshot_tables/issue_labels.csv
b/backend/plugins/customize/e2e/snapshot_tables/issue_labels.csv
index d7523c026..72e9ce73a 100644
--- a/backend/plugins/customize/e2e/snapshot_tables/issue_labels.csv
+++ b/backend/plugins/customize/e2e/snapshot_tables/issue_labels.csv
@@ -1,5 +1,5 @@
issue_id,label_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
csv:10,hello worlds,csv-board2,,0,
-csv:14,label1,csv-board,,0,
-csv:14,label2,csv-board,,0,
-csv:14,label3,csv-board,,0,
+csv:14,label1,csv-board2,,0,
+csv:14,label2,csv-board2,,0,
+csv:14,label3,csv-board2,,0,
diff --git
a/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv
b/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv
index 311a93678..b9e43abf0 100644
--- a/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv
+++ b/backend/plugins/customize/e2e/snapshot_tables/issue_repo_commits.csv
@@ -1,25 +1,26 @@
issue_id,repo_url,commit_sha,host,namespace,repo_name,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-csv:1,https://example.com:8080/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}";,,,
-csv:10,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}";,,,
-csv:11,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}";,,,
-csv:12,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo,"{""ConnectionId"":1,""BoardId"":8}";,,,
-csv:13,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}";,,,
-csv:13,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}";,,,
-csv:13,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co,"{""ConnectionId"":1,""BoardId"":8}";,,,
-csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}";,,,
-csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}";,,,
-csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}";,,,
-csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}";,,,
-csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}";,,,
-jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}";,,,
-jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}";,,,
-jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake,"{""ConnectionId"":1,""BoardId"":8}";,,,
-jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo,"{""ConnectionId"":1,""BoardId"":8}";,,,
-jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co,"{""ConnectionId"":1,""BoardId"":8}";,,,
-jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}";,,,
-jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}";,,,
-jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}";,,,
-jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}";,,,
-jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}";,,,
-jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}";,,,
-jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira,"{""ConnectionId"":1,""BoardId"":8}";,,,
+csv:1,https://example.com:8080/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira,csv-board,,,
+csv:10,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake,csv-board,,,
+csv:11,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake,csv-board,,,
+csv:12,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo,csv-board,,,
+csv:13,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira,csv-board,,,
+csv:13,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira,csv-board,,,
+csv:13,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co,csv-board,,,
+csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira,csv-board,,,
+csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira,csv-board,,,
+csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira,csv-board,,,
+csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira,csv-board,,,
+csv:14,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira,csv-board,,,
+jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8748a066cbaf67b15e86f2c636f9931347e987cf,8748a066cbaf67b15e86f2c636f9931347e987cf,example.com,PROJECTNAME,ui_jira,csv-board,,,
+jira:JiraIssue:1:10063,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/commit_id_should_be_remained,commit_id_should_be_remained,example.com,PROJECTNAME,ui_jira,,,,
+jira:JiraIssue:1:10064,https://bitbucket.org/mynamespace/incubator-devlake/commits/abc0892edaee00dd7ee268dbee71620407a29bca,abc0892edaee00dd7ee268dbee71620407a29bca,bitbucket.org,mynamespace,incubator-devlake,csv-board,,,
+jira:JiraIssue:1:10064,https://github.com/apache/incubator-devlake/commit/e6bde456807818c5c78d7b265964d6d48b653af6,e6bde456807818c5c78d7b265964d6d48b653af6,github.com,apache,incubator-devlake,csv-board,,,
+jira:JiraIssue:1:10065,https://gitlab.com/namespace1/namespace2/myrepo/-/commit/8f91020bcf684c6ad07adfafa3d8a2f826686c42,8f91020bcf684c6ad07adfafa3d8a2f826686c42,gitlab.com,namespace1/namespace2,murepo,csv-board,,,
+jira:JiraIssue:1:10066,https://gitlab.com/meri.co/vdev.co/-/commit/0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,0dfe2e9ed88ad4e27f825d9b67d4d56ac983c5ef,gitlab.com,meri.co,vdev.co,csv-board,,,
+jira:JiraIssue:1:10139,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,8993c04249e9d549e8950daec86717548c53c423,example.com,PROJECTNAME,ui_jira,csv-board,,,
+jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/07aa2ebed68e286dc51a7e0082031196a6135f74,07aa2ebed68e286dc51a7e0082031196a6135f74,example.com,PROJECTNAME,ui_jira,csv-board,,,
+jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/8993c04249e9d549e8950daec86717548c53c423,ef5ab26111744f65f5191b247767a473c70d6c95,example.com,PROJECTNAME,ui_jira,csv-board,,,
+jira:JiraIssue:1:10145,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,d70d6687e06304d9b6e0cb32b3f8c0f0928400f7,example.com,PROJECTNAME,ui_jira,csv-board,,,
+jira:JiraIssue:1:10159,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/d28785ff09229ac9e3c6734f0c97466ab00eb4da,d28785ff09229ac9e3c6734f0c97466ab00eb4da,example.com,PROJECTNAME,ui_jira,csv-board,,,
+jira:JiraIssue:1:10202,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/0ab12c4d4064003602edceed900d1456b6209894,0ab12c4d4064003602edceed900d1456b6209894,example.com,PROJECTNAME,ui_jira,csv-board,,,
+jira:JiraIssue:1:10203,https://example.com/bitbucket/projects/PROJECTNAME/repos/ui_jira/commits/980e9fe7bc3e22a0409f7241a024eaf9c53680dd,980e9fe7bc3e22a0409f7241a024eaf9c53680dd,example.com,PROJECTNAME,ui_jira,csv-board,,,
diff --git a/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv
b/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv
index 4e3528e1a..43d522071 100644
--- a/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv
+++ b/backend/plugins/customize/e2e/snapshot_tables/issues_output.csv
@@ -1,7 +1,7 @@
id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,parent_issue_id,priority,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,severity,component,original_project,x_varchar,x_text,x_time,x_float,x_int,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
csv:1,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/1,,1,issue
test,,,,BUG,,new,0,,2022-07-17T07:15:55.959+00:00,,,,major,0,0,,,tgp,,tgp,,,,world,,2022-09-15T15:27:56.000+00:00,8,10,csv-board2,,,
-csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,,10,issue
test007,,,,BUG,,new,0,,2022-08-12T13:43:00.783+00:00,,,,trivial,0,0,,,tgp,,tgp,,,,abc,,2022-09-15T15:27:56.000+00:00,24590,30,csv-board2,,,
+csv:10,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/10,,10,issue
title
edited,,,,BUG,,new,0,,2022-08-12T13:43:00.783+00:00,,,,trivial,0,0,,,tgp,,tgp,,,,abc,,2022-09-15T15:27:56.000+00:00,24590,30,csv-board2,,,
csv:11,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/11,,11,issue
test011,,,,REQUIREMENT,,new,0,,2022-08-10T13:44:46.508+00:00,,,,major,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,,
csv:12,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/12,,12,issue
test012,,,,REQUIREMENT,,new,0,,2022-08-11T13:44:46.508+00:00,,,,major,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,,
-csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,,13,issue
test013,,,,REQUIREMENT,,new,0,,2022-08-12T13:44:46.508+00:00,,,,critical,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board,,,
-csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,,14,issue
test014,,,,INCIDENT,,new,0,,2022-08-12T13:45:12.810+00:00,,,,blocker,0,0,,,tgp,,tgp,,,,,,2022-09-15T15:27:56.000+00:00,,41534568464351,csv-board,,,
+csv:13,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/13,,13,issue
test013,,,,REQUIREMENT,,new,0,,2022-08-12T13:44:46.508+00:00,,,,critical,0,0,,,tgp,,,,,,,,2022-09-15T15:27:56.000+00:00,0.00014,1,csv-board2,,,
+csv:14,https://api.bitbucket.org/2.0/repositories/thenicetgp/lake/issues/14,,14,issue
test014,,,,INCIDENT,,new,0,,2022-08-12T13:45:12.810+00:00,,,,blocker,0,0,,,tgp,,tgp,,,,,,2022-09-15T15:27:56.000+00:00,,41534568464351,csv-board2,,,
diff --git a/backend/plugins/customize/service/service.go
b/backend/plugins/customize/service/service.go
index a5187c28f..6586dd13a 100644
--- a/backend/plugins/customize/service/service.go
+++ b/backend/plugins/customize/service/service.go
@@ -154,32 +154,33 @@ func (s *Service) getCustomizedFields(table string)
([]models.CustomizedField, e
// ImportIssue import csv file to the table `issues`, and create relations to
boards
// issue could exist in multiple boards, so we should only delete an old
records when it doesn't belong to another board
-func (s *Service) ImportIssue(boardId string, file io.ReadCloser) errors.Error
{
- err := s.dal.Delete(
- &ticket.Issue{},
- dal.Where("id IN (SELECT issue_id FROM board_issues WHERE
board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE
board_id!=?))", boardId, boardId),
- )
- if err != nil {
- return err
- }
+func (s *Service) ImportIssue(boardId string, file io.ReadCloser, incremental
bool) errors.Error {
+ if !incremental {
+ err := s.dal.Delete(
+ &ticket.Issue{},
+ dal.Where("id IN (SELECT issue_id FROM board_issues
WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE
board_id!=?))", boardId, boardId),
+ )
+ if err != nil {
+ return err
+ }
- err = s.dal.Delete(
- &ticket.IssueLabel{},
- dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE
board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE
board_id!=?))", boardId, boardId),
- )
- if err != nil {
- return err
- }
+ err = s.dal.Delete(
+ &ticket.IssueLabel{},
+ dal.Where("issue_id IN (SELECT issue_id FROM
board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM
board_issues WHERE board_id!=?))", boardId, boardId),
+ )
+ if err != nil {
+ return err
+ }
- err = s.dal.Delete(
- &ticket.BoardIssue{},
- dal.Where("board_id = ?", boardId),
- )
- if err != nil {
- return err
+ err = s.dal.Delete(
+ &ticket.BoardIssue{},
+ dal.Where("board_id = ?", boardId),
+ )
+ if err != nil {
+ return err
+ }
}
-
- return s.importCSV(file, boardId, s.issueHandlerFactory(boardId))
+ return s.importCSV(file, boardId, s.issueHandlerFactory(boardId,
incremental))
}
// SaveBoard make sure the board exists in table `boards`
@@ -206,21 +207,23 @@ func (s *Service) ImportIssueCommit(boardId string, file
io.ReadCloser) errors.E
}
// ImportIssueRepoCommit imports data to the table `issue_repo_commits` and
`issue_commits`
-func (s *Service) ImportIssueRepoCommit(boardId string, file io.ReadCloser)
errors.Error {
- // delete old records of the table `issue_repo_commit` and
`issue_commit`
- err := s.dal.Delete(
- &crossdomain.IssueRepoCommit{},
- dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE
board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE
board_id!=?))", boardId, boardId),
- )
- if err != nil {
- return err
- }
- err = s.dal.Delete(
- &crossdomain.IssueCommit{},
- dal.Where("issue_id IN (SELECT issue_id FROM board_issues WHERE
board_id=? AND issue_id NOT IN (SELECT issue_id FROM board_issues WHERE
board_id!=?))", boardId, boardId),
- )
- if err != nil {
- return err
+func (s *Service) ImportIssueRepoCommit(boardId string, file io.ReadCloser,
incremental bool) errors.Error {
+ if !incremental {
+ // delete old records of the table `issue_repo_commit` and
`issue_commit`
+ err := s.dal.Delete(
+ &crossdomain.IssueRepoCommit{},
+ dal.Where("issue_id IN (SELECT issue_id FROM
board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM
board_issues WHERE board_id!=?))", boardId, boardId),
+ )
+ if err != nil {
+ return err
+ }
+ err = s.dal.Delete(
+ &crossdomain.IssueCommit{},
+ dal.Where("issue_id IN (SELECT issue_id FROM
board_issues WHERE board_id=? AND issue_id NOT IN (SELECT issue_id FROM
board_issues WHERE board_id!=?))", boardId, boardId),
+ )
+ if err != nil {
+ return err
+ }
}
return s.importCSV(file, boardId, s.issueRepoCommitHandler)
}
@@ -260,7 +263,7 @@ func (s *Service) importCSV(file io.ReadCloser,
rawDataParams string, recordHand
}
// issueHandlerFactory returns a handler that save record into `issues`,
`board_issues` and `issue_labels` table
-func (s *Service) issueHandlerFactory(boardId string) func(record
map[string]interface{}) errors.Error {
+func (s *Service) issueHandlerFactory(boardId string, incremental bool)
func(record map[string]interface{}) errors.Error {
return func(record map[string]interface{}) errors.Error {
var err errors.Error
var id string
