This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new f0be93c70 fix: circleci date fields are inconsistent (#7757)
f0be93c70 is described below
commit f0be93c70be4db3e83115a4d649e4b86c6e4bd23
Author: Klesh Wong <[email protected]>
AuthorDate: Wed Jul 17 22:03:32 2024 +0800
fix: circleci date fields are inconsistent (#7757)
* fix: circleci date fields are inconsistent
* fix: e2e test
* fix: migration script versio
---
backend/helpers/e2ehelper/data_flow_tester.go | 7 +++
backend/plugins/circleci/e2e/pipeline_test.go | 2 +-
.../snapshot_tables/_tool_circleci_pipelines.csv | 22 +++++-----
.../snapshot_tables/_tool_circleci_workflows.csv | 20 ++++-----
.../20240717_add_createddate_to_workflow.go | 51 ++++++++++++++++++++++
.../circleci/models/migrationscripts/register.go | 1 +
backend/plugins/circleci/models/pipeline.go | 16 +++----
backend/plugins/circleci/models/workflow.go | 4 +-
.../plugins/circleci/tasks/workflow_converter.go | 4 +-
.../plugins/circleci/tasks/workflow_extractor.go | 7 +--
10 files changed, 97 insertions(+), 37 deletions(-)
diff --git a/backend/helpers/e2ehelper/data_flow_tester.go
b/backend/helpers/e2ehelper/data_flow_tester.go
index 831b859b8..b409fd360 100644
--- a/backend/helpers/e2ehelper/data_flow_tester.go
+++ b/backend/helpers/e2ehelper/data_flow_tester.go
@@ -319,6 +319,13 @@ func (t *DataFlowTester) CreateSnapshot(dst schema.Tabler,
opts TableOptions) {
}
allFields := append(pkColumnNames, targetFields...)
allFields = utils.StringsUniq(allFields)
+ for i, field := range allFields {
+ if t.Dal.Dialect() == "mysql" {
+ allFields[i] = fmt.Sprintf("`%s`", field)
+ } else {
+ allFields[i] = fmt.Sprintf(`"%s"`, field)
+ }
+ }
dbCursor, err := t.Dal.Cursor(
dal.Select(strings.Join(allFields, `,`)),
dal.From(dst.TableName()),
diff --git a/backend/plugins/circleci/e2e/pipeline_test.go
b/backend/plugins/circleci/e2e/pipeline_test.go
index 28d65c07c..006ce48bc 100644
--- a/backend/plugins/circleci/e2e/pipeline_test.go
+++ b/backend/plugins/circleci/e2e/pipeline_test.go
@@ -50,7 +50,7 @@ func TestCircleciPipeline(t *testing.T) {
e2ehelper.TableOptions{
CSVRelPath:
"./snapshot_tables/_tool_circleci_pipelines.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
- IgnoreFields: []string{"started_at", "stopped_at"},
+ IgnoreFields: []string{"stopped_date"},
},
)
}
diff --git
a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_pipelines.csv
b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_pipelines.csv
index dd01015be..ac203f861 100644
--- a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_pipelines.csv
+++ b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_pipelines.csv
@@ -1,11 +1,11 @@
-connection_id,id,project_slug,errors,updated_at,number,trigger_parameters,state,created_at,trigger,vcs,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,23622ee4-e150-4920-9d66-81533fa765a4,github/coldgust/coldgust.github.io,[],2023-03-25
17:52:15.651,5,,created,2023-03-25
17:52:15.651,"{""type"":""webhook"",""received_at"":""2023-03-25T17:52:15.446Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revisio
[...]
-1,2c45280f-7fb3-4025-b703-a547c4a94916,github/coldgust/coldgust.github.io,[],2023-03-25
17:50:20.316,4,,created,2023-03-25
17:50:20.316,"{""type"":""webhook"",""received_at"":""2023-03-25T17:50:20.150Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revisio
[...]
-1,625ca634-68fe-4515-91f0-7ba8af51dc99,github/coldgust/coldgust.github.io,[],2023-03-25
17:39:23.233,3,,created,2023-03-25
17:39:23.233,"{""type"":""webhook"",""received_at"":""2023-03-25T17:39:22.921Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review_url";
[...]
-1,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,github/coldgust/coldgust.github.io,[],2023-03-25
17:54:08.987,6,,created,2023-03-25
17:54:08.987,"{""type"":""webhook"",""received_at"":""2023-03-25T17:54:08.742Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revisio
[...]
-1,7fcc1623-edcc-4a76-ad20-cd81aa83519f,github/coldgust/coldgust.github.io,[],2023-03-25
18:13:21.590,9,,created,2023-03-25
18:13:21.590,"{""type"":""webhook"",""received_at"":""2023-03-25T18:13:21.404Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revisio
[...]
-1,866e967d-f826-4470-aed6-fc0c92e98703,github/coldgust/coldgust.github.io,[],2023-03-25
17:56:24.409,7,,created,2023-03-25
17:56:24.409,"{""type"":""webhook"",""received_at"":""2023-03-25T17:56:24.245Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revisio
[...]
-1,87aad008-1ad5-486a-8174-fdeed846561a,github/coldgust/coldgust.github.io,[],2023-03-25
17:12:18.747,2,,created,2023-03-25
17:12:18.747,"{""type"":""webhook"",""received_at"":""2023-03-25T17:12:18.624Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review_url";
[...]
-1,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,github/coldgust/coldgust.github.io,[],2023-03-25
18:06:13.404,8,,created,2023-03-25
18:06:13.404,"{""type"":""webhook"",""received_at"":""2023-03-25T18:06:13.246Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revisio
[...]
-1,afef32b3-5ffe-48d2-8d9e-46dcedd82554,github/coldgust/coldgust.github.io,[],2023-03-25
17:12:18.398,1,,created,2023-03-25
17:12:18.398,"{""type"":""api"",""received_at"":""2023-03-25T17:12:16.438Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review_url"":"";
[...]
-1,d323f088-02fa-4ed5-9696-fc2f89a27150,github/coldgust/coldgust.github.io,"[{""type"":""config-fetch"",""message"":""Failed
to fetch config.yml file.""}]",2023-03-25
18:13:43.446,10,"{""circleci"":{""org_name"":""coldgust"",""project_name"":""coldgust.github.io"",""provider_name"":""github""},""git"":{""checkout_sha"":""6ba0cdaf7e7791c5bc5aa89a13b9cd184f8ad296""}}",errored,2023-03-25
18:13:43.446,"{""type"":""Decoupled Ingestion
System"",""received_at"":""1970-01-01T00:00:00.000Z"",""act [...]
+connection_id,id,project_slug,updated_date,created_date,number,trigger_parameters,state,trigger,vcs
+1,23622ee4-e150-4920-9d66-81533fa765a4,github/coldgust/coldgust.github.io,2023-03-25T17:52:15.651+00:00,2023-03-25T17:52:15.651+00:00,5,,created,"{""type"":""webhook"",""received_at"":""2023-03-25T17:52:15.446Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""";,
[...]
+1,2c45280f-7fb3-4025-b703-a547c4a94916,github/coldgust/coldgust.github.io,2023-03-25T17:50:20.316+00:00,2023-03-25T17:50:20.316+00:00,4,,created,"{""type"":""webhook"",""received_at"":""2023-03-25T17:50:20.150Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""";,
[...]
+1,625ca634-68fe-4515-91f0-7ba8af51dc99,github/coldgust/coldgust.github.io,2023-03-25T17:39:23.233+00:00,2023-03-25T17:39:23.233+00:00,3,,created,"{""type"":""webhook"",""received_at"":""2023-03-25T17:39:22.921Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""re
[...]
+1,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,github/coldgust/coldgust.github.io,2023-03-25T17:54:08.987+00:00,2023-03-25T17:54:08.987+00:00,6,,created,"{""type"":""webhook"",""received_at"":""2023-03-25T17:54:08.742Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""";,
[...]
+1,7fcc1623-edcc-4a76-ad20-cd81aa83519f,github/coldgust/coldgust.github.io,2023-03-25T18:13:21.590+00:00,2023-03-25T18:13:21.590+00:00,9,,created,"{""type"":""webhook"",""received_at"":""2023-03-25T18:13:21.404Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""";,
[...]
+1,866e967d-f826-4470-aed6-fc0c92e98703,github/coldgust/coldgust.github.io,2023-03-25T17:56:24.409+00:00,2023-03-25T17:56:24.409+00:00,7,,created,"{""type"":""webhook"",""received_at"":""2023-03-25T17:56:24.245Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""";,
[...]
+1,87aad008-1ad5-486a-8174-fdeed846561a,github/coldgust/coldgust.github.io,2023-03-25T17:12:18.747+00:00,2023-03-25T17:12:18.747+00:00,2,,created,"{""type"":""webhook"",""received_at"":""2023-03-25T17:12:18.624Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""re
[...]
+1,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,github/coldgust/coldgust.github.io,2023-03-25T18:06:13.404+00:00,2023-03-25T18:06:13.404+00:00,8,,created,"{""type"":""webhook"",""received_at"":""2023-03-25T18:06:13.246Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""";,
[...]
+1,afef32b3-5ffe-48d2-8d9e-46dcedd82554,github/coldgust/coldgust.github.io,2023-03-25T17:12:18.398+00:00,2023-03-25T17:12:18.398+00:00,1,,created,"{""type"":""api"",""received_at"":""2023-03-25T17:12:16.438Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review
[...]
+1,d323f088-02fa-4ed5-9696-fc2f89a27150,github/coldgust/coldgust.github.io,2023-03-25T18:13:43.446+00:00,2023-03-25T18:13:43.446+00:00,10,"{""circleci"":{""org_name"":""coldgust"",""project_name"":""coldgust.github.io"",""provider_name"":""github""},""git"":{""checkout_sha"":""6ba0cdaf7e7791c5bc5aa89a13b9cd184f8ad296""}}",errored,"{""type"":""Decoupled
Ingestion
System"",""received_at"":""1970-01-01T00:00:00.000Z"",""actor"":{""login"":"""",""avatar_url"":""""}}","{""provider_name"":"""",
[...]
diff --git
a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv
b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv
index 0c43df3ea..2ca84d9a0 100644
--- a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv
+++ b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv
@@ -1,10 +1,10 @@
-connection_id,id,project_slug,pipeline_id,canceled_by,name,errored_by,tag,status,started_by,pipeline_number,created_at,stopped_at,duration_sec,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,6731159f-5275-4bfa-ba70-39d343d63814,github/coldgust/coldgust.github.io,23622ee4-e150-4920-9d66-81533fa765a4,"",workflow,"","",failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,5,2023-03-25
17:52:15,2023-03-25 17:52:23,8,2023-03-28
15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,19,""
-1,7370985a-9de3-4a47-acbc-e6a1fe8e5812,github/coldgust/coldgust.github.io,866e967d-f826-4470-aed6-fc0c92e98703,"",workflow,"","",failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,7,2023-03-25
17:56:24,2023-03-25 17:56:43,19,2023-03-28
15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,24,""
-1,89054eb2-8e85-4f5c-9a93-66d753a0e970,github/coldgust/coldgust.github.io,625ca634-68fe-4515-91f0-7ba8af51dc99,"",say-hello-workflow,"","",success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,3,2023-03-25
17:39:23,2023-03-25 17:39:28,5,2023-03-28
15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,21,""
-1,8971a56b-5547-4824-94dd-07bb467524c5,github/coldgust/coldgust.github.io,87aad008-1ad5-486a-8174-fdeed846561a,"",say-hello-workflow,"","",success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,2,2023-03-25
17:12:18,2023-03-25 17:12:23,5,2023-03-28
15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,23,""
-1,8fe60291-68f7-40e2-acec-d99bff4da713,github/coldgust/coldgust.github.io,afef32b3-5ffe-48d2-8d9e-46dcedd82554,"",say-hello-workflow,"","",success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,1,2023-03-25
17:12:18,2023-03-25 17:12:22,4,2023-03-28
15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,25,""
-1,b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,github/coldgust/coldgust.github.io,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,"",workflow,"","",failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,6,2023-03-25
17:54:09,2023-03-25 17:54:23,14,2023-03-28
15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,26,""
-1,c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,github/coldgust/coldgust.github.io,2c45280f-7fb3-4025-b703-a547c4a94916,"",workflow,"","",failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,4,2023-03-25
17:50:20,2023-03-25 17:50:25,5,2023-03-28
15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,20,""
-1,fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,github/coldgust/coldgust.github.io,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,"",workflow,"","",failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,8,2023-03-25
18:06:13,2023-03-25 18:06:28,15,2023-03-28
15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,27,""
-1,fd0bd4f5-264f-4e3c-a151-06153c018f78,github/coldgust/coldgust.github.io,7fcc1623-edcc-4a76-ad20-cd81aa83519f,"",workflow,"","",success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,9,2023-03-25
18:13:21,2023-03-25 18:13:38,17,2023-03-28
15:39:58.345,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_workflows,22,""
+connection_id,id,project_slug,pipeline_id,canceled_by,name,errored_by,tag,status,started_by,pipeline_number,created_date,stopped_date,duration_sec
+1,6731159f-5275-4bfa-ba70-39d343d63814,github/coldgust/coldgust.github.io,23622ee4-e150-4920-9d66-81533fa765a4,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,5,2023-03-25T17:52:15.000+00:00,2023-03-25T17:52:23.000+00:00,8
+1,7370985a-9de3-4a47-acbc-e6a1fe8e5812,github/coldgust/coldgust.github.io,866e967d-f826-4470-aed6-fc0c92e98703,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,7,2023-03-25T17:56:24.000+00:00,2023-03-25T17:56:43.000+00:00,19
+1,89054eb2-8e85-4f5c-9a93-66d753a0e970,github/coldgust/coldgust.github.io,625ca634-68fe-4515-91f0-7ba8af51dc99,,say-hello-workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,3,2023-03-25T17:39:23.000+00:00,2023-03-25T17:39:28.000+00:00,5
+1,8971a56b-5547-4824-94dd-07bb467524c5,github/coldgust/coldgust.github.io,87aad008-1ad5-486a-8174-fdeed846561a,,say-hello-workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,2,2023-03-25T17:12:18.000+00:00,2023-03-25T17:12:23.000+00:00,5
+1,8fe60291-68f7-40e2-acec-d99bff4da713,github/coldgust/coldgust.github.io,afef32b3-5ffe-48d2-8d9e-46dcedd82554,,say-hello-workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,1,2023-03-25T17:12:18.000+00:00,2023-03-25T17:12:22.000+00:00,4
+1,b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,github/coldgust/coldgust.github.io,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,6,2023-03-25T17:54:09.000+00:00,2023-03-25T17:54:23.000+00:00,14
+1,c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,github/coldgust/coldgust.github.io,2c45280f-7fb3-4025-b703-a547c4a94916,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,4,2023-03-25T17:50:20.000+00:00,2023-03-25T17:50:25.000+00:00,5
+1,fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,github/coldgust/coldgust.github.io,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,8,2023-03-25T18:06:13.000+00:00,2023-03-25T18:06:28.000+00:00,15
+1,fd0bd4f5-264f-4e3c-a151-06153c018f78,github/coldgust/coldgust.github.io,7fcc1623-edcc-4a76-ad20-cd81aa83519f,,workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,9,2023-03-25T18:13:21.000+00:00,2023-03-25T18:13:38.000+00:00,17
diff --git
a/backend/plugins/circleci/models/migrationscripts/20240717_add_createddate_to_workflow.go
b/backend/plugins/circleci/models/migrationscripts/20240717_add_createddate_to_workflow.go
new file mode 100644
index 000000000..73f0a12a1
--- /dev/null
+++
b/backend/plugins/circleci/models/migrationscripts/20240717_add_createddate_to_workflow.go
@@ -0,0 +1,51 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+)
+
+type workflow20240717 struct {
+ CreatedDate *time.Time
+}
+
+func (workflow20240717) TableName() string {
+ return "_tool_circleci_workflows"
+}
+
+type addCreatedStoppedDateToWorkflow struct{}
+
+func (*addCreatedStoppedDateToWorkflow) Up(basicRes context.BasicRes)
errors.Error {
+ db := basicRes.GetDal()
+ if err := db.AutoMigrate(&workflow20240717{}); err != nil {
+ return err
+ }
+ return db.RenameColumn(workflow20240717{}.TableName(), "stopped_at",
"stopped_date")
+}
+
+func (*addCreatedStoppedDateToWorkflow) Version() uint64 {
+ return 20240717210714
+}
+
+func (*addCreatedStoppedDateToWorkflow) Name() string {
+ return "add created_date and stopped_date to _tool_circleci_workflows,
drop stopped_at column"
+}
diff --git a/backend/plugins/circleci/models/migrationscripts/register.go
b/backend/plugins/circleci/models/migrationscripts/register.go
index 5a23f27cc..0233b3b2b 100644
--- a/backend/plugins/circleci/models/migrationscripts/register.go
+++ b/backend/plugins/circleci/models/migrationscripts/register.go
@@ -24,5 +24,6 @@ func All() []plugin.MigrationScript {
return []plugin.MigrationScript{
new(addInitTables),
new(addFieldsToCircleciJob20231129),
+ new(addCreatedStoppedDateToWorkflow),
}
}
diff --git a/backend/plugins/circleci/models/pipeline.go
b/backend/plugins/circleci/models/pipeline.go
index a18cded10..80cd4749c 100644
--- a/backend/plugins/circleci/models/pipeline.go
+++ b/backend/plugins/circleci/models/pipeline.go
@@ -46,14 +46,14 @@ type CircleciVcs struct {
}
type CircleciPipeline struct {
- ConnectionId uint64 `gorm:"primaryKey;type:BIGINT"
json:"connectionId" mapstructure:"connectionId"`
- Id string
`gorm:"primaryKey;type:varchar(100)" json:"id" mapstructure:"id"`
- ProjectSlug string `gorm:"type:varchar(255)"
json:"projectSlug" mapstructure:"projectSlug"`
- UpdatedDate *common.Iso8601Time `json:"updatedDate"
mapstructure:"updatedDate"`
- CreatedDate *common.Iso8601Time `json:"createdDate"
mapstructure:"createdDate"`
- Number int64 `json:"number"
mapstructure:"number"` // pipeline number within the project?
- TriggerParameters any `gorm:"serializer:json"
json:"trigger_parameters" mapstructure:"triggerParameters"`
- State string `gorm:"type:varchar(100)"
json:"state" mapstructure:"state"`
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT"
json:"connectionId"`
+ Id string
`gorm:"primaryKey;type:varchar(100)" json:"id"`
+ ProjectSlug string `gorm:"type:varchar(255)"
json:"project_slug"`
+ UpdatedDate *common.Iso8601Time `json:"updated_at"`
+ CreatedDate *common.Iso8601Time `json:"created_at"`
+ Number int64 `json:"number"` // pipeline
number within the project?
+ TriggerParameters any `gorm:"serializer:json"
json:"trigger_parameters"`
+ State string `gorm:"type:varchar(100)"
json:"state"`
Trigger CircleciTrigger `gorm:"serializer:json"
json:"trigger"`
Vcs CircleciVcs `gorm:"serializer:json"
json:"vcs"`
common.NoPKModel `swaggerignore:"true" json:"-" mapstructure:"-"`
diff --git a/backend/plugins/circleci/models/workflow.go
b/backend/plugins/circleci/models/workflow.go
index f4e5e9f71..c70ee1a77 100644
--- a/backend/plugins/circleci/models/workflow.go
+++ b/backend/plugins/circleci/models/workflow.go
@@ -33,8 +33,8 @@ type CircleciWorkflow struct {
Status string `gorm:"type:varchar(100)"
json:"status"`
StartedBy string `gorm:"type:varchar(100)"
json:"started_by"`
PipelineNumber int64 `json:"pipeline_number"`
- CreatedAt *common.Iso8601Time `json:"created_at"`
- StoppedAt *common.Iso8601Time `json:"stopped_at"`
+ CreatedDate *common.Iso8601Time `json:"created_at"`
+ StoppedDate *common.Iso8601Time `json:"stopped_at"`
DurationSec float64 `json:"duration_sec"`
common.NoPKModel `swaggerignore:"true" json:"-" mapstructure:"-"`
diff --git a/backend/plugins/circleci/tasks/workflow_converter.go
b/backend/plugins/circleci/tasks/workflow_converter.go
index 91806fad5..412bd4181 100644
--- a/backend/plugins/circleci/tasks/workflow_converter.go
+++ b/backend/plugins/circleci/tasks/workflow_converter.go
@@ -57,7 +57,7 @@ func ConvertWorkflows(taskCtx plugin.SubTaskContext)
errors.Error {
Input: cursor,
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
userTool := inputRow.(*models.CircleciWorkflow)
- createdAt := userTool.CreatedAt.ToTime()
+ createdAt := userTool.CreatedDate.ToTime()
pipeline := &devops.CICDPipeline{
DomainEntity: domainlayer.DomainEntity{
Id:
getPipelineIdGen().Generate(data.Options.ConnectionId, userTool.Id),
@@ -67,7 +67,7 @@ func ConvertWorkflows(taskCtx plugin.SubTaskContext)
errors.Error {
TaskDatesInfo: devops.TaskDatesInfo{
CreatedDate: createdAt,
StartedDate: &createdAt,
- FinishedDate:
userTool.StoppedAt.ToNullableTime(),
+ FinishedDate:
userTool.StoppedDate.ToNullableTime(),
},
CicdScopeId:
getProjectIdGen().Generate(data.Options.ConnectionId, userTool.ProjectSlug),
// reference:
https://circleci.com/docs/api/v2/index.html#operation/getWorkflowById
diff --git a/backend/plugins/circleci/tasks/workflow_extractor.go
b/backend/plugins/circleci/tasks/workflow_extractor.go
index ba0639bc8..37d22f307 100644
--- a/backend/plugins/circleci/tasks/workflow_extractor.go
+++ b/backend/plugins/circleci/tasks/workflow_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -48,9 +49,9 @@ func ExtractWorkflows(taskCtx plugin.SubTaskContext)
errors.Error {
toolL := userRes
toolL.ConnectionId = data.Options.ConnectionId
toolL.ProjectSlug = data.Options.ProjectSlug
- if userRes.CreatedAt != nil && userRes.StoppedAt != nil
{
- startTime := userRes.CreatedAt.ToTime()
- endTime := userRes.StoppedAt.ToTime()
+ if userRes.CreatedDate != nil && userRes.StoppedDate !=
nil {
+ startTime := userRes.CreatedDate.ToTime()
+ endTime := userRes.StoppedDate.ToTime()
toolL.DurationSec =
float64(endTime.Sub(startTime).Milliseconds() / 1e3)
}
return []interface{}{
