This is an automated email from the ASF dual-hosted git repository. warren pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 65fa50514d03fcb4e24bfbda7251a374fe9f9a53 Author: abeizn <[email protected]> AuthorDate: Thu Jun 16 09:32:47 2022 +0800 feat: jenkins add e2e test for jobs and builds --- plugins/jenkins/e2e/builds_test.go | 86 ++++++++++++++++++++++ plugins/jenkins/e2e/jobs_test.go | 81 ++++++++++++++++++++ .../jenkins/e2e/tables/_raw_jenkins_api_builds.csv | 21 ++++++ .../jenkins/e2e/tables/_raw_jenkins_api_jobs.csv | 12 +++ .../jenkins/e2e/tables/_tool_jenkins_builds.csv | 21 ++++++ plugins/jenkins/e2e/tables/_tool_jenkins_jobs.csv | 12 +++ plugins/jenkins/e2e/tables/builds.csv | 12 +++ plugins/jenkins/e2e/tables/jobs.csv | 12 +++ plugins/jenkins/tasks/build_collector.go | 2 +- plugins/jenkins/tasks/build_convertor.go | 2 +- plugins/jenkins/tasks/build_extractor.go | 4 +- plugins/jenkins/tasks/job_collector.go | 2 +- plugins/jenkins/tasks/job_convertor.go | 2 +- plugins/jenkins/tasks/job_extractor.go | 4 +- 14 files changed, 265 insertions(+), 8 deletions(-) diff --git a/plugins/jenkins/e2e/builds_test.go b/plugins/jenkins/e2e/builds_test.go new file mode 100644 index 00000000..24c92b5b --- /dev/null +++ b/plugins/jenkins/e2e/builds_test.go @@ -0,0 +1,86 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/models/domainlayer/devops" + + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/jenkins/impl" + "github.com/apache/incubator-devlake/plugins/jenkins/models" + "github.com/apache/incubator-devlake/plugins/jenkins/tasks" +) + +func TestJenkinsBuildsDataFlow(t *testing.T) { + + var jenkins impl.Jenkins + dataflowTester := e2ehelper.NewDataFlowTester(t, "jenkins", jenkins) + + taskData := &tasks.JenkinsTaskData{ + Options: &tasks.JenkinsOptions{ + ConnectionId: 1, + }, + } + + // import raw data table + dataflowTester.ImportCsvIntoRawTable("./tables/_raw_jenkins_api_builds.csv", "_raw_jenkins_api_builds") + + // verify extraction + dataflowTester.FlushTabler(&models.JenkinsBuild{}) + dataflowTester.Subtask(tasks.ExtractApiBuildsMeta, taskData) + dataflowTester.VerifyTable( + models.JenkinsBuild{}, + "tables/_tool_jenkins_builds.csv", + []string{"connection_id", "job_name", "number"}, + []string{ + "_raw_data_params", + "_raw_data_table", + "_raw_data_id", + "_raw_data_remark", + "connection_id", + "job_name", + "duration", + "display_name", + "estimated_duration", + "number", + "result", + "timestamp", + "start_time", + "commit_sha", + }, + ) + + // verify conversion + dataflowTester.FlushTabler(&devops.Build{}) + dataflowTester.Subtask(tasks.ConvertBuildsMeta, taskData) + dataflowTester.VerifyTable( + devops.Job{}, + "tables/builds.csv", + []string{"id"}, + []string{ + "id", + "_raw_data_params", + "_raw_data_table", + "_raw_data_id", + "_raw_data_remark", + "name", + }, + ) +} diff --git a/plugins/jenkins/e2e/jobs_test.go b/plugins/jenkins/e2e/jobs_test.go new file mode 100644 index 00000000..267e71ad --- /dev/null +++ b/plugins/jenkins/e2e/jobs_test.go @@ -0,0 +1,81 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/models/domainlayer/devops" + + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/jenkins/impl" + "github.com/apache/incubator-devlake/plugins/jenkins/models" + "github.com/apache/incubator-devlake/plugins/jenkins/tasks" +) + +func TestJenkinsJobsDataFlow(t *testing.T) { + + var jenkins impl.Jenkins + dataflowTester := e2ehelper.NewDataFlowTester(t, "jenkins", jenkins) + + taskData := &tasks.JenkinsTaskData{ + Options: &tasks.JenkinsOptions{ + ConnectionId: 1, + }, + } + + // import raw data table + dataflowTester.ImportCsvIntoRawTable("./tables/_raw_jenkins_api_jobs.csv", "_raw_jenkins_api_jobs") + + // verify extraction + dataflowTester.FlushTabler(&models.JenkinsJob{}) + dataflowTester.Subtask(tasks.ExtractApiJobsMeta, taskData) + dataflowTester.VerifyTable( + models.JenkinsJob{}, + "tables/_tool_jenkins_jobs.csv", + []string{"connection_id", "name"}, + []string{ + "connection_id", + "name", + "class", + "color", + "base", + "_raw_data_params", + "_raw_data_table", + "_raw_data_id", + "_raw_data_remark", + }, + ) + + // verify conversion + dataflowTester.FlushTabler(&devops.Job{}) + dataflowTester.Subtask(tasks.ConvertJobsMeta, taskData) + dataflowTester.VerifyTable( + devops.Job{}, + "tables/jobs.csv", + []string{"id"}, + []string{ + "name", + "id", + "_raw_data_params", + "_raw_data_table", + "_raw_data_id", + "_raw_data_remark", + }, + ) +} diff --git a/plugins/jenkins/e2e/tables/_raw_jenkins_api_builds.csv b/plugins/jenkins/e2e/tables/_raw_jenkins_api_builds.csv new file mode 100644 index 00000000..f5399bd1 --- /dev/null +++ b/plugins/jenkins/e2e/tables/_raw_jenkins_api_builds.csv @@ -0,0 +1,21 @@ +"id","params","data","url","input","created_at" +"1","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#205"",""duration"":843,""estimatedDuration"":845,""number"":205,""result"":""SUCCESS"",""timestamp"":1647841830657,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdurat [...] +"2","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#204"",""duration"":849,""estimatedDuration"":845,""number"":204,""result"":""SUCCESS"",""timestamp"":1645152359034,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdurat [...] +"3","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#203"",""duration"":823,""estimatedDuration"":845,""number"":203,""result"":""SUCCESS"",""timestamp"":1642560020873,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdurat [...] +"4","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#202"",""duration"":814,""estimatedDuration"":845,""number"":202,""result"":""SUCCESS"",""timestamp"":1641524412998,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdurat [...] +"5","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#201"",""duration"":872,""estimatedDuration"":845,""number"":201,""result"":""SUCCESS"",""timestamp"":1641286809958,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdurat [...] +"6","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#200"",""duration"":826,""estimatedDuration"":845,""number"":200,""result"":""SUCCESS"",""timestamp"":1638425697887,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdurat [...] +"7","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#199"",""duration"":711,""estimatedDuration"":845,""number"":199,""result"":""FAILURE"",""timestamp"":1637636904142,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdurat [...] +"8","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#198"",""duration"":855,""estimatedDuration"":845,""number"":198,""result"":""SUCCESS"",""timestamp"":1635842717468,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdurat [...] +"9","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#197"",""duration"":806,""estimatedDuration"":845,""number"":197,""result"":""SUCCESS"",""timestamp"":1635143830616,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdurat [...] +"10","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#196"",""duration"":840,""estimatedDuration"":845,""number"":196,""result"":""SUCCESS"",""timestamp"":1634713846944,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...] +"11","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#195"",""duration"":756,""estimatedDuration"":845,""number"":195,""result"":""SUCCESS"",""timestamp"":1634289009179,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...] +"12","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#194"",""duration"":735,""estimatedDuration"":845,""number"":194,""result"":""SUCCESS"",""timestamp"":1634111680194,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...] +"13","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#193"",""duration"":805,""estimatedDuration"":845,""number"":193,""result"":""SUCCESS"",""timestamp"":1634108932844,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...] +"14","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#192"",""duration"":776,""estimatedDuration"":845,""number"":192,""result"":""SUCCESS"",""timestamp"":1631935845537,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...] +"15","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#191"",""duration"":833,""estimatedDuration"":845,""number"":191,""result"":""SUCCESS"",""timestamp"":1631237966948,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...] +"16","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#190"",""duration"":819,""estimatedDuration"":845,""number"":190,""result"":""SUCCESS"",""timestamp"":1631000546032,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...] +"17","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#189"",""duration"":785,""estimatedDuration"":845,""number"":189,""result"":""SUCCESS"",""timestamp"":1630632234339,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...] +"18","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#188"",""duration"":752,""estimatedDuration"":845,""number"":188,""result"":""SUCCESS"",""timestamp"":1629169239734,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...] +"19","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#187"",""duration"":822,""estimatedDuration"":845,""number"":187,""result"":""SUCCESS"",""timestamp"":1628838712486,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...] +"20","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleBuild"",""actions"":[{""_class"":""hudson.model.ParametersAction""},{""_class"":""hudson.model.CauseAction""},{},{},{}],""displayName"":""#186"",""duration"":725,""estimatedDuration"":845,""number"":186,""result"":""SUCCESS"",""timestamp"":1628674732761,""changeSet"":{""_class"":""hudson.scm.EmptyChangeLogSet"",""kind"":null}}","https://jenkins.merico.cn/job/Create_License/api/json?tree=allBuilds%5Bnumber%2Ctimestamp%2Cdura [...] \ No newline at end of file diff --git a/plugins/jenkins/e2e/tables/_raw_jenkins_api_jobs.csv b/plugins/jenkins/e2e/tables/_raw_jenkins_api_jobs.csv new file mode 100644 index 00000000..b4e45e6a --- /dev/null +++ b/plugins/jenkins/e2e/tables/_raw_jenkins_api_jobs.csv @@ -0,0 +1,12 @@ +"id","params","data","url","input","created_at" +"1","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleProject"",""name"":""auto_test"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697" +"2","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleProject"",""name"":""auto_test_framework"",""color"":""disabled""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697" +"3","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleProject"",""name"":""build_devlake"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697" +"4","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleProject"",""name"":""Create_License"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697" +"5","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleProject"",""name"":""Create_License_new"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697" +"6","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleProject"",""name"":""Deploy k8s single_data"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697" +"7","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleProject"",""name"":""Deploy k8s single_new"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697" +"8","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleProject"",""name"":""Deploy k8s test"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697" +"9","{""ConnectionId"":1}","{""_class"":""hudson.model.FreeStyleProject"",""name"":""devlake_empty_build"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697" +"10","{""ConnectionId"":1}","{""_class"":""org.jenkinsci.plugins.workflow.job.WorkflowJob"",""name"":""test-platform-backend"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697" +"11","{""ConnectionId"":1}","{""_class"":""org.jenkinsci.plugins.workflow.job.WorkflowJob"",""name"":""test-platform-frontend"",""color"":""blue""}","https://jenkins.merico.cn/api/json?tree=jobs%5Bname%2Cclass%2Ccolor%2Cbase%5D%7B1%2C100%7D","null","2022-06-15 15:24:40.697" diff --git a/plugins/jenkins/e2e/tables/_tool_jenkins_builds.csv b/plugins/jenkins/e2e/tables/_tool_jenkins_builds.csv new file mode 100644 index 00000000..9a5c5a04 --- /dev/null +++ b/plugins/jenkins/e2e/tables/_tool_jenkins_builds.csv @@ -0,0 +1,21 @@ +connection_id,job_name,number,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,connection_id,job_name,duration,display_name,estimated_duration,number,result,timestamp,start_time,commit_sha +1,Create_License,186,"{""ConnectionId"":1}",_raw_jenkins_api_builds,20,,1,Create_License,725,#186,845,186,SUCCESS,1628674732761,2021-08-11T09:38:52.000+00:00, +1,Create_License,187,"{""ConnectionId"":1}",_raw_jenkins_api_builds,19,,1,Create_License,822,#187,845,187,SUCCESS,1628838712486,2021-08-13T07:11:52.000+00:00, +1,Create_License,188,"{""ConnectionId"":1}",_raw_jenkins_api_builds,18,,1,Create_License,752,#188,845,188,SUCCESS,1629169239734,2021-08-17T03:00:39.000+00:00, +1,Create_License,189,"{""ConnectionId"":1}",_raw_jenkins_api_builds,17,,1,Create_License,785,#189,845,189,SUCCESS,1630632234339,2021-09-03T01:23:54.000+00:00, +1,Create_License,190,"{""ConnectionId"":1}",_raw_jenkins_api_builds,16,,1,Create_License,819,#190,845,190,SUCCESS,1631000546032,2021-09-07T07:42:26.000+00:00, +1,Create_License,191,"{""ConnectionId"":1}",_raw_jenkins_api_builds,15,,1,Create_License,833,#191,845,191,SUCCESS,1631237966948,2021-09-10T01:39:26.000+00:00, +1,Create_License,192,"{""ConnectionId"":1}",_raw_jenkins_api_builds,14,,1,Create_License,776,#192,845,192,SUCCESS,1631935845537,2021-09-18T03:30:45.000+00:00, +1,Create_License,193,"{""ConnectionId"":1}",_raw_jenkins_api_builds,13,,1,Create_License,805,#193,845,193,SUCCESS,1634108932844,2021-10-13T07:08:52.000+00:00, +1,Create_License,194,"{""ConnectionId"":1}",_raw_jenkins_api_builds,12,,1,Create_License,735,#194,845,194,SUCCESS,1634111680194,2021-10-13T07:54:40.000+00:00, +1,Create_License,195,"{""ConnectionId"":1}",_raw_jenkins_api_builds,11,,1,Create_License,756,#195,845,195,SUCCESS,1634289009179,2021-10-15T09:10:09.000+00:00, +1,Create_License,196,"{""ConnectionId"":1}",_raw_jenkins_api_builds,10,,1,Create_License,840,#196,845,196,SUCCESS,1634713846944,2021-10-20T07:10:46.000+00:00, +1,Create_License,197,"{""ConnectionId"":1}",_raw_jenkins_api_builds,9,,1,Create_License,806,#197,845,197,SUCCESS,1635143830616,2021-10-25T06:37:10.000+00:00, +1,Create_License,198,"{""ConnectionId"":1}",_raw_jenkins_api_builds,8,,1,Create_License,855,#198,845,198,SUCCESS,1635842717468,2021-11-02T08:45:17.000+00:00, +1,Create_License,199,"{""ConnectionId"":1}",_raw_jenkins_api_builds,7,,1,Create_License,711,#199,845,199,FAILURE,1637636904142,2021-11-23T03:08:24.000+00:00, +1,Create_License,200,"{""ConnectionId"":1}",_raw_jenkins_api_builds,6,,1,Create_License,826,#200,845,200,SUCCESS,1638425697887,2021-12-02T06:14:57.000+00:00, +1,Create_License,201,"{""ConnectionId"":1}",_raw_jenkins_api_builds,5,,1,Create_License,872,#201,845,201,SUCCESS,1641286809958,2022-01-04T09:00:09.000+00:00, +1,Create_License,202,"{""ConnectionId"":1}",_raw_jenkins_api_builds,4,,1,Create_License,814,#202,845,202,SUCCESS,1641524412998,2022-01-07T03:00:12.000+00:00, +1,Create_License,203,"{""ConnectionId"":1}",_raw_jenkins_api_builds,3,,1,Create_License,823,#203,845,203,SUCCESS,1642560020873,2022-01-19T02:40:20.000+00:00, +1,Create_License,204,"{""ConnectionId"":1}",_raw_jenkins_api_builds,2,,1,Create_License,849,#204,845,204,SUCCESS,1645152359034,2022-02-18T02:45:59.000+00:00, +1,Create_License,205,"{""ConnectionId"":1}",_raw_jenkins_api_builds,1,,1,Create_License,843,#205,845,205,SUCCESS,1647841830657,2022-03-21T05:50:30.000+00:00, diff --git a/plugins/jenkins/e2e/tables/_tool_jenkins_jobs.csv b/plugins/jenkins/e2e/tables/_tool_jenkins_jobs.csv new file mode 100644 index 00000000..5d45f725 --- /dev/null +++ b/plugins/jenkins/e2e/tables/_tool_jenkins_jobs.csv @@ -0,0 +1,12 @@ +connection_id,name,name,class,color,base,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +1,auto_test,auto_test,hudson.model.FreeStyleProject,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,1, +1,auto_test_framework,auto_test_framework,hudson.model.FreeStyleProject,disabled,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,2, +1,build_devlake,build_devlake,hudson.model.FreeStyleProject,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,3, +1,Create_License,Create_License,hudson.model.FreeStyleProject,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,4, +1,Create_License_new,Create_License_new,hudson.model.FreeStyleProject,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,5, +1,Deploy k8s single_data,Deploy k8s single_data,hudson.model.FreeStyleProject,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,6, +1,Deploy k8s single_new,Deploy k8s single_new,hudson.model.FreeStyleProject,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,7, +1,Deploy k8s test,Deploy k8s test,hudson.model.FreeStyleProject,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,8, +1,devlake_empty_build,devlake_empty_build,hudson.model.FreeStyleProject,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,9, +1,test-platform-backend,test-platform-backend,org.jenkinsci.plugins.workflow.job.WorkflowJob,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,10, +1,test-platform-frontend,test-platform-frontend,org.jenkinsci.plugins.workflow.job.WorkflowJob,blue,,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,11, diff --git a/plugins/jenkins/e2e/tables/builds.csv b/plugins/jenkins/e2e/tables/builds.csv new file mode 100644 index 00000000..a26be590 --- /dev/null +++ b/plugins/jenkins/e2e/tables/builds.csv @@ -0,0 +1,12 @@ +id,id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,name +jenkins:JenkinsJob:1:auto_test,jenkins:JenkinsJob:1:auto_test,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,1,,auto_test +jenkins:JenkinsJob:1:auto_test_framework,jenkins:JenkinsJob:1:auto_test_framework,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,2,,auto_test_framework +jenkins:JenkinsJob:1:build_devlake,jenkins:JenkinsJob:1:build_devlake,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,3,,build_devlake +jenkins:JenkinsJob:1:Create_License,jenkins:JenkinsJob:1:Create_License,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,4,,Create_License +jenkins:JenkinsJob:1:Create_License_new,jenkins:JenkinsJob:1:Create_License_new,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,5,,Create_License_new +jenkins:JenkinsJob:1:Deploy k8s single_data,jenkins:JenkinsJob:1:Deploy k8s single_data,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,6,,Deploy k8s single_data +jenkins:JenkinsJob:1:Deploy k8s single_new,jenkins:JenkinsJob:1:Deploy k8s single_new,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,7,,Deploy k8s single_new +jenkins:JenkinsJob:1:Deploy k8s test,jenkins:JenkinsJob:1:Deploy k8s test,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,8,,Deploy k8s test +jenkins:JenkinsJob:1:devlake_empty_build,jenkins:JenkinsJob:1:devlake_empty_build,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,9,,devlake_empty_build +jenkins:JenkinsJob:1:test-platform-backend,jenkins:JenkinsJob:1:test-platform-backend,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,10,,test-platform-backend +jenkins:JenkinsJob:1:test-platform-frontend,jenkins:JenkinsJob:1:test-platform-frontend,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,11,,test-platform-frontend diff --git a/plugins/jenkins/e2e/tables/jobs.csv b/plugins/jenkins/e2e/tables/jobs.csv new file mode 100644 index 00000000..7348d739 --- /dev/null +++ b/plugins/jenkins/e2e/tables/jobs.csv @@ -0,0 +1,12 @@ +id,name,id,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +jenkins:JenkinsJob:1:auto_test,auto_test,jenkins:JenkinsJob:1:auto_test,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,1, +jenkins:JenkinsJob:1:auto_test_framework,auto_test_framework,jenkins:JenkinsJob:1:auto_test_framework,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,2, +jenkins:JenkinsJob:1:build_devlake,build_devlake,jenkins:JenkinsJob:1:build_devlake,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,3, +jenkins:JenkinsJob:1:Create_License,Create_License,jenkins:JenkinsJob:1:Create_License,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,4, +jenkins:JenkinsJob:1:Create_License_new,Create_License_new,jenkins:JenkinsJob:1:Create_License_new,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,5, +jenkins:JenkinsJob:1:Deploy k8s single_data,Deploy k8s single_data,jenkins:JenkinsJob:1:Deploy k8s single_data,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,6, +jenkins:JenkinsJob:1:Deploy k8s single_new,Deploy k8s single_new,jenkins:JenkinsJob:1:Deploy k8s single_new,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,7, +jenkins:JenkinsJob:1:Deploy k8s test,Deploy k8s test,jenkins:JenkinsJob:1:Deploy k8s test,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,8, +jenkins:JenkinsJob:1:devlake_empty_build,devlake_empty_build,jenkins:JenkinsJob:1:devlake_empty_build,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,9, +jenkins:JenkinsJob:1:test-platform-backend,test-platform-backend,jenkins:JenkinsJob:1:test-platform-backend,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,10, +jenkins:JenkinsJob:1:test-platform-frontend,test-platform-frontend,jenkins:JenkinsJob:1:test-platform-frontend,2022-06-16T00:58:39.909+00:00,2022-06-16T00:58:39.909+00:00,"{""ConnectionId"":1}",_raw_jenkins_api_jobs,11, diff --git a/plugins/jenkins/tasks/build_collector.go b/plugins/jenkins/tasks/build_collector.go index ca711333..8ce663fa 100644 --- a/plugins/jenkins/tasks/build_collector.go +++ b/plugins/jenkins/tasks/build_collector.go @@ -65,7 +65,7 @@ func CollectApiBuilds(taskCtx core.SubTaskContext) error { collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ Params: JenkinsApiParams{ - ConnectionId: data.Connection.ID, + ConnectionId: data.Options.ConnectionId, }, Ctx: taskCtx, Table: RAW_BUILD_TABLE, diff --git a/plugins/jenkins/tasks/build_convertor.go b/plugins/jenkins/tasks/build_convertor.go index 5f12ef47..cb160f69 100644 --- a/plugins/jenkins/tasks/build_convertor.go +++ b/plugins/jenkins/tasks/build_convertor.go @@ -58,7 +58,7 @@ func ConvertBuilds(taskCtx core.SubTaskContext) error { Input: cursor, RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ Params: JenkinsApiParams{ - ConnectionId: data.Connection.ID, + ConnectionId: data.Options.ConnectionId, }, Ctx: taskCtx, Table: RAW_BUILD_TABLE, diff --git a/plugins/jenkins/tasks/build_extractor.go b/plugins/jenkins/tasks/build_extractor.go index ac57728e..a3df36fb 100644 --- a/plugins/jenkins/tasks/build_extractor.go +++ b/plugins/jenkins/tasks/build_extractor.go @@ -41,7 +41,7 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) error { extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ Params: JenkinsApiParams{ - ConnectionId: data.Connection.ID, + ConnectionId: data.Options.ConnectionId, }, Ctx: taskCtx, /* @@ -68,7 +68,7 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) error { results := make([]interface{}, 0, 1) build := &models.JenkinsBuild{ - ConnectionId: data.Connection.ID, + ConnectionId: data.Options.ConnectionId, JobName: input.Name, Duration: body.Duration, DisplayName: body.DisplayName, diff --git a/plugins/jenkins/tasks/job_collector.go b/plugins/jenkins/tasks/job_collector.go index a1dcbb36..e608d63d 100644 --- a/plugins/jenkins/tasks/job_collector.go +++ b/plugins/jenkins/tasks/job_collector.go @@ -42,7 +42,7 @@ func CollectApiJobs(taskCtx core.SubTaskContext) error { collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ Params: JenkinsApiParams{ - ConnectionId: data.Connection.ID, + ConnectionId: data.Options.ConnectionId, }, Ctx: taskCtx, /* diff --git a/plugins/jenkins/tasks/job_convertor.go b/plugins/jenkins/tasks/job_convertor.go index 25461a2a..55d08098 100644 --- a/plugins/jenkins/tasks/job_convertor.go +++ b/plugins/jenkins/tasks/job_convertor.go @@ -57,7 +57,7 @@ func ConvertJobs(taskCtx core.SubTaskContext) error { Input: cursor, RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ Params: JenkinsApiParams{ - ConnectionId: data.Connection.ID, + ConnectionId: data.Options.ConnectionId, }, Ctx: taskCtx, Table: RAW_JOB_TABLE, diff --git a/plugins/jenkins/tasks/job_extractor.go b/plugins/jenkins/tasks/job_extractor.go index 762aa37d..18967eea 100644 --- a/plugins/jenkins/tasks/job_extractor.go +++ b/plugins/jenkins/tasks/job_extractor.go @@ -39,7 +39,7 @@ func ExtractApiJobs(taskCtx core.SubTaskContext) error { extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ Params: JenkinsApiParams{ - ConnectionId: data.Connection.ID, + ConnectionId: data.Options.ConnectionId, }, Ctx: taskCtx, /* @@ -61,7 +61,7 @@ func ExtractApiJobs(taskCtx core.SubTaskContext) error { job := &models.JenkinsJob{ JenkinsJobProps: models.JenkinsJobProps{ - ConnectionId: data.Connection.ID, + ConnectionId: data.Options.ConnectionId, Name: body.Name, Class: body.Class, Color: body.Color,
