This is an automated email from the ASF dual-hosted git repository. klesh pushed a commit to branch kw-7010-worklog-incomplete in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit ec7659b545b85620e7e9f82e951fda089ef90545 Author: Klesh Wong <[email protected]> AuthorDate: Tue May 14 15:46:11 2024 +0800 fix: jira worklog collector fetches only the 1st page --- backend/plugins/jira/models/issue.go | 1 + .../20240514_add_worklog_total_to_issue.go | 56 ++++++++++++++++++++++ .../jira/models/migrationscripts/register.go | 1 + backend/plugins/jira/tasks/apiv2models/issue.go | 3 ++ backend/plugins/jira/tasks/worklog_collector.go | 32 +++++-------- 5 files changed, 72 insertions(+), 21 deletions(-) diff --git a/backend/plugins/jira/models/issue.go b/backend/plugins/jira/models/issue.go index 489118831..d7c44edb6 100644 --- a/backend/plugins/jira/models/issue.go +++ b/backend/plugins/jira/models/issue.go @@ -64,6 +64,7 @@ type JiraIssue struct { StdStatus string `gorm:"type:varchar(255)"` Components string `gorm:"type:varchar(255)"` ChangelogTotal int + WorklogTotal int common.NoPKModel } diff --git a/backend/plugins/jira/models/migrationscripts/20240514_add_worklog_total_to_issue.go b/backend/plugins/jira/models/migrationscripts/20240514_add_worklog_total_to_issue.go new file mode 100644 index 000000000..8d32cbec0 --- /dev/null +++ b/backend/plugins/jira/models/migrationscripts/20240514_add_worklog_total_to_issue.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" +) + +type jiraIssue20240514 struct { + WorklogTotal int +} + +func (jiraIssue20240514) TableName() string { + return "_tool_jira_issues" +} + +type addWorklogToIssue struct{} + +func (script *addWorklogToIssue) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + err := db.AutoMigrate(&jiraIssue20240514{}) + if err != nil { + return err + } + // force full issue extraction so issue.worklog_total can be updated + err = db.Exec("DELETE FROM _devlake_subtask_states WHERE plugin = ? AND subtask = ?", "jira", "extractIssues") + if err != nil { + return err + } + // force full collection for all jira worklogs + return db.Exec("DELETE FROM _devlake_collector_latest_state WHERE raw_data_table = ?", "_raw_jira_api_worklogs") +} + +func (*addWorklogToIssue) Version() uint64 { + return 20240514145131 +} + +func (*addWorklogToIssue) Name() string { + return "add worklog to _tool_jira_issues" +} diff --git a/backend/plugins/jira/models/migrationscripts/register.go b/backend/plugins/jira/models/migrationscripts/register.go index b4c410c9f..1a8f8367b 100644 --- a/backend/plugins/jira/models/migrationscripts/register.go +++ b/backend/plugins/jira/models/migrationscripts/register.go @@ -47,5 +47,6 @@ func All() []plugin.MigrationScript { new(modifyIssueRelationship), new(addComponents20230412), new(addFilterJQL), + new(addWorklogToIssue), } } diff --git a/backend/plugins/jira/tasks/apiv2models/issue.go b/backend/plugins/jira/tasks/apiv2models/issue.go index 2ebace7c4..8a4417a3d 100644 --- a/backend/plugins/jira/tasks/apiv2models/issue.go +++ b/backend/plugins/jira/tasks/apiv2models/issue.go @@ -242,6 +242,9 @@ func (i Issue) toToolLayer(connectionId uint64) *models.JiraIssue { if i.Changelog != nil { result.ChangelogTotal = i.Changelog.Total } + if i.Fields.Worklog != nil { + result.WorklogTotal = i.Fields.Worklog.Total + } if i.Fields.Epic != nil { result.EpicKey = i.Fields.Epic.Key } diff --git a/backend/plugins/jira/tasks/worklog_collector.go b/backend/plugins/jira/tasks/worklog_collector.go index 8112a6d0a..b364a7004 100644 --- a/backend/plugins/jira/tasks/worklog_collector.go +++ b/backend/plugins/jira/tasks/worklog_collector.go @@ -20,6 +20,7 @@ package tasks import ( "encoding/json" "net/http" + "net/url" "reflect" "github.com/apache/incubator-devlake/core/dal" @@ -59,32 +60,15 @@ func CollectWorklogs(taskCtx plugin.SubTaskContext) errors.Error { // filter out issue_ids that needed collection clauses := []dal.Clause{ - dal.Select("i.issue_id, i.updated AS update_time"), + dal.Select("i.issue_id AS issue_id, i.updated AS update_time"), dal.From("_tool_jira_board_issues bi"), dal.Join("LEFT JOIN _tool_jira_issues i ON (bi.connection_id = i.connection_id AND bi.issue_id = i.issue_id)"), - dal.Join("LEFT JOIN _tool_jira_worklogs wl ON (wl.connection_id = i.connection_id AND wl.issue_id = i.issue_id)"), - dal.Where("i.updated > i.created AND bi.connection_id = ? AND bi.board_id = ? ", data.Options.ConnectionId, data.Options.BoardId), - dal.Groupby("i.issue_id, i.updated"), + dal.Where("bi.connection_id=? and bi.board_id = ? AND i.std_type != ? and i.worklog_total > 20", data.Options.ConnectionId, data.Options.BoardId, "Epic"), } if apiCollector.IsIncremental() && apiCollector.GetSince() != nil { - clauses = append( - clauses, - dal.Having( - "i.updated > ? AND (i.updated > max(wl.issue_updated) OR (max(wl.issue_updated) IS NULL AND COUNT(wl.worklog_id) > 0))", - apiCollector.GetSince(), - ), - ) - } else { - /* - i.updated > max(wl.issue_updated) was deleted because for non-incremental collection, - max(wl.issue_updated) will only be one of null, less or equal to i.updated - so i.updated > max(wl.issue_updated) is always false. - max(wl.issue_updated) IS NULL AND COUNT(wl.worklog_id) > 0 infers the issue has more than 100 worklogs, - because we collected worklogs when collecting issues, and assign worklog.issue_updated if num of worklogs < 100, - and max(wl.issue_updated) IS NULL AND COUNT(wl.worklog_id) > 0 means all worklogs for the issue were not assigned issue_updated - */ - clauses = append(clauses, dal.Having("max(wl.issue_updated) IS NULL AND COUNT(wl.worklog_id) > 0")) + clauses = append(clauses, dal.Where("i.updated > ?", apiCollector.GetSince())) } + // construct the input iterator cursor, err := db.Cursor(clauses...) if err != nil { @@ -101,6 +85,12 @@ func CollectWorklogs(taskCtx plugin.SubTaskContext) errors.Error { UrlTemplate: "api/2/issue/{{ .Input.IssueId }}/worklog", PageSize: 50, GetTotalPages: GetTotalPagesFromResponse, + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + // According to the following resources, the worklogs API returns all worklogs without pagination + // https://community.atlassian.com/t5/Jira-questions/Worklog-Pagination/qaq-p/117614 + // https://community.atlassian.com/t5/Jira-questions/Worklog-Pagination-JIRA-REST-API/qaq-p/2173832 + return nil, nil + }, ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { var data struct { Worklogs []json.RawMessage `json:"worklogs"`
