This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new c42a584cd Jira switch to timeAfter option (#4426)
c42a584cd is described below
commit c42a584cdb900ec09448e2a871b4407eb8c08aa4
Author: Klesh Wong <[email protected]>
AuthorDate: Thu Feb 16 11:04:48 2023 +0800
Jira switch to timeAfter option (#4426)
* feat: jira supports timefilter by updated_date
* fix: createdDateAfter follow previous logic
* fix: remove redundant condition
* refactor: rename UpdatedDateAfter to TimeAfter
* feat: jira supports time_after option
* fix: linting
---
backend/core/models/blueprint.go | 7 ++-
backend/core/models/collector_state.go | 10 ++--
.../20230213_add_time_after_to_collector_state.go} | 31 ++++++++---
backend/core/models/migrationscripts/register.go | 1 +
backend/core/plugin/plugin_blueprint.go | 9 ++-
.../pluginhelper/api/api_collector_with_state.go | 64 +++++++++++++++-------
backend/plugins/jira/api/blueprint_v200.go | 7 ++-
backend/plugins/jira/impl/impl.go | 25 ++++-----
.../jira/tasks/issue_changelog_collector.go | 11 ++--
backend/plugins/jira/tasks/issue_collector.go | 20 ++++---
backend/plugins/jira/tasks/remotelink_collector.go | 9 +--
backend/plugins/jira/tasks/task_data.go | 13 +++--
backend/plugins/jira/tasks/worklog_collector.go | 9 +--
backend/server/services/blueprint.go | 5 +-
backend/test/helper/api.go | 15 +++--
15 files changed, 146 insertions(+), 90 deletions(-)
diff --git a/backend/core/models/blueprint.go b/backend/core/models/blueprint.go
index 69438acf7..5d1ebcdfe 100644
--- a/backend/core/models/blueprint.go
+++ b/backend/core/models/blueprint.go
@@ -19,10 +19,11 @@ package models
import (
"encoding/json"
+ "time"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
- "time"
)
const (
@@ -47,8 +48,10 @@ type Blueprint struct {
}
type BlueprintSettings struct {
- Version string `json:"version"
validate:"required,semver,oneof=1.0.0"`
+ Version string `json:"version" validate:"required,semver,oneof=1.0.0"`
+ // Deprecating(timeAfter): copy to TimeAfter and delete the field in
last step
CreatedDateAfter *time.Time `json:"createdDateAfter"`
+ TimeAfter *time.Time `json:"timeAfter"`
Connections json.RawMessage `json:"connections"
validate:"required"`
BeforePlan json.RawMessage `json:"before_plan"`
AfterPlan json.RawMessage `json:"after_plan"`
diff --git a/backend/core/models/collector_state.go
b/backend/core/models/collector_state.go
index e0230a408..654556779 100644
--- a/backend/core/models/collector_state.go
+++ b/backend/core/models/collector_state.go
@@ -22,11 +22,13 @@ import (
)
type CollectorLatestState struct {
- CreatedAt time.Time `json:"createdAt"`
- UpdatedAt time.Time `json:"updatedAt"`
- RawDataParams string
`gorm:"primaryKey;column:raw_data_params;type:varchar(255);index"
json:"raw_data_params"`
- RawDataTable string
`gorm:"primaryKey;column:raw_data_table;type:varchar(255)"
json:"raw_data_table"`
+ CreatedAt time.Time `json:"createdAt"`
+ UpdatedAt time.Time `json:"updatedAt"`
+ RawDataParams string
`gorm:"primaryKey;column:raw_data_params;type:varchar(255);index"
json:"raw_data_params"`
+ RawDataTable string
`gorm:"primaryKey;column:raw_data_table;type:varchar(255)"
json:"raw_data_table"`
+ // Deprecating(timeAfter): copy to TimeAfter and delete the field in
last step
CreatedDateAfter *time.Time
+ TimeAfter *time.Time
LatestSuccessStart *time.Time
}
diff --git a/backend/core/models/collector_state.go
b/backend/core/models/migrationscripts/20230213_add_time_after_to_collector_state.go
similarity index 52%
copy from backend/core/models/collector_state.go
copy to
backend/core/models/migrationscripts/20230213_add_time_after_to_collector_state.go
index e0230a408..882e8a5b5 100644
--- a/backend/core/models/collector_state.go
+++
b/backend/core/models/migrationscripts/20230213_add_time_after_to_collector_state.go
@@ -15,21 +15,34 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package models
+package migrationscripts
import (
"time"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
)
-type CollectorLatestState struct {
- CreatedAt time.Time `json:"createdAt"`
- UpdatedAt time.Time `json:"updatedAt"`
- RawDataParams string
`gorm:"primaryKey;column:raw_data_params;type:varchar(255);index"
json:"raw_data_params"`
- RawDataTable string
`gorm:"primaryKey;column:raw_data_table;type:varchar(255)"
json:"raw_data_table"`
- CreatedDateAfter *time.Time
- LatestSuccessStart *time.Time
+type collectorLatestState20230213 struct {
+ TimeAfter *time.Time
}
-func (CollectorLatestState) TableName() string {
+func (collectorLatestState20230213) TableName() string {
return "_devlake_collector_latest_state"
}
+
+type addTimeAfterToCollectorMeta20230213 struct{}
+
+func (script *addTimeAfterToCollectorMeta20230213) Up(basicRes
context.BasicRes) errors.Error {
+ return migrationhelper.AutoMigrateTables(basicRes,
&collectorLatestState20230213{})
+}
+
+func (*addTimeAfterToCollectorMeta20230213) Version() uint64 {
+ return 20230213200040
+}
+
+func (*addTimeAfterToCollectorMeta20230213) Name() string {
+ return "add time_after to _devlake_collector_latest_state"
+}
diff --git a/backend/core/models/migrationscripts/register.go
b/backend/core/models/migrationscripts/register.go
index ad81a52d1..da0ad9d46 100644
--- a/backend/core/models/migrationscripts/register.go
+++ b/backend/core/models/migrationscripts/register.go
@@ -69,6 +69,7 @@ func All() []plugin.MigrationScript {
new(encryptTask221221),
new(renameProjectMetrics),
new(addOriginalTypeToIssue221230),
+ new(addTimeAfterToCollectorMeta20230213),
new(addCodeQuality),
new(modifyIssueStorypointToFloat64),
}
diff --git a/backend/core/plugin/plugin_blueprint.go
b/backend/core/plugin/plugin_blueprint.go
index a6202672b..c842f909d 100644
--- a/backend/core/plugin/plugin_blueprint.go
+++ b/backend/core/plugin/plugin_blueprint.go
@@ -19,8 +19,9 @@ package plugin
import (
"encoding/json"
- "github.com/apache/incubator-devlake/core/errors"
"time"
+
+ "github.com/apache/incubator-devlake/core/errors"
)
// PipelineTask represents a smallest unit of execution inside a PipelinePlan
@@ -172,7 +173,9 @@ type CompositePluginBlueprintV200 interface {
}
type BlueprintSyncPolicy struct {
- Version string `json:"version"
validate:"required,semver,oneof=1.0.0"`
- SkipOnFail bool `json:"skipOnFail"`
+ Version string `json:"version"
validate:"required,semver,oneof=1.0.0"`
+ SkipOnFail bool `json:"skipOnFail"`
+ // Deprecating(timeAfter): use TimeAfter instead
CreatedDateAfter *time.Time `json:"createdDateAfter"`
+ TimeAfter *time.Time `json:"timeAfter"`
}
diff --git a/backend/helpers/pluginhelper/api/api_collector_with_state.go
b/backend/helpers/pluginhelper/api/api_collector_with_state.go
index 81308b0eb..3d8b11eda 100644
--- a/backend/helpers/pluginhelper/api/api_collector_with_state.go
+++ b/backend/helpers/pluginhelper/api/api_collector_with_state.go
@@ -18,10 +18,11 @@ limitations under the License.
package api
import (
+ "time"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models"
- "time"
)
// ApiCollectorStateManager save collector state in framework table
@@ -29,13 +30,15 @@ type ApiCollectorStateManager struct {
RawDataSubTaskArgs
*ApiCollector
*GraphqlCollector
- LatestState models.CollectorLatestState
+ LatestState models.CollectorLatestState
+ // Deprecating(timeAfter): to be deleted
CreatedDateAfter *time.Time
+ TimeAfter *time.Time
ExecuteStart time.Time
}
-// NewApiCollectorWithState create a new ApiCollectorStateManager
-func NewApiCollectorWithState(args RawDataSubTaskArgs, createdDateAfter
*time.Time) (*ApiCollectorStateManager, errors.Error) {
+// NewApiCollectorWithStateEx create a new ApiCollectorStateManager
+func NewApiCollectorWithStateEx(args RawDataSubTaskArgs, createdDateAfter
*time.Time, timeAfter *time.Time) (*ApiCollectorStateManager, errors.Error) {
db := args.Ctx.GetDal()
rawDataSubTask, err := NewRawDataSubTask(args)
@@ -57,19 +60,37 @@ func NewApiCollectorWithState(args RawDataSubTaskArgs,
createdDateAfter *time.Ti
return &ApiCollectorStateManager{
RawDataSubTaskArgs: args,
LatestState: latestState,
- CreatedDateAfter: createdDateAfter,
- ExecuteStart: time.Now(),
+ // Deprecating(timeAfter): to be deleted
+ CreatedDateAfter: createdDateAfter,
+ TimeAfter: timeAfter,
+ ExecuteStart: time.Now(),
}, nil
}
-// IsIncremental return if the old data can support collect incrementally.
-// only when latest collection is success &&
-// (m.LatestState.CreatedDateAfter == nil means all data have been collected ||
-// CreatedDateAfter at this time exists and no before than in the LatestState)
-// if CreatedDateAfter at this time not exists, collect incrementally only
when "m.LatestState.CreatedDateAfter == nil"
-func (m ApiCollectorStateManager) IsIncremental() bool {
- return m.LatestState.LatestSuccessStart != nil &&
- (m.LatestState.CreatedDateAfter == nil || m.CreatedDateAfter !=
nil && !m.CreatedDateAfter.Before(*m.LatestState.CreatedDateAfter))
+// NewApiCollectorWithState create a new ApiCollectorStateManager
+// Deprecating(timeAfter): use NewStatefulApiCollector instead
+func NewApiCollectorWithState(args RawDataSubTaskArgs, createdDateAfter
*time.Time) (*ApiCollectorStateManager, errors.Error) {
+ return NewApiCollectorWithStateEx(args, createdDateAfter, nil)
+}
+
+// NewApiCollectorWithState create a new ApiCollectorStateManager
+func NewStatefulApiCollector(args RawDataSubTaskArgs, timeAfter *time.Time)
(*ApiCollectorStateManager, errors.Error) {
+ return NewApiCollectorWithStateEx(args, nil, timeAfter)
+}
+
+// IsIncremental indicates if the collector should operate in incremental mode
+func (m *ApiCollectorStateManager) IsIncremental() bool {
+ // the initial collection
+ if m.LatestState.LatestSuccessStart == nil {
+ return false
+ }
+ // prioritize TimeAfter parameter: collector should filter data by
`updated_date`
+ if m.TimeAfter != nil {
+ return m.LatestState.TimeAfter == nil ||
!m.TimeAfter.Before(*m.LatestState.TimeAfter)
+ }
+ // Deprecating(timeAfter): to be removed
+ // fallback to CreatedDateAfter: collector should filter data by
`created_date`
+ return m.LatestState.CreatedDateAfter == nil || m.CreatedDateAfter !=
nil && !m.CreatedDateAfter.Before(*m.LatestState.CreatedDateAfter)
}
// InitCollector init the embedded collector
@@ -93,11 +114,7 @@ func (m ApiCollectorStateManager) Execute() errors.Error {
return err
}
- db := m.Ctx.GetDal()
- m.LatestState.LatestSuccessStart = &m.ExecuteStart
- m.LatestState.CreatedDateAfter = m.CreatedDateAfter
- err = db.CreateOrUpdate(&m.LatestState)
- return err
+ return m.updateState()
}
// ExecuteGraphQL the embedded collector and record execute state
@@ -107,9 +124,14 @@ func (m ApiCollectorStateManager) ExecuteGraphQL()
errors.Error {
return err
}
+ return m.updateState()
+}
+
+func (m ApiCollectorStateManager) updateState() errors.Error {
db := m.Ctx.GetDal()
m.LatestState.LatestSuccessStart = &m.ExecuteStart
+ // Deprecating(timeAfter): to be deleted
m.LatestState.CreatedDateAfter = m.CreatedDateAfter
- err = db.CreateOrUpdate(&m.LatestState)
- return err
+ m.LatestState.TimeAfter = m.TimeAfter
+ return db.CreateOrUpdate(&m.LatestState)
}
diff --git a/backend/plugins/jira/api/blueprint_v200.go
b/backend/plugins/jira/api/blueprint_v200.go
index d30a5c802..7ae0af6fb 100644
--- a/backend/plugins/jira/api/blueprint_v200.go
+++ b/backend/plugins/jira/api/blueprint_v200.go
@@ -19,6 +19,8 @@ package api
import (
"fmt"
+ "time"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -28,7 +30,6 @@ import (
"github.com/apache/incubator-devlake/core/utils"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/jira/models"
- "time"
)
func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta,
connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy
*plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope,
errors.Error) {
@@ -61,8 +62,8 @@ func makeDataSourcePipelinePlanV200(
options := make(map[string]interface{})
options["scopeId"] = bpScope.Id
options["connectionId"] = connectionId
- if syncPolicy.CreatedDateAfter != nil {
- options["createdDateAfter"] =
syncPolicy.CreatedDateAfter.Format(time.RFC3339)
+ if syncPolicy.TimeAfter != nil {
+ options["timeAfter"] =
syncPolicy.TimeAfter.Format(time.RFC3339)
}
subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
bpScope.Entities)
diff --git a/backend/plugins/jira/impl/impl.go
b/backend/plugins/jira/impl/impl.go
index 34f82dd79..4431364e5 100644
--- a/backend/plugins/jira/impl/impl.go
+++ b/backend/plugins/jira/impl/impl.go
@@ -19,6 +19,9 @@ package impl
import (
"fmt"
+ "net/http"
+ "time"
+
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
@@ -29,8 +32,6 @@ import (
"github.com/apache/incubator-devlake/plugins/jira/models/migrationscripts"
"github.com/apache/incubator-devlake/plugins/jira/tasks"
"github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models"
- "net/http"
- "time"
)
var _ interface {
@@ -221,14 +222,6 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext,
options map[string]int
}
}
- var createdDateAfter time.Time
- if op.CreatedDateAfter != "" {
- createdDateAfter, err =
errors.Convert01(time.Parse(time.RFC3339, op.CreatedDateAfter))
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "invalid value
for `createdDateAfter`")
- }
- }
-
info, code, err := tasks.GetJiraServerInfo(jiraApiClient)
if err != nil || code != http.StatusOK || info == nil {
return nil, errors.HttpStatus(code).Wrap(err, "fail to get Jira
server info")
@@ -238,11 +231,15 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext,
options map[string]int
ApiClient: jiraApiClient,
JiraServerInfo: *info,
}
- if !createdDateAfter.IsZero() {
- taskData.CreatedDateAfter = &createdDateAfter
- logger.Debug("collect data created from %s", createdDateAfter)
+ if op.TimeAfter != "" {
+ var timeAfter time.Time
+ timeAfter, err = errors.Convert01(time.Parse(time.RFC3339,
op.TimeAfter))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "invalid value
for `timeAfter`")
+ }
+ taskData.TimeAfter = &timeAfter
+ logger.Debug("collect data created from %s", timeAfter)
}
-
return taskData, nil
}
diff --git a/backend/plugins/jira/tasks/issue_changelog_collector.go
b/backend/plugins/jira/tasks/issue_changelog_collector.go
index 480d853b1..96502a3be 100644
--- a/backend/plugins/jira/tasks/issue_changelog_collector.go
+++ b/backend/plugins/jira/tasks/issue_changelog_collector.go
@@ -20,6 +20,10 @@ package tasks
import (
"encoding/json"
"fmt"
+ "net/http"
+ "net/url"
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/log"
@@ -27,9 +31,6 @@ import (
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/jira/models"
"github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models"
- "net/http"
- "net/url"
- "reflect"
)
var _ plugin.SubTaskEntryPoint = CollectIssueChangelogs
@@ -52,14 +53,14 @@ func CollectIssueChangelogs(taskCtx plugin.SubTaskContext)
errors.Error {
logger := taskCtx.GetLogger()
db := taskCtx.GetDal()
- collectorWithState, err :=
api.NewApiCollectorWithState(api.RawDataSubTaskArgs{
+ collectorWithState, err :=
api.NewStatefulApiCollector(api.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: JiraApiParams{
ConnectionId: data.Options.ConnectionId,
BoardId: data.Options.BoardId,
},
Table: RAW_CHANGELOG_TABLE,
- }, data.CreatedDateAfter)
+ }, data.TimeAfter)
if err != nil {
return err
}
diff --git a/backend/plugins/jira/tasks/issue_collector.go
b/backend/plugins/jira/tasks/issue_collector.go
index 124c2dde3..0e60cc56d 100644
--- a/backend/plugins/jira/tasks/issue_collector.go
+++ b/backend/plugins/jira/tasks/issue_collector.go
@@ -20,12 +20,13 @@ package tasks
import (
"encoding/json"
"fmt"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"io"
"net/http"
"net/url"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
const RAW_ISSUE_TABLE = "jira_api_issues"
@@ -43,7 +44,7 @@ var CollectIssuesMeta = plugin.SubTaskMeta{
func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*JiraTaskData)
- collectorWithState, err :=
api.NewApiCollectorWithState(api.RawDataSubTaskArgs{
+ collectorWithState, err :=
api.NewStatefulApiCollector(api.RawDataSubTaskArgs{
Ctx: taskCtx,
/*
This struct will be JSONEncoded and stored into
database along with raw data itself, to identity minimal
@@ -57,7 +58,7 @@ func CollectIssues(taskCtx plugin.SubTaskContext)
errors.Error {
Table store raw data
*/
Table: RAW_ISSUE_TABLE,
- }, data.CreatedDateAfter)
+ }, data.TimeAfter)
if err != nil {
return err
}
@@ -65,13 +66,14 @@ func CollectIssues(taskCtx plugin.SubTaskContext)
errors.Error {
// build jql
// IMPORTANT: we have to keep paginated data in a consistence order to
avoid data-missing, if we sort issues by
// `updated`, issue will be jumping between pages if it got updated
during the collection process
- createdDateAfter := data.CreatedDateAfter
jql := "created is not null ORDER BY created ASC"
- if createdDateAfter != nil {
- // prepend a time range criteria if `since` was specified,
either by user or from database
- jql = fmt.Sprintf("created >= '%v' AND %v",
createdDateAfter.Format("2006/01/02 15:04"), jql)
+
+ // timer filter
+ if data.TimeAfter != nil {
+ jql = fmt.Sprintf("updated >= '%v' AND %v",
data.TimeAfter.Format("2006/01/02 15:04"), jql)
}
+ // diff sync
incremental := collectorWithState.IsIncremental()
if incremental {
jql = fmt.Sprintf("updated >= '%v' AND %v",
collectorWithState.LatestState.LatestSuccessStart.Format("2006/01/02 15:04"),
jql)
diff --git a/backend/plugins/jira/tasks/remotelink_collector.go
b/backend/plugins/jira/tasks/remotelink_collector.go
index 3ebed4c39..7abdb25b5 100644
--- a/backend/plugins/jira/tasks/remotelink_collector.go
+++ b/backend/plugins/jira/tasks/remotelink_collector.go
@@ -19,13 +19,14 @@ package tasks
import (
"encoding/json"
+ "net/http"
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models"
- "net/http"
- "reflect"
)
const RAW_REMOTELINK_TABLE = "jira_api_remotelinks"
@@ -46,14 +47,14 @@ func CollectRemotelinks(taskCtx plugin.SubTaskContext)
errors.Error {
logger := taskCtx.GetLogger()
logger.Info("collect remotelink")
- collectorWithState, err :=
api.NewApiCollectorWithState(api.RawDataSubTaskArgs{
+ collectorWithState, err :=
api.NewStatefulApiCollector(api.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: JiraApiParams{
ConnectionId: data.Options.ConnectionId,
BoardId: data.Options.BoardId,
},
Table: RAW_REMOTELINK_TABLE,
- }, data.CreatedDateAfter)
+ }, data.TimeAfter)
if err != nil {
return err
}
diff --git a/backend/plugins/jira/tasks/task_data.go
b/backend/plugins/jira/tasks/task_data.go
index 740004b19..5ac5a9d13 100644
--- a/backend/plugins/jira/tasks/task_data.go
+++ b/backend/plugins/jira/tasks/task_data.go
@@ -20,10 +20,11 @@ package tasks
import (
"encoding/json"
"fmt"
+ "time"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/jira/models"
- "time"
)
type StatusMapping struct {
@@ -93,17 +94,17 @@ func MakeTransformationRules(rule
models.JiraTransformationRule) (*JiraTransform
type JiraOptions struct {
ConnectionId uint64 `json:"connectionId"`
BoardId uint64 `json:"boardId"`
- CreatedDateAfter string
+ TimeAfter string
TransformationRules *JiraTransformationRule
`json:"transformationRules"`
ScopeId string
TransformationRuleId uint64
}
type JiraTaskData struct {
- Options *JiraOptions
- ApiClient *api.ApiAsyncClient
- CreatedDateAfter *time.Time
- JiraServerInfo models.JiraServerInfo
+ Options *JiraOptions
+ ApiClient *api.ApiAsyncClient
+ TimeAfter *time.Time
+ JiraServerInfo models.JiraServerInfo
}
type JiraApiParams struct {
diff --git a/backend/plugins/jira/tasks/worklog_collector.go
b/backend/plugins/jira/tasks/worklog_collector.go
index 2d9d0c85d..26e8f2734 100644
--- a/backend/plugins/jira/tasks/worklog_collector.go
+++ b/backend/plugins/jira/tasks/worklog_collector.go
@@ -19,13 +19,14 @@ package tasks
import (
"encoding/json"
+ "net/http"
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models"
- "net/http"
- "reflect"
)
const RAW_WORKLOGS_TABLE = "jira_api_worklogs"
@@ -44,14 +45,14 @@ func CollectWorklogs(taskCtx plugin.SubTaskContext)
errors.Error {
logger := taskCtx.GetLogger()
- collectorWithState, err :=
api.NewApiCollectorWithState(api.RawDataSubTaskArgs{
+ collectorWithState, err :=
api.NewStatefulApiCollector(api.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: JiraApiParams{
ConnectionId: data.Options.ConnectionId,
BoardId: data.Options.BoardId,
},
Table: RAW_WORKLOGS_TABLE,
- }, data.CreatedDateAfter)
+ }, data.TimeAfter)
if err != nil {
return err
}
diff --git a/backend/server/services/blueprint.go
b/backend/server/services/blueprint.go
index e8def9bb8..158921bd9 100644
--- a/backend/server/services/blueprint.go
+++ b/backend/server/services/blueprint.go
@@ -20,6 +20,8 @@ package services
import (
"encoding/json"
"fmt"
+ "strings"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models"
@@ -27,7 +29,6 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/impls/logruslog"
"github.com/robfig/cron/v3"
- "strings"
)
// BlueprintQuery is a query for GetBlueprints
@@ -273,7 +274,9 @@ func MakePlanForBlueprint(blueprint *models.Blueprint)
(plugin.PipelinePlan, err
}
bpSyncPolicy := plugin.BlueprintSyncPolicy{}
+ // Deprecating(timeAfter): to be deleted
bpSyncPolicy.CreatedDateAfter = bpSettings.CreatedDateAfter
+ bpSyncPolicy.TimeAfter = bpSettings.TimeAfter
var plan plugin.PipelinePlan
switch bpSettings.Version {
diff --git a/backend/test/helper/api.go b/backend/test/helper/api.go
index a29b987a7..f22a9e70a 100644
--- a/backend/test/helper/api.go
+++ b/backend/test/helper/api.go
@@ -19,13 +19,14 @@ package helper
import (
"fmt"
+ "net/http"
+ "reflect"
+ "time"
+
"github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
apiProject "github.com/apache/incubator-devlake/server/api/project"
- "net/http"
- "reflect"
- "time"
)
type Connection struct {
@@ -64,8 +65,10 @@ func (d *DevlakeClient) ListConnections(pluginName string)
[]*Connection {
}
type BlueprintV2Config struct {
- Connection *plugin.BlueprintConnectionV200
+ Connection *plugin.BlueprintConnectionV200
+ // Deprecating(timeAfter): to be deleted
CreatedDateAfter *time.Time
+ TimeAfter *time.Time
SkipOnFail bool
ProjectName string
}
@@ -73,8 +76,10 @@ type BlueprintV2Config struct {
// CreateBasicBlueprintV2 FIXME
func (d *DevlakeClient) CreateBasicBlueprintV2(name string, config
*BlueprintV2Config) models.Blueprint {
settings := &models.BlueprintSettings{
- Version: "2.0.0",
+ Version: "2.0.0",
+ // Deprecating(timeAfter): to be deleted
CreatedDateAfter: config.CreatedDateAfter,
+ TimeAfter: config.TimeAfter,
Connections: ToJson([]*plugin.BlueprintConnectionV200{
config.Connection,
}),
