This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch release-v1.0
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v1.0 by this push:
new 48dd54464 Cherry pick #8152 to release v1.0 (#8153)
48dd54464 is described below
commit 48dd54464fe189527b93d04172635e2120c229d1
Author: NaRro <[email protected]>
AuthorDate: Wed Oct 23 11:12:34 2024 +0800
Cherry pick #8152 to release v1.0 (#8153)
* fix(framework): fix finished_record count in _devlake_subtasks (#8054)
* feat: not update sub task progress if progress less than 1 pct (#8152)
[Refactor][core]Data inflation when using postgres #8142
---------
Co-authored-by: Lynwee <[email protected]>
---
backend/core/runner/run_task.go | 22 +++++++++++++++-------
1 file changed, 15 insertions(+), 7 deletions(-)
diff --git a/backend/core/runner/run_task.go b/backend/core/runner/run_task.go
index 02d92c34c..8a3515768 100644
--- a/backend/core/runner/run_task.go
+++ b/backend/core/runner/run_task.go
@@ -20,10 +20,11 @@ package runner
import (
gocontext "context"
"fmt"
- "github.com/apache/incubator-devlake/core/models/common"
"strings"
"time"
+ "github.com/apache/incubator-devlake/core/models/common"
+
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
@@ -353,6 +354,7 @@ func UpdateProgressDetail(basicRes context.BasicRes, taskId
uint64, progressDeta
Model: common.Model{ID: taskId},
}
subtask := &models.Subtask{}
+ originalFinishedRecords := progressDetail.FinishedRecords
switch p.Type {
case plugin.TaskSetProgress:
progressDetail.TotalSubTasks = p.Total
@@ -369,6 +371,17 @@ func UpdateProgressDetail(basicRes context.BasicRes,
taskId uint64, progressDeta
progressDetail.TotalRecords = p.Total
case plugin.SubTaskIncProgress:
progressDetail.FinishedRecords = p.Current
+ case plugin.SetCurrentSubTask:
+ progressDetail.SubTaskName = p.SubTaskName
+ progressDetail.SubTaskNumber = p.SubTaskNumber
+ // reset finished records
+ progressDetail.FinishedRecords = 0
+ }
+ currentFinishedRecords := progressDetail.FinishedRecords
+ currentTotalRecords := progressDetail.TotalRecords
+ // update progress if progress is more than 1%
+ // or there is progress if no total record provided
+ if (currentTotalRecords > 0 &&
float64(currentFinishedRecords-originalFinishedRecords)/float64(currentTotalRecords)
> 0.01) || (currentTotalRecords <= 0 && currentFinishedRecords >
originalFinishedRecords) {
// update subtask progress
where := dal.Where("task_id = ? and name = ?", taskId,
progressDetail.SubTaskName)
err := basicRes.GetDal().UpdateColumns(subtask, []dal.DalSet{
@@ -377,11 +390,6 @@ func UpdateProgressDetail(basicRes context.BasicRes,
taskId uint64, progressDeta
if err != nil {
basicRes.GetLogger().Error(err, "failed to update
_devlake_subtasks progress")
}
- case plugin.SetCurrentSubTask:
- progressDetail.SubTaskName = p.SubTaskName
- progressDetail.SubTaskNumber = p.SubTaskNumber
- default:
- return
}
}
@@ -417,7 +425,7 @@ func recordSubtask(basicRes context.BasicRes, subtask
*models.Subtask) {
{ColumnName: "began_at", Value: subtask.BeganAt},
{ColumnName: "finished_at", Value: subtask.FinishedAt},
{ColumnName: "spent_seconds", Value: subtask.SpentSeconds},
- {ColumnName: "finished_records", Value:
subtask.FinishedRecords},
+ //{ColumnName: "finished_records", Value:
subtask.FinishedRecords}, // FinishedRecords is zero always.
{ColumnName: "number", Value: subtask.Number},
}, where); err != nil {
basicRes.GetLogger().Error(err, "error writing subtask %d
status to DB: %v", subtask.ID)
