Copilot commented on code in PR #62238:
URL: https://github.com/apache/doris/pull/62238#discussion_r3055233073
##########
fe/fe-core/src/test/java/org/apache/doris/job/base/AbstractJobStatusTest.java:
##########
@@ -217,4 +217,68 @@ void testFinishSetsFinishTime() throws Exception {
Assertions.assertEquals(JobStatus.FINISHED, job.getJobStatus());
Assertions.assertTrue(job.getFinishTimeMs() >= before);
}
+
+ /**
+ * Test retrying status transitions
+ */
+ @Test
+ void testRetryingFromRunning() throws Exception {
+ DummyJob job = new DummyJob(JobStatus.RUNNING);
+ job.updateJobStatus(JobStatus.RETRYING);
+ Assertions.assertEquals(JobStatus.RETRYING, job.getJobStatus());
+ }
+
+ @Test
+ void testRunningFromRetrying() throws Exception {
+ DummyJob job = new DummyJob(JobStatus.RETRYING);
+ job.updateJobStatus(JobStatus.RUNNING);
+ Assertions.assertEquals(JobStatus.RUNNING, job.getJobStatus());
+ }
+
+ @Test
+ void testPausedFromRetrying() throws Exception {
+ DummyJob job = new DummyJob(JobStatus.RETRYING);
+ job.updateJobStatus(JobStatus.PAUSED);
+ Assertions.assertEquals(JobStatus.PAUSED, job.getJobStatus());
+ }
+
+ @Test
+ void testStoppedFromRetrying() throws Exception {
+ DummyJob job = new DummyJob(JobStatus.RETRYING);
+ job.updateJobStatus(JobStatus.STOPPED);
+ Assertions.assertEquals(JobStatus.STOPPED, job.getJobStatus());
+ }
+
+ @Test
+ void testFinishedFromRetrying() throws Exception {
+ DummyJob job = new DummyJob(JobStatus.RETRYING);
+ job.updateJobStatus(JobStatus.FINISHED);
+ Assertions.assertEquals(JobStatus.FINISHED, job.getJobStatus());
+ }
+
+ @Test
+ void testRetryingFromStoppedIsInvalid() {
+ DummyJob job = new DummyJob(JobStatus.STOPPED);
+ Assertions.assertThrows(IllegalArgumentException.class, () ->
job.updateJobStatus(JobStatus.RETRYING));
+ }
+
+ @Test
+ void testPendingFromRetrying() throws Exception {
+ DummyJob job = new DummyJob(JobStatus.RETRYING);
+ job.updateJobStatus(JobStatus.PENDING);
+ Assertions.assertEquals(JobStatus.PENDING, job.getJobStatus());
+ }
+
+ @Test
+ void testRetryingCancelsTasks() throws Exception {
+ DummyJob job = new DummyJob(JobStatus.RUNNING);
+ job.createTasks(TaskType.STREAMING, null);
+ job.updateJobStatus(JobStatus.RETRYING);
+ Assertions.assertEquals(JobStatus.RETRYING, job.getJobStatus());
Review Comment:
`testRetryingCancelsTasks` calls `job.createTasks(...)` directly, which does
not add tasks to `AbstractJob`’s `runningTasks` (only
`commonCreateTasks()/initTasks()` does). As a result,
`updateJobStatus(RETRYING)` won’t actually cancel anything and the test doesn’t
validate the intended behavior implied by its name. Consider creating tasks via
`commonCreateTasks(TaskType.STREAMING, ...)` (or calling `initTasks(...)`) and
asserting that the task’s cancel logic was invoked and/or `canceledTaskCount`
incremented.
```suggestion
job.commonCreateTasks(TaskType.STREAMING, null);
job.updateJobStatus(JobStatus.RETRYING);
Assertions.assertEquals(JobStatus.RETRYING, job.getJobStatus());
Assertions.assertEquals(1, job.canceledTaskCount);
```
##########
fe/fe-core/src/main/java/org/apache/doris/job/scheduler/StreamingTaskScheduler.java:
##########
@@ -94,9 +94,9 @@ private void scheduleTasks(List<AbstractStreamingTask> tasks)
{
(StreamingInsertJob)
Env.getCurrentEnv().getJobManager().getJob(task.getJobId());
job.setFailureReason(new FailureReason(e.getMessage()));
try {
- job.updateJobStatus(JobStatus.PAUSED);
+ job.updateJobStatus(JobStatus.RETRYING);
} catch (JobException ex) {
- log.warn("Failed to pause job {} after task {}
scheduling failed",
+ log.warn("Failed to set job {} to RETRYING after task
{} scheduling failed",
task.getJobId(), task.getTaskId(), ex);
}
Review Comment:
In the scheduling failure handler,
`Env.getCurrentEnv().getJobManager().getJob(task.getJobId())` can return null
(e.g., job dropped concurrently), which would cause an NPE at
`job.setFailureReason(...)`/`job.updateJobStatus(...)`. Add a null-check before
using `job` (and consider logging that the job no longer exists) to avoid the
scheduler thread dying unexpectedly.
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ResumeJobCommand.java:
##########
@@ -43,6 +43,9 @@ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
public void doRun(ConnectContext ctx, StmtExecutor executor) throws
Exception {
AbstractJob job =
ctx.getEnv().getJobManager().getJobByName(super.getJobName());
if (job instanceof StreamingInsertJob) {
+ if (JobStatus.RETRYING.equals(job.getJobStatus())) {
+ throw new Exception("Job is in RETRYING state, cannot be
manually resumed");
+ }
Review Comment:
Throwing a generic `Exception` here will be treated as an internal error in
some call paths and loses the structured “analysis/user error” handling used
elsewhere in commands (e.g., `AlterJobStatusCommand.validate()` uses
`AnalysisException`). Consider throwing `AnalysisException` (or a job-specific
exception type used by other job commands) and include the job name/status in
the message so users get a clear, actionable error.
##########
fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobSchedulerTask.java:
##########
@@ -83,28 +86,41 @@ private void handleRunningState() throws JobException {
streamingInsertJob.fetchMeta();
}
- private void autoResumeHandler() throws JobException {
- final FailureReason failureReason =
streamingInsertJob.getFailureReason();
+ private void handleRetryingState() throws JobException {
final long latestAutoResumeTimestamp =
streamingInsertJob.getLatestAutoResumeTimestamp();
final long autoResumeCount = streamingInsertJob.getAutoResumeCount();
final long current = System.currentTimeMillis();
- if (failureReason != null
- && failureReason.getCode() !=
InternalErrorCode.MANUAL_PAUSE_ERR
- && failureReason.getCode() !=
InternalErrorCode.TOO_MANY_FAILURE_ROWS_ERR
- && failureReason.getCode() !=
InternalErrorCode.CANNOT_RESUME_ERR) {
- long autoResumeIntervalTimeSec = autoResumeCount < 5
- ? Math.min((long) Math.pow(2, autoResumeCount) *
BACK_OFF_BASIC_TIME_SEC,
- MAX_BACK_OFF_TIME_SEC) : MAX_BACK_OFF_TIME_SEC;
- if (current - latestAutoResumeTimestamp >
autoResumeIntervalTimeSec * 1000L) {
- streamingInsertJob.setLatestAutoResumeTimestamp(current);
- if (autoResumeCount < Long.MAX_VALUE) {
- streamingInsertJob.setAutoResumeCount(autoResumeCount + 1);
- }
- streamingInsertJob.updateJobStatus(JobStatus.PENDING);
+ long autoResumeIntervalTimeSec = autoResumeCount < 5
+ ? Math.min((long) Math.pow(2, autoResumeCount) *
BACK_OFF_BASIC_TIME_SEC,
+ MAX_BACK_OFF_TIME_SEC) : MAX_BACK_OFF_TIME_SEC;
+ if (current - latestAutoResumeTimestamp <= autoResumeIntervalTimeSec *
1000L) {
+ // backoff not reached, wait
+ return;
+ }
+
+ streamingInsertJob.setLatestAutoResumeTimestamp(current);
+ if (autoResumeCount < Long.MAX_VALUE) {
+ streamingInsertJob.setAutoResumeCount(autoResumeCount + 1);
+ }
+
+ if (Config.isCloudMode()) {
+ try {
+ streamingInsertJob.replayOnCloudMode();
+ } catch (JobException e) {
+ streamingInsertJob.setFailureReason(
+ new FailureReason(InternalErrorCode.INTERNAL_ERR,
e.getMessage()));
+ // stay in RETRYING, will retry next round
return;
}
}
+ if (streamingInsertJob.hasReachedEnd()) {
+ streamingInsertJob.updateJobStatus(JobStatus.FINISHED);
+ return;
+ }
Review Comment:
`handleRetryingState()` recreates tasks directly but does not call
`streamingInsertJob.replayOffsetProviderIfNeed()` like `handlePendingState()`
does (see `handlePendingState()` around line 71). Since the previous
auto-resume path transitioned to `PENDING` first, this replay step may be
skipped when a job is persisted/restored in `RETRYING`, potentially creating
tasks with an un-replayed offset provider after FE restart. Consider replaying
the offset provider in `handleRetryingState()` before `createStreamingTask()`
(and/or reusing the `PENDING` flow internally while keeping the external status
as `RETRYING`).
```suggestion
}
streamingInsertJob.replayOffsetProviderIfNeed();
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]