This is an automated email from the ASF dual-hosted git repository.
dimberman pushed a commit to branch handle-stuck-in-queue
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/handle-stuck-in-queue by this
push:
new 765bbd6dd07 Allow for retry when tasks are stuck in queued
765bbd6dd07 is described below
commit 765bbd6dd07a6ce7726406f1b985ff2a54d95dd6
Author: Daniel Imberman <[email protected]>
AuthorDate: Wed Oct 30 10:24:11 2024 -0700
Allow for retry when tasks are stuck in queued
---
airflow/jobs/scheduler_job_runner.py | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/airflow/jobs/scheduler_job_runner.py
b/airflow/jobs/scheduler_job_runner.py
index 15042b0d3f1..3701c0b01e2 100644
--- a/airflow/jobs/scheduler_job_runner.py
+++ b/airflow/jobs/scheduler_job_runner.py
@@ -1822,9 +1822,30 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
),
)
)
+ num_times_stuck = self._get_num_times_stuck(ti,
session)
+ if num_times_stuck < conf.getint("core",
"num_stuck_retries", fallback=3):
+ self._reset_task_instance(ti, session)
+
+
except NotImplementedError:
self.log.debug("Executor doesn't support cleanup of stuck
queued tasks. Skipping.")
+ @provide_session
+ def _get_num_times_stuck(self, ti: TaskInstance, session: Session =
NEW_SESSION) -> int:
+ return len(session.scalars(select(Log).where(Log.dag_id == ti.task_id)
+ .where(Log.dag_id == ti.dag_id)
+ .where(Log.run_id == ti.run_id)
+ .where(Log.map_index == ti.map_index)
+ .where(Log.try_number == ti.try_number)
+ .where(Log.event == "stuck in queued")
+ ))
+
+ def _reset_task_instance(self, ti: TaskInstance, session: Session =
NEW_SESSION):
+ ti.external_executor_id = None
+ ti.state = State.SCHEDULED
+ session.add(ti)
+ session.commit()
+
@provide_session
def _emit_pool_metrics(self, session: Session = NEW_SESSION) -> None:
from airflow.models.pool import Pool