This is an automated email from the ASF dual-hosted git repository.

dimberman pushed a commit to branch handle-stuck-in-queue
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/handle-stuck-in-queue by this 
push:
     new 765bbd6dd07 Allow for retry when tasks are stuck in queued
765bbd6dd07 is described below

commit 765bbd6dd07a6ce7726406f1b985ff2a54d95dd6
Author: Daniel Imberman <[email protected]>
AuthorDate: Wed Oct 30 10:24:11 2024 -0700

    Allow for retry when tasks are stuck in queued
---
 airflow/jobs/scheduler_job_runner.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/airflow/jobs/scheduler_job_runner.py 
b/airflow/jobs/scheduler_job_runner.py
index 15042b0d3f1..3701c0b01e2 100644
--- a/airflow/jobs/scheduler_job_runner.py
+++ b/airflow/jobs/scheduler_job_runner.py
@@ -1822,9 +1822,30 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
                                 ),
                             )
                         )
+                        num_times_stuck = self._get_num_times_stuck(ti, 
session)
+                        if num_times_stuck < conf.getint("core", 
"num_stuck_retries", fallback=3):
+                            self._reset_task_instance(ti, session)
+
+
             except NotImplementedError:
                 self.log.debug("Executor doesn't support cleanup of stuck 
queued tasks. Skipping.")
 
+    @provide_session
+    def _get_num_times_stuck(self, ti: TaskInstance, session: Session = 
NEW_SESSION) -> int:
+        return len(session.scalars(select(Log).where(Log.dag_id == ti.task_id)
+                        .where(Log.dag_id == ti.dag_id)
+                        .where(Log.run_id == ti.run_id)
+                        .where(Log.map_index == ti.map_index)
+                        .where(Log.try_number == ti.try_number)
+                        .where(Log.event == "stuck in queued")
+                        ))
+
+    def _reset_task_instance(self, ti: TaskInstance, session: Session = 
NEW_SESSION):
+        ti.external_executor_id = None
+        ti.state = State.SCHEDULED
+        session.add(ti)
+        session.commit()
+
     @provide_session
     def _emit_pool_metrics(self, session: Session = NEW_SESSION) -> None:
         from airflow.models.pool import Pool

Reply via email to