This is an automated email from the ASF dual-hosted git repository.

vincbeck pushed a commit to branch v3-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/v3-4-test by this push:
     new 049f4edb782 Fix scheduler queuing task instances that left the 
scheduled state (#77247) (#77341)
049f4edb782 is described below

commit 049f4edb78232442c233b0562a11bdc2b734aaf2
Author: Vincent <[email protected]>
AuthorDate: Thu Nov 19 15:28:35 2026 -0500

    Fix scheduler queuing task instances that left the scheduled state (#77247) 
(#77341)
    
    Since the query that picks task instances to queue ranks them in a subquery,
    its scheduled-state filter only applies to the snapshot the statement 
started
    from. A locking read re-checks a row against the outer query only, on
    PostgreSQL and on MySQL alike, so a task instance that a user marked 
success or
    failed, or that another scheduler skipped for a Dag run timeout, while the
    query ran was still locked, returned and set to queued.
---
 .../src/airflow/jobs/scheduler_job_runner.py       |  19 +++-
 airflow-core/tests/unit/jobs/test_scheduler_job.py | 117 ++++++++++++++++++++-
 2 files changed, 133 insertions(+), 3 deletions(-)

diff --git a/airflow-core/src/airflow/jobs/scheduler_job_runner.py 
b/airflow-core/src/airflow/jobs/scheduler_job_runner.py
index 49b16c7251d..fad235acb55 100644
--- a/airflow-core/src/airflow/jobs/scheduler_job_runner.py
+++ b/airflow-core/src/airflow/jobs/scheduler_job_runner.py
@@ -797,8 +797,23 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
             timer.start()
 
             try:
-                locked_query = with_row_locks(query, of=TI, session=session, 
skip_locked=True)
-                task_instances_to_examine = session.scalars(locked_query).all()
+                # The ranking subquery reads the statement's snapshot, but a 
locking read returns the latest
+                # version of the rows it locks, which can have left SCHEDULED 
since. The state is read as a
+                # column so that the check sees the locked row whatever TIs 
the session already holds. It is
+                # not checked in the outer WHERE: when the statistics show no 
SCHEDULED rows, PostgreSQL
+                # then joins through the ti_state index instead of the primary 
key.
+                locked_query = with_row_locks(
+                    query.add_columns(TI.state), of=TI, session=session, 
skip_locked=True
+                )
+                locked_rows = session.execute(locked_query).all()
+                task_instances_to_examine = [
+                    ti for ti, state in locked_rows if state == 
TaskInstanceState.SCHEDULED
+                ]
+
+                if skipped := len(locked_rows) - 
len(task_instances_to_examine):
+                    self.log.debug(
+                        "Skipped locked task instances that are no longer 
SCHEDULED", count=skipped
+                    )
 
                 if self.log.isEnabledFor(logging.DEBUG):
                     self.log.debug("Length of the tis to examine is %d", 
len(task_instances_to_examine))
diff --git a/airflow-core/tests/unit/jobs/test_scheduler_job.py 
b/airflow-core/tests/unit/jobs/test_scheduler_job.py
index 3c9b6070a88..edd05b81a09 100644
--- a/airflow-core/tests/unit/jobs/test_scheduler_job.py
+++ b/airflow-core/tests/unit/jobs/test_scheduler_job.py
@@ -39,7 +39,7 @@ import pendulum
 import psutil
 import pytest
 import time_machine
-from sqlalchemy import delete, event as sa_event, func, inspect, select, update
+from sqlalchemy import delete, event as sa_event, func, inspect, select, text, 
update
 from sqlalchemy.dialects import mysql
 from sqlalchemy.orm import joinedload
 
@@ -374,6 +374,26 @@ def make_pool_stats(
     }
 
 
+def _get_connection_id(session: Session) -> int:
+    if session.get_bind().dialect.name == "postgresql":
+        return session.execute(text("SELECT pg_backend_pid()")).scalar_one()
+    return session.execute(text("SELECT CONNECTION_ID()")).scalar_one()
+
+
+def _is_waiting_for_lock(waiter_id: int, blocker_id: int) -> bool:
+    with settings.get_engine().connect() as probe:
+        if probe.dialect.name == "postgresql":
+            blocked = text("SELECT CAST(:blocker AS integer) = 
ANY(pg_blocking_pids(:waiter))")
+            return bool(probe.execute(blocked, {"waiter": waiter_id, 
"blocker": blocker_id}).scalar())
+        # Only that the waiter waits for a row lock: CI runs MySQL with 
performance_schema off, so
+        # performance_schema.data_lock_waits cannot name the blocking 
transaction there.
+        waiting = text(
+            "SELECT count(*) > 0 FROM information_schema.innodb_trx "
+            "WHERE trx_mysql_thread_id = :waiter AND trx_state = 'LOCK WAIT'"
+        )
+        return bool(probe.execute(waiting, {"waiter": waiter_id}).scalar())
+
+
 def assign_executor_to_team(executor: MagicMock, team_name: str) -> None:
     # ExecutorLoader.load_executor sets the team on both; the scheduler reads 
it from the name.
     executor.team_name = team_name
@@ -3704,6 +3724,101 @@ class TestSchedulerJob:
 
         session.rollback()
 
+    @mock.patch.object(SchedulerJobRunner, "_build_schedulable_tis_query", 
autospec=True)
+    def 
test_select_task_instances_to_queue_excludes_locked_rows_no_longer_scheduled(
+        self, mock_build_query, dag_maker, mock_executors, session, caplog
+    ):
+        with dag_maker(dag_id="locked_rows_state", session=session):
+            EmptyOperator(task_id="scheduled_in_bulk")
+            EmptyOperator(task_id="marked_success")
+        dag_run = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED, 
session=session)
+        tis = {ti.task_id: ti for ti in 
dag_run.get_task_instances(session=session)}
+        tis["marked_success"].state = State.SCHEDULED
+        session.flush()
+        # Update the rows but not the TIs in the session, so the session keeps 
no state for
+        # "scheduled_in_bulk" and SCHEDULED for "marked_success".
+        for task_id, state in (("scheduled_in_bulk", State.SCHEDULED), 
("marked_success", State.SUCCESS)):
+            session.execute(
+                update(TaskInstance)
+                .where(TaskInstance.id == tis[task_id].id)
+                .values(state=state)
+                .execution_options(synchronize_session=False)
+            )
+        # Returns both rows, as a locking read does for a row the ranking 
subquery saw as SCHEDULED in the
+        # statement's snapshot and that was marked success before the lock. 
SQLite cannot reproduce that.
+        mock_build_query.return_value = 
select(TaskInstance).where(TaskInstance.dag_id == "locked_rows_state")
+        self.job_runner = SchedulerJobRunner(job=Job())
+
+        with caplog.at_level(logging.DEBUG, 
logger="airflow.jobs.scheduler_job_runner"):
+            queued_tis = self.job_runner._select_task_instances_to_queue(
+                32, make_pool_stats(), set(), session=session
+            )
+
+        assert [ti.task_id for ti in queued_tis] == ["scheduled_in_bulk"]
+        assert {"event": "Skipped locked task instances that are no longer 
SCHEDULED", "count": 1} in caplog
+
+    @pytest.mark.backend("postgres", "mysql")
+    # The scheduler locks with SKIP LOCKED, so the bug needs the writer to 
commit after the statement takes
+    # its snapshot and before it locks the row, a window a test cannot time. 
With SKIP LOCKED patched out,
+    # the selection waits on the writer's lock instead, so the commit always 
lands in that window. The test
+    # covers how the locking read treats that commit, not the locking options 
the scheduler runs with.
+    @mock.patch(
+        "airflow.jobs.scheduler_job_runner.with_row_locks",
+        autospec=True,
+        side_effect=lambda query, **kwargs: with_row_locks(query, **{**kwargs, 
"skip_locked": False}),
+    )
+    def 
test_select_task_instances_to_queue_skips_a_task_instance_moved_out_of_scheduled_while_it_runs(
+        self, mock_with_row_locks, dag_maker, mock_executors, session
+    ):
+        with dag_maker(dag_id="state_changed_while_selecting", 
session=session):
+            EmptyOperator(task_id="unchanged")
+            EmptyOperator(task_id="marked_success")
+        dag_run = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED, 
session=session)
+        ti_ids = {}
+        for ti in dag_run.get_task_instances(session=session):
+            ti.state = State.SCHEDULED
+            ti_ids[ti.task_id] = ti.id
+        session.commit()
+        runner = SchedulerJobRunner(job=Job())
+        selector_ids: list[int] = []
+
+        def select_task_instances() -> list[str]:
+            with create_session(scoped=False) as selector:
+                selector_ids.append(_get_connection_id(selector))
+                # Held by the selector's session from before the update below, 
with their SCHEDULED state.
+                tis_in_session = selector.scalars(
+                    
select(TaskInstance).where(TaskInstance.id.in_(ti_ids.values()))
+                ).all()
+                assert {ti.id for ti in tis_in_session if ti in selector} == 
set(ti_ids.values())
+                queued_tis = runner._select_task_instances_to_queue(
+                    32, make_pool_stats(), set(), session=selector
+                )
+                selector.rollback()
+                return [ti.task_id for ti in queued_tis]
+
+        with ThreadPoolExecutor(max_workers=1) as pool, 
create_session(scoped=False) as writer:
+            writer_id = _get_connection_id(writer)
+            # The selection reads both task instances from its snapshot, then 
waits on these locks.
+            writer.scalars(
+                
select(TaskInstance.id).where(TaskInstance.id.in_(ti_ids.values())).with_for_update()
+            ).all()
+            future = pool.submit(select_task_instances)
+            deadline = time.monotonic() + 30
+            while not (selector_ids and _is_waiting_for_lock(selector_ids[0], 
writer_id)):
+                if future.done():
+                    pytest.fail(f"the selection did not wait on the writer's 
locks: {future.result()}")
+                assert time.monotonic() < deadline, "the selection did not 
wait on the writer's locks"
+                time.sleep(0.05)
+            # A user marks one of the task instances success while the 
selection runs.
+            writer.execute(
+                update(TaskInstance)
+                .where(TaskInstance.id == ti_ids["marked_success"])
+                .values(state=TaskInstanceState.SUCCESS)
+            )
+            writer.commit()
+
+            assert future.result(timeout=30) == ["unchanged"]
+
     def 
test_find_executable_task_instances_task_concurrency_per_dagrun_for_first(self, 
dag_maker):
         scheduler_job = Job()
         self.job_runner = SchedulerJobRunner(job=scheduler_job)

Reply via email to