This is an automated email from the ASF dual-hosted git repository.
vincbeck pushed a commit to branch v3-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-4-test by this push:
new 049f4edb782 Fix scheduler queuing task instances that left the
scheduled state (#77247) (#77341)
049f4edb782 is described below
commit 049f4edb78232442c233b0562a11bdc2b734aaf2
Author: Vincent <[email protected]>
AuthorDate: Thu Nov 19 15:28:35 2026 -0500
Fix scheduler queuing task instances that left the scheduled state (#77247)
(#77341)
Since the query that picks task instances to queue ranks them in a subquery,
its scheduled-state filter only applies to the snapshot the statement
started
from. A locking read re-checks a row against the outer query only, on
PostgreSQL and on MySQL alike, so a task instance that a user marked
success or
failed, or that another scheduler skipped for a Dag run timeout, while the
query ran was still locked, returned and set to queued.
---
.../src/airflow/jobs/scheduler_job_runner.py | 19 +++-
airflow-core/tests/unit/jobs/test_scheduler_job.py | 117 ++++++++++++++++++++-
2 files changed, 133 insertions(+), 3 deletions(-)
diff --git a/airflow-core/src/airflow/jobs/scheduler_job_runner.py
b/airflow-core/src/airflow/jobs/scheduler_job_runner.py
index 49b16c7251d..fad235acb55 100644
--- a/airflow-core/src/airflow/jobs/scheduler_job_runner.py
+++ b/airflow-core/src/airflow/jobs/scheduler_job_runner.py
@@ -797,8 +797,23 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
timer.start()
try:
- locked_query = with_row_locks(query, of=TI, session=session,
skip_locked=True)
- task_instances_to_examine = session.scalars(locked_query).all()
+ # The ranking subquery reads the statement's snapshot, but a
locking read returns the latest
+ # version of the rows it locks, which can have left SCHEDULED
since. The state is read as a
+ # column so that the check sees the locked row whatever TIs
the session already holds. It is
+ # not checked in the outer WHERE: when the statistics show no
SCHEDULED rows, PostgreSQL
+ # then joins through the ti_state index instead of the primary
key.
+ locked_query = with_row_locks(
+ query.add_columns(TI.state), of=TI, session=session,
skip_locked=True
+ )
+ locked_rows = session.execute(locked_query).all()
+ task_instances_to_examine = [
+ ti for ti, state in locked_rows if state ==
TaskInstanceState.SCHEDULED
+ ]
+
+ if skipped := len(locked_rows) -
len(task_instances_to_examine):
+ self.log.debug(
+ "Skipped locked task instances that are no longer
SCHEDULED", count=skipped
+ )
if self.log.isEnabledFor(logging.DEBUG):
self.log.debug("Length of the tis to examine is %d",
len(task_instances_to_examine))
diff --git a/airflow-core/tests/unit/jobs/test_scheduler_job.py
b/airflow-core/tests/unit/jobs/test_scheduler_job.py
index 3c9b6070a88..edd05b81a09 100644
--- a/airflow-core/tests/unit/jobs/test_scheduler_job.py
+++ b/airflow-core/tests/unit/jobs/test_scheduler_job.py
@@ -39,7 +39,7 @@ import pendulum
import psutil
import pytest
import time_machine
-from sqlalchemy import delete, event as sa_event, func, inspect, select, update
+from sqlalchemy import delete, event as sa_event, func, inspect, select, text,
update
from sqlalchemy.dialects import mysql
from sqlalchemy.orm import joinedload
@@ -374,6 +374,26 @@ def make_pool_stats(
}
+def _get_connection_id(session: Session) -> int:
+ if session.get_bind().dialect.name == "postgresql":
+ return session.execute(text("SELECT pg_backend_pid()")).scalar_one()
+ return session.execute(text("SELECT CONNECTION_ID()")).scalar_one()
+
+
+def _is_waiting_for_lock(waiter_id: int, blocker_id: int) -> bool:
+ with settings.get_engine().connect() as probe:
+ if probe.dialect.name == "postgresql":
+ blocked = text("SELECT CAST(:blocker AS integer) =
ANY(pg_blocking_pids(:waiter))")
+ return bool(probe.execute(blocked, {"waiter": waiter_id,
"blocker": blocker_id}).scalar())
+ # Only that the waiter waits for a row lock: CI runs MySQL with
performance_schema off, so
+ # performance_schema.data_lock_waits cannot name the blocking
transaction there.
+ waiting = text(
+ "SELECT count(*) > 0 FROM information_schema.innodb_trx "
+ "WHERE trx_mysql_thread_id = :waiter AND trx_state = 'LOCK WAIT'"
+ )
+ return bool(probe.execute(waiting, {"waiter": waiter_id}).scalar())
+
+
def assign_executor_to_team(executor: MagicMock, team_name: str) -> None:
# ExecutorLoader.load_executor sets the team on both; the scheduler reads
it from the name.
executor.team_name = team_name
@@ -3704,6 +3724,101 @@ class TestSchedulerJob:
session.rollback()
+ @mock.patch.object(SchedulerJobRunner, "_build_schedulable_tis_query",
autospec=True)
+ def
test_select_task_instances_to_queue_excludes_locked_rows_no_longer_scheduled(
+ self, mock_build_query, dag_maker, mock_executors, session, caplog
+ ):
+ with dag_maker(dag_id="locked_rows_state", session=session):
+ EmptyOperator(task_id="scheduled_in_bulk")
+ EmptyOperator(task_id="marked_success")
+ dag_run = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED,
session=session)
+ tis = {ti.task_id: ti for ti in
dag_run.get_task_instances(session=session)}
+ tis["marked_success"].state = State.SCHEDULED
+ session.flush()
+ # Update the rows but not the TIs in the session, so the session keeps
no state for
+ # "scheduled_in_bulk" and SCHEDULED for "marked_success".
+ for task_id, state in (("scheduled_in_bulk", State.SCHEDULED),
("marked_success", State.SUCCESS)):
+ session.execute(
+ update(TaskInstance)
+ .where(TaskInstance.id == tis[task_id].id)
+ .values(state=state)
+ .execution_options(synchronize_session=False)
+ )
+ # Returns both rows, as a locking read does for a row the ranking
subquery saw as SCHEDULED in the
+ # statement's snapshot and that was marked success before the lock.
SQLite cannot reproduce that.
+ mock_build_query.return_value =
select(TaskInstance).where(TaskInstance.dag_id == "locked_rows_state")
+ self.job_runner = SchedulerJobRunner(job=Job())
+
+ with caplog.at_level(logging.DEBUG,
logger="airflow.jobs.scheduler_job_runner"):
+ queued_tis = self.job_runner._select_task_instances_to_queue(
+ 32, make_pool_stats(), set(), session=session
+ )
+
+ assert [ti.task_id for ti in queued_tis] == ["scheduled_in_bulk"]
+ assert {"event": "Skipped locked task instances that are no longer
SCHEDULED", "count": 1} in caplog
+
+ @pytest.mark.backend("postgres", "mysql")
+ # The scheduler locks with SKIP LOCKED, so the bug needs the writer to
commit after the statement takes
+ # its snapshot and before it locks the row, a window a test cannot time.
With SKIP LOCKED patched out,
+ # the selection waits on the writer's lock instead, so the commit always
lands in that window. The test
+ # covers how the locking read treats that commit, not the locking options
the scheduler runs with.
+ @mock.patch(
+ "airflow.jobs.scheduler_job_runner.with_row_locks",
+ autospec=True,
+ side_effect=lambda query, **kwargs: with_row_locks(query, **{**kwargs,
"skip_locked": False}),
+ )
+ def
test_select_task_instances_to_queue_skips_a_task_instance_moved_out_of_scheduled_while_it_runs(
+ self, mock_with_row_locks, dag_maker, mock_executors, session
+ ):
+ with dag_maker(dag_id="state_changed_while_selecting",
session=session):
+ EmptyOperator(task_id="unchanged")
+ EmptyOperator(task_id="marked_success")
+ dag_run = dag_maker.create_dagrun(run_type=DagRunType.SCHEDULED,
session=session)
+ ti_ids = {}
+ for ti in dag_run.get_task_instances(session=session):
+ ti.state = State.SCHEDULED
+ ti_ids[ti.task_id] = ti.id
+ session.commit()
+ runner = SchedulerJobRunner(job=Job())
+ selector_ids: list[int] = []
+
+ def select_task_instances() -> list[str]:
+ with create_session(scoped=False) as selector:
+ selector_ids.append(_get_connection_id(selector))
+ # Held by the selector's session from before the update below,
with their SCHEDULED state.
+ tis_in_session = selector.scalars(
+
select(TaskInstance).where(TaskInstance.id.in_(ti_ids.values()))
+ ).all()
+ assert {ti.id for ti in tis_in_session if ti in selector} ==
set(ti_ids.values())
+ queued_tis = runner._select_task_instances_to_queue(
+ 32, make_pool_stats(), set(), session=selector
+ )
+ selector.rollback()
+ return [ti.task_id for ti in queued_tis]
+
+ with ThreadPoolExecutor(max_workers=1) as pool,
create_session(scoped=False) as writer:
+ writer_id = _get_connection_id(writer)
+ # The selection reads both task instances from its snapshot, then
waits on these locks.
+ writer.scalars(
+
select(TaskInstance.id).where(TaskInstance.id.in_(ti_ids.values())).with_for_update()
+ ).all()
+ future = pool.submit(select_task_instances)
+ deadline = time.monotonic() + 30
+ while not (selector_ids and _is_waiting_for_lock(selector_ids[0],
writer_id)):
+ if future.done():
+ pytest.fail(f"the selection did not wait on the writer's
locks: {future.result()}")
+ assert time.monotonic() < deadline, "the selection did not
wait on the writer's locks"
+ time.sleep(0.05)
+ # A user marks one of the task instances success while the
selection runs.
+ writer.execute(
+ update(TaskInstance)
+ .where(TaskInstance.id == ti_ids["marked_success"])
+ .values(state=TaskInstanceState.SUCCESS)
+ )
+ writer.commit()
+
+ assert future.result(timeout=30) == ["unchanged"]
+
def
test_find_executable_task_instances_task_concurrency_per_dagrun_for_first(self,
dag_maker):
scheduler_job = Job()
self.job_runner = SchedulerJobRunner(job=scheduler_job)