This is an automated email from the ASF dual-hosted git repository.
Lee-W pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 2dae3e4e161 Log and audit when the partitioned Dag run per-tick cap is
reached (#71072)
2dae3e4e161 is described below
commit 2dae3e4e161636ffb00f8c49c018b7ecf337831e
Author: Wei Lee <[email protected]>
AuthorDate: Mon Nov 16 17:06:49 2026 +0800
Log and audit when the partitioned Dag run per-tick cap is reached (#71072)
---
.../src/airflow/jobs/scheduler_job_runner.py | 90 ++-
airflow-core/tests/unit/jobs/test_scheduler_job.py | 643 ++++++++++++++++++++-
2 files changed, 726 insertions(+), 7 deletions(-)
diff --git a/airflow-core/src/airflow/jobs/scheduler_job_runner.py
b/airflow-core/src/airflow/jobs/scheduler_job_runner.py
index 7cc8f06dd0e..1c47da65f9e 100644
--- a/airflow-core/src/airflow/jobs/scheduler_job_runner.py
+++ b/airflow-core/src/airflow/jobs/scheduler_job_runner.py
@@ -178,6 +178,14 @@ MAX_PARTITION_DAG_RUNS_PER_LOOP = 500
# Max number of ADRQ rows deleted per statement, to keep the bind parameter
count bounded.
_ADRQ_DELETE_CHUNK_SIZE = 500
+# Caps how many dag_ids the cap log lists and how many per-Dag audit rows
+# are written per callback run. The backlog's distinct dag_id count is
unbounded, so
+# writing it straight into a log field or one DB row per Dag without a cap
risks
+# an unbounded payload. 20 is enough for an operator to diagnose which Dags are
+# backlogged without letting the log field or the per-run row count run away.
+# Internal constant, not a user setting, for the same reason as the constant
above.
+MAX_PARTITION_CAP_BACKLOG_DAG_IDS_LOGGED = 20
+
MAX_TI_REFILL_QUERIES_PER_LOOP = 3
"""
Max extra task instance queries per critical section to fill a partially
selected batch.
@@ -341,6 +349,22 @@ def _get_current_dr_task_concurrency(states:
Iterable[TaskInstanceState]) -> Sub
)
+def _build_pending_partitioned_apdr_filters() -> tuple[ColumnElement[bool],
...]:
+ """
+ Build the predicates selecting the pending APDRs the scheduler can act on.
+
+ Callers join ``DagModel`` on ``target_dag_id``. The per-tick fetch and the
periodic cap
+ backlog count share these predicates: counting rows the fetch can never
select produces a
+ false cap warning.
+ """
+ return (
+ AssetPartitionDagRun.created_dag_run_id.is_(None),
+ DagModel.is_paused.is_(False),
+ DagModel.is_draining.is_(False),
+ DagModel.is_stale.is_(False),
+ )
+
+
class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
"""
SchedulerJobRunner runs for a specific time interval and schedules jobs
that are ready to run.
@@ -400,6 +424,12 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
self._multi_team = conf.getboolean("core", "multi_team")
self._dag_tags_in_metrics = conf.getboolean("metrics",
"dag_tags_in_metrics", fallback=False)
self._max_partition_dag_runs_per_loop = MAX_PARTITION_DAG_RUNS_PER_LOOP
+ # Edge-triggers the "partition Dag run cap reached" audit Log rows:
the dag_ids whose row
+ # was committed during the current backlog episode. Cleared by
+ # `_report_partition_cap_backlog` when the backlog is at or below the
cap.
+ # Process-local and deliberately not de-duplicated across schedulers:
in an HA deployment
+ # each scheduler writes its own row, so a Dag gets one row per
scheduler per episode.
+ self._partition_cap_reported_dag_ids: set[str] = set()
self._dag_id_to_team_name: dict[str, str | None] = {}
self.executors: list[BaseExecutor] = executors if executors else
ExecutorLoader.init_executors()
@@ -2090,6 +2120,7 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
timers.call_regular_interval(60.0,
self._update_dag_run_state_for_paused_dags)
timers.call_regular_interval(5.0, self._finalize_draining_dags)
+ timers.call_regular_interval(60.0, self._report_partition_cap_backlog,
non_fatal=True)
timers.call_regular_interval(
conf.getfloat("scheduler", "parsing_cleanup_interval"),
cleanup_processor_metadata, non_fatal=True
)
@@ -2537,12 +2568,7 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
with_row_locks(
select(AssetPartitionDagRun)
.join(DagModel, DagModel.dag_id ==
AssetPartitionDagRun.target_dag_id)
- .where(
- AssetPartitionDagRun.created_dag_run_id.is_(None),
- DagModel.is_paused.is_(False),
- DagModel.is_draining.is_(False),
- DagModel.is_stale.is_(False),
- )
+ .where(*_build_pending_partitioned_apdr_filters())
.order_by(AssetPartitionDagRun.created_at,
AssetPartitionDagRun.id)
.limit(self._max_partition_dag_runs_per_loop),
of=AssetPartitionDagRun,
@@ -4332,6 +4358,58 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
.execution_options(synchronize_session="fetch")
)
+ def _report_partition_cap_backlog(self) -> None:
+ """Log and audit the Dags in the pending partitioned Dag run backlog
above the per-tick cap."""
+ with create_session() as session:
+ backlogs_per_dag = {
+ dag_id: count
+ for dag_id, count in session.execute(
+ select(AssetPartitionDagRun.target_dag_id, func.count())
+ .select_from(AssetPartitionDagRun)
+ .join(DagModel, DagModel.dag_id ==
AssetPartitionDagRun.target_dag_id)
+ .where(*_build_pending_partitioned_apdr_filters())
+ .group_by(AssetPartitionDagRun.target_dag_id)
+ )
+ }
+ backlog_total = sum(backlogs_per_dag.values())
+ if backlog_total <= self._max_partition_dag_runs_per_loop:
+ self._partition_cap_reported_dag_ids.clear()
+ return
+
+ sorted_dag_ids = sorted(backlogs_per_dag, key=lambda dag_id:
(-backlogs_per_dag[dag_id], dag_id))
+ displayed_dag_ids =
sorted_dag_ids[:MAX_PARTITION_CAP_BACKLOG_DAG_IDS_LOGGED]
+ new_dag_ids = [
+ dag_id for dag_id in displayed_dag_ids if dag_id not in
self._partition_cap_reported_dag_ids
+ ]
+ log_cap_reached = self.log.warning if new_dag_ids else
self.log.debug
+ log_cap_reached(
+ (
+ "Pending partitioned Dag runs exceed the per-tick cap;
only the oldest pending "
+ "runs up to the cap are evaluated per tick, newer ones
wait behind them"
+ ),
+ cap=self._max_partition_dag_runs_per_loop,
+ backlog_total=backlog_total,
+ dag_ids=displayed_dag_ids,
+ )
+ session.add_all(
+ Log(
+ event="partition Dag run cap reached",
+ dag_id=dag_id,
+ owner="scheduler",
+ owner_display_name="Scheduler",
+ extra=(
+ f"This Dag has {backlogs_per_dag[dag_id]} pending
partitioned Dag runs and "
+ f"{backlog_total} are pending across all Dags, above
the internal per-tick cap "
+ f"of {self._max_partition_dag_runs_per_loop}; pending
runs are counted whether "
+ f"or not their partitions are satisfied yet. Only the
oldest pending runs up "
+ f"to the cap are evaluated per tick, and newer ones
wait behind them."
+ ),
+ )
+ for dag_id in new_dag_ids
+ )
+ # Must follow the commit above: a failed write leaves these Dags
unreported so the next run retries.
+ self._partition_cap_reported_dag_ids.update(new_dag_ids)
+
@provide_session
def _update_asset_orphanage(self, *, session: Session = NEW_SESSION) ->
None:
"""
diff --git a/airflow-core/tests/unit/jobs/test_scheduler_job.py
b/airflow-core/tests/unit/jobs/test_scheduler_job.py
index d5e20a50ad5..f5469183f9f 100644
--- a/airflow-core/tests/unit/jobs/test_scheduler_job.py
+++ b/airflow-core/tests/unit/jobs/test_scheduler_job.py
@@ -41,6 +41,7 @@ import pytest
import time_machine
from sqlalchemy import Delete, delete, event as sa_event, func, inspect,
select, update
from sqlalchemy.dialects import mysql
+from sqlalchemy.exc import IntegrityError, InvalidRequestError
from sqlalchemy.orm import joinedload
from airflow import settings
@@ -184,6 +185,7 @@ from tests_common.test_utils.db import (
clear_db_deadline,
clear_db_import_errors,
clear_db_jobs,
+ clear_db_logs,
clear_db_pakl,
clear_db_pools,
clear_db_runs,
@@ -4801,6 +4803,20 @@ class TestSchedulerJob:
call_regular_interval.assert_any_call(mock.ANY, 123.0,
cleanup_processor_metadata, non_fatal=True)
+ @mock.patch.object(EventScheduler, "call_regular_interval", autospec=True)
+ def test_partition_cap_callback_registered_non_fatal(self,
call_regular_interval):
+ scheduler_job = Job()
+ scheduler_job.heartbeat = mock.create_autospec(scheduler_job.heartbeat)
+ self.job_runner = SchedulerJobRunner(
+ job=scheduler_job, num_runs=1,
executors=[MockExecutor(do_update=False)]
+ )
+
+ self.job_runner._run_scheduler_loop()
+
+ call_regular_interval.assert_any_call(
+ mock.ANY, 60.0, self.job_runner._report_partition_cap_backlog,
non_fatal=True
+ )
+
def test_adopt_or_reset_orphaned_tasks(self, dag_maker, session):
with dag_maker("test_execute_helper_reset_orphaned_tasks",
session=session):
op1 = EmptyOperator(task_id="op1")
@@ -14514,6 +14530,22 @@ def clear_asset_partition_rows() -> Iterator:
clear_db_pakl()
[email protected]
+def clear_audit_log_rows() -> Iterator:
+ """
+ Isolate ``Log`` rows for tests that assert on audit entries.
+
+ The partition-cap audit rows are written through an independent,
self-committing session, so
+ they escape the test session's rollback in both directions: leftovers from
earlier tests
+ would be counted here, and rows written here would leak into later ones.
+ """
+ clear_db_logs()
+
+ yield
+
+ clear_db_logs()
+
+
def _produce_and_register_asset_event(
*,
dag_id: str,
@@ -16438,6 +16470,14 @@ def
test_consumer_dag_listen_to_two_partitioned_asset_with_key_1_mapper(
assert asset_event.source_run_id == "test"
+# ``StructlogCapture.__contains__`` matches a plain string against the full
event text, so a
+# truncated message would make every ``not in caplog`` assertion vacuously
true.
+PARTITION_CAP_REACHED_EVENT = (
+ "Pending partitioned Dag runs exceed the per-tick cap; only the oldest
pending "
+ "runs up to the cap are evaluated per tick, newer ones wait behind them"
+)
+
+
def _make_n_satisfied_apdrs(
*,
consumer_dag_id: str,
@@ -16470,6 +16510,56 @@ def _make_n_satisfied_apdrs(
]
+def _make_n_consumer_dags_each_with_one_pending_apdr(
+ *,
+ name_prefix: str,
+ n: int,
+ session: Session,
+ dag_maker: DagMaker,
+) -> list[AssetPartitionDagRun]:
+ """
+ Build *n* distinct consumer Dags named
``cap-consumer-{name_prefix}-{i:02d}``, each with
+ exactly one pending, satisfied APDR, and return those APDRs in creation
order.
+
+ Used (instead of :func:`_make_n_satisfied_apdrs`, which numbers its
producer dag_id starting
+ at 1 on every call) when a test needs many distinct consumer Dags in one
go: a single shared
+ producer dag_id counter, namespaced by ``name_prefix``, avoids the
producer dag_id collisions
+ that calling :func:`_make_n_satisfied_apdrs` once per consumer Dag would
cause.
+ """
+ apdrs = []
+ for i in range(1, n + 1):
+ asset = Asset(name=f"asset-{name_prefix}-{i:02d}")
+ with dag_maker(
+ dag_id=f"cap-consumer-{name_prefix}-{i:02d}",
+ schedule=PartitionedAssetTimetable(assets=asset,
default_partition_mapper=IdentityMapper()),
+ session=session,
+ ):
+ EmptyOperator(task_id="hi")
+ session.commit()
+ apdrs.append(
+ _produce_and_register_asset_event(
+ dag_id=f"asset-event-producer-{name_prefix}-{i:02d}",
+ asset=asset,
+ partition_key=f"k{i:02d}",
+ session=session,
+ dag_maker=dag_maker,
+ )
+ )
+ return apdrs
+
+
+def _count_audit_log(session: Session) -> int:
+ return session.scalar(select(func.count()).where(Log.event == "partition
Dag run cap reached")) or 0
+
+
+def _get_audit_logs(session: Session) -> list[Log]:
+ return list(session.scalars(select(Log).where(Log.event == "partition Dag
run cap reached")))
+
+
+def _get_cap_reached_levels(caplog) -> list[str]:
+ return [e["log_level"] for e in caplog if e.get("event") ==
PARTITION_CAP_REACHED_EVENT]
+
+
@pytest.mark.need_serialized_dag
@pytest.mark.usefixtures("clear_asset_partition_rows")
def test_partition_cap_at_exactly_n_processes_all(dag_maker: DagMaker,
session: Session):
@@ -16501,7 +16591,7 @@ def
test_partition_cap_at_exactly_n_processes_all(dag_maker: DagMaker, session:
@pytest.mark.need_serialized_dag
[email protected]("clear_asset_partition_rows")
[email protected]("clear_asset_partition_rows", "clear_audit_log_rows")
def test_partition_cap_at_n_minus_one_leaves_one_pending(dag_maker: DagMaker,
session: Session):
"""
cap == APDR count - 1: oldest cap APDRs fire, the newest remains pending.
@@ -16539,6 +16629,557 @@ def
test_partition_cap_at_n_minus_one_leaves_one_pending(dag_maker: DagMaker, se
assert apdrs[1].created_dag_run_id is not None
assert apdrs[2].created_dag_run_id is None
assert partition_dags == {"cap-consumer-n-minus-one"}
+ # Reporting the backlog belongs to the periodic callback, not the tick.
+ assert _count_audit_log(session) == 0
+ assert runner._partition_cap_reported_dag_ids == set()
+
+
[email protected]_serialized_dag
[email protected]("clear_asset_partition_rows", "clear_audit_log_rows")
[email protected](
+ ("cap", "partition_keys", "expect_cap_hit"),
+ [
+ pytest.param(2, ["k1", "k2", "k3"], True, id="over-cap"),
+ pytest.param(3, ["k1", "k2"], False, id="under-cap"),
+ # Regression guard for the ``backlog_total > cap`` boundary:
``backlog_total == cap``
+ # must not be misreported as a backlog (that would be the "exactly
cap, nothing left"
+ # case wrongly reported as "more than cap, backlog remains").
+ pytest.param(3, ["k1", "k2", "k3"], False, id="exactly-cap"),
+ ],
+)
+def test_partition_cap_reporting(
+ dag_maker: DagMaker,
+ session: Session,
+ caplog,
+ cap: int,
+ partition_keys: list[str],
+ expect_cap_hit: bool,
+):
+ """Only a pending count strictly above the cap reports a backlog, via log
and audit `Log` row."""
+ suffix = f"{cap}-{len(partition_keys)}"
+ _make_n_satisfied_apdrs(
+ consumer_dag_id=f"cap-consumer-{suffix}",
+ asset=Asset(name=f"asset-cap-{suffix}"),
+ partition_keys=partition_keys,
+ session=session,
+ dag_maker=dag_maker,
+ )
+
+ runner = SchedulerJobRunner(
+ job=Job(job_type=SchedulerJobRunner.job_type),
executors=[MockExecutor(do_update=False)]
+ )
+ runner._max_partition_dag_runs_per_loop = cap
+ runner._report_partition_cap_backlog()
+
+ dag_id = f"cap-consumer-{suffix}"
+ audit_rows = session.scalars(select(Log).where(Log.event == "partition Dag
run cap reached")).all()
+ if expect_cap_hit:
+ assert {
+ "event": PARTITION_CAP_REACHED_EVENT,
+ "cap": cap,
+ "backlog_total": 3,
+ "dag_ids": [dag_id],
+ } in caplog
+ assert [row.event for row in audit_rows] == ["partition Dag run cap
reached"]
+ assert audit_rows[0].dag_id == dag_id
+ assert audit_rows[0].owner == "scheduler"
+ assert audit_rows[0].owner_display_name == "Scheduler"
+ extra = audit_rows[0].extra
+ assert extra is not None
+ assert "This Dag has 3 pending partitioned Dag runs" in extra
+ else:
+ assert PARTITION_CAP_REACHED_EVENT not in caplog
+ assert audit_rows == []
+
+
[email protected]_serialized_dag
[email protected]("clear_asset_partition_rows", "clear_audit_log_rows")
+def test_partition_cap_reporting_excludes_already_fired_decoy(dag_maker:
DagMaker, session: Session, caplog):
+ """
+ The ``backlog_total`` count query's ``WHERE`` clause must mirror the main
query's.
+
+ ``exactly-cap`` in :func:`test_partition_cap_reporting` cannot catch a
filter drift in the
+ count query (e.g. a dropped ``created_dag_run_id.is_(None)`` filter):
every row in that
+ table at that point is a genuine, unfired, non-stale APDR, so a looser
filter has nothing
+ extra to over-count. This test adds a decoy APDR that already fired
+ (``created_dag_run_id`` set) — a naive/unfiltered ``count()`` would
wrongly include it.
+ """
+ cap = 3
+ asset = Asset(name="asset-cap-decoy")
+ _make_n_satisfied_apdrs(
+ consumer_dag_id="cap-consumer-decoy",
+ asset=asset,
+ partition_keys=["k1", "k2", "k3"],
+ session=session,
+ dag_maker=dag_maker,
+ )
+ decoy = _produce_and_register_asset_event(
+ dag_id="asset-event-producer-decoy",
+ asset=asset,
+ partition_key="decoy",
+ session=session,
+ dag_maker=dag_maker,
+ )
+ fired_dag_run_id =
session.scalar(select(DagRun.id).order_by(DagRun.id.desc()).limit(1))
+ assert fired_dag_run_id is not None
+ decoy.created_dag_run_id = fired_dag_run_id
+ session.commit()
+
+ runner = SchedulerJobRunner(
+ job=Job(job_type=SchedulerJobRunner.job_type),
executors=[MockExecutor(do_update=False)]
+ )
+ runner._max_partition_dag_runs_per_loop = cap
+ runner._report_partition_cap_backlog()
+
+ assert PARTITION_CAP_REACHED_EVENT not in caplog
+ audit_rows = session.scalars(select(Log).where(Log.event == "partition Dag
run cap reached")).all()
+ assert audit_rows == []
+
+
[email protected]_serialized_dag
[email protected]("clear_asset_partition_rows", "clear_audit_log_rows")
[email protected]("attr", ["is_stale", "is_paused", "is_draining"])
+def test_partition_cap_reporting_excludes_inactive_dag_decoy(
+ attr: str, dag_maker: DagMaker, session: Session, caplog
+):
+ """
+ The ``backlog_total`` count query's ``WHERE`` clause must mirror the main
query's.
+
+ Mirrors :func:`test_partition_cap_reporting_excludes_already_fired_decoy`
for the fetch
+ query's Dag-state predicates: ``is_stale``, ``is_paused`` and
``is_draining`` must all be
+ ``False``. This test adds a decoy pending APDR whose target Dag became
inactive after the
+ APDR was written -- a count query missing that filter would wrongly
include it,
+ over-counting the backlog even though the fetch query never selects it.
+ """
+ cap = 3
+ suffix = attr.replace("_", "-")
+ _make_n_satisfied_apdrs(
+ consumer_dag_id=f"cap-consumer-inactive-decoy-{suffix}",
+ asset=Asset(name=f"asset-cap-inactive-decoy-{suffix}"),
+ partition_keys=["k1", "k2", "k3"],
+ session=session,
+ dag_maker=dag_maker,
+ )
+
+ inactive_consumer_dag_id = f"cap-consumer-inactive-decoy-{suffix}-inactive"
+ inactive_asset = Asset(name=f"asset-cap-inactive-decoy-{suffix}-inactive")
+ with dag_maker(
+ dag_id=inactive_consumer_dag_id,
+ schedule=PartitionedAssetTimetable(assets=inactive_asset,
default_partition_mapper=IdentityMapper()),
+ session=session,
+ ):
+ EmptyOperator(task_id="hi")
+ session.commit()
+
+ _produce_and_register_asset_event(
+ dag_id=f"asset-event-producer-inactive-decoy-{suffix}",
+ asset=inactive_asset,
+ partition_key="decoy",
+ session=session,
+ dag_maker=dag_maker,
+ )
+
+ # Flip the flag only after the APDR exists: AssetManager drops paused and
draining Dags
+ # before writing APDRs, so flag-first would produce no decoy at all.
+ dm = session.get(DagModel, inactive_consumer_dag_id)
+ assert dm is not None
+ setattr(dm, attr, True)
+ session.commit()
+
+ runner = SchedulerJobRunner(
+ job=Job(job_type=SchedulerJobRunner.job_type),
executors=[MockExecutor(do_update=False)]
+ )
+ runner._max_partition_dag_runs_per_loop = cap
+ runner._report_partition_cap_backlog()
+
+ assert PARTITION_CAP_REACHED_EVENT not in caplog
+ audit_rows = session.scalars(select(Log).where(Log.event == "partition Dag
run cap reached")).all()
+ assert audit_rows == []
+
+
[email protected]_serialized_dag
[email protected]("clear_asset_partition_rows", "clear_audit_log_rows")
+def test_partition_cap_reporting_lists_all_affected_dag_ids(dag_maker:
DagMaker, session: Session, caplog):
+ """
+ The log/audit ``dag_ids`` list surfaces every distinct dag_id in the
backlog, not just the
+ oldest-cap slice a tick would fetch.
+
+ Seven consumer Dags each get one satisfied APDR; with the cap at 6, a
tick's fetch would span
+ only six distinct dag_ids, but the seventh Dag's APDR is still part of the
backlog the count
+ sees — its dag_id must appear in both the log event and the audit row too.
+ """
+ apdrs = _make_n_consumer_dags_each_with_one_pending_apdr(
+ name_prefix="many", n=7, session=session, dag_maker=dag_maker
+ )
+ base = timezone.utcnow()
+ for i, apdr in enumerate(apdrs):
+ apdr.created_at = base + timedelta(seconds=i)
+ session.commit()
+
+ runner = SchedulerJobRunner(
+ job=Job(job_type=SchedulerJobRunner.job_type),
executors=[MockExecutor(do_update=False)]
+ )
+ runner._max_partition_dag_runs_per_loop = 6
+ runner._report_partition_cap_backlog()
+
+ # The dag_ids list is sorted by (-count, dag_id); all seven Dags tie on
count (one pending
+ # APDR each), so the tie-break is dag_id ascending — a deterministic,
reproducible order,
+ # not just membership. All seven Dags are expected, including the seventh
whose APDR is
+ # beyond the cap.
+ expected_dag_ids = sorted(f"cap-consumer-many-{i:02d}" for i in range(1,
8))
+ assert {
+ "event": PARTITION_CAP_REACHED_EVENT,
+ "cap": 6,
+ "backlog_total": 7,
+ "dag_ids": expected_dag_ids,
+ } in caplog
+
+ audit_rows = _get_audit_logs(session)
+ assert sorted(row.dag_id or "" for row in audit_rows) == expected_dag_ids
+ for row in audit_rows:
+ assert row.owner == "scheduler"
+ assert row.owner_display_name == "Scheduler"
+ assert row.extra is not None
+ assert "This Dag has 1 pending partitioned Dag runs" in row.extra
+ assert "7 are pending across all Dags" in row.extra
+
+
[email protected]_serialized_dag
[email protected]("clear_asset_partition_rows", "clear_audit_log_rows")
+def test_partition_cap_reporting_truncates_dag_id_list_beyond_max(
+ dag_maker: DagMaker, session: Session, caplog
+):
+ """
+ The log/audit ``dag_ids`` list is capped at
``MAX_PARTITION_CAP_BACKLOG_DAG_IDS_LOGGED``,
+ ordered by descending per-dag pending count with a dag_id tie-break — not
alphabetically
+ truncated — and only the displayed Dags get an audit row.
+ """
+ _make_n_consumer_dags_each_with_one_pending_apdr(
+ name_prefix="trunc-low", n=20, session=session, dag_maker=dag_maker
+ )
+
+ high_asset = Asset(name="asset-cap-trunc-zz-high")
+ with dag_maker(
+ dag_id="cap-consumer-trunc-zz-high",
+ schedule=PartitionedAssetTimetable(assets=high_asset,
default_partition_mapper=IdentityMapper()),
+ session=session,
+ ):
+ EmptyOperator(task_id="hi")
+ session.commit()
+ for key in ["high-1", "high-2", "high-3"]:
+ _produce_and_register_asset_event(
+ dag_id=f"asset-event-producer-trunc-zz-high-{key}",
+ asset=high_asset,
+ partition_key=key,
+ session=session,
+ dag_maker=dag_maker,
+ )
+
+ runner = SchedulerJobRunner(
+ job=Job(job_type=SchedulerJobRunner.job_type),
executors=[MockExecutor(do_update=False)]
+ )
+ runner._max_partition_dag_runs_per_loop = 5
+ runner._report_partition_cap_backlog()
+
+ expected_dag_ids = ["cap-consumer-trunc-zz-high"] + [
+ f"cap-consumer-trunc-low-{i:02d}" for i in range(1, 20)
+ ]
+ assert {
+ "event": PARTITION_CAP_REACHED_EVENT,
+ "cap": 5,
+ "backlog_total": 23,
+ "dag_ids": expected_dag_ids,
+ } in caplog
+
+ audit_rows = _get_audit_logs(session)
+ assert len(audit_rows) == 20
+ assert sorted(row.dag_id or "" for row in audit_rows) ==
sorted(expected_dag_ids)
+ assert "cap-consumer-trunc-low-20" not in {row.dag_id for row in
audit_rows}
+ high_row = next(row for row in audit_rows if row.dag_id ==
"cap-consumer-trunc-zz-high")
+ assert high_row.extra is not None
+ assert "This Dag has 3 pending partitioned Dag runs" in high_row.extra
+
+
[email protected]_serialized_dag
[email protected]("clear_asset_partition_rows", "clear_audit_log_rows")
+def test_partition_cap_reporting_at_exactly_max_dag_ids_no_truncation(
+ dag_maker: DagMaker, session: Session, caplog
+):
+ """
+ Cap-boundary pair to
:func:`test_partition_cap_reporting_truncates_dag_id_list_beyond_max`:
+ when the backlog's distinct dag_id count equals
``MAX_PARTITION_CAP_BACKLOG_DAG_IDS_LOGGED``
+ exactly, no dag_id is dropped and every Dag gets an audit row.
+ """
+ _make_n_consumer_dags_each_with_one_pending_apdr(
+ name_prefix="atmax", n=20, session=session, dag_maker=dag_maker
+ )
+
+ runner = SchedulerJobRunner(
+ job=Job(job_type=SchedulerJobRunner.job_type),
executors=[MockExecutor(do_update=False)]
+ )
+ runner._max_partition_dag_runs_per_loop = 5
+ runner._report_partition_cap_backlog()
+
+ expected_dag_ids = sorted(f"cap-consumer-atmax-{i:02d}" for i in range(1,
21))
+ assert {
+ "event": PARTITION_CAP_REACHED_EVENT,
+ "cap": 5,
+ "backlog_total": 20,
+ "dag_ids": expected_dag_ids,
+ } in caplog
+
+ audit_rows = _get_audit_logs(session)
+ assert sorted(row.dag_id or "" for row in audit_rows) == expected_dag_ids
+
+
[email protected]_serialized_dag
[email protected]("clear_asset_partition_rows", "clear_audit_log_rows")
+def test_partition_cap_backlog_audit_row_written_once_per_episode(
+ dag_maker: DagMaker, session: Session, caplog
+):
+ """
+ The cap-reached audit rows are written once per backlog episode, not once
per callback run.
+
+ A persistent backlog stays above the cap on every run; only the first run
of the episode
+ should write the audit rows. Draining below the cap and then exceeding it
again starts a new
+ episode and writes them again.
+ """
+ asset = Asset(name="asset-cap-episode")
+ apdrs = _make_n_satisfied_apdrs(
+ consumer_dag_id="cap-consumer-episode",
+ asset=asset,
+ partition_keys=["k1", "k2", "k3", "k4", "k5"],
+ session=session,
+ dag_maker=dag_maker,
+ )
+ base = timezone.utcnow()
+ for i, apdr in enumerate(apdrs):
+ apdr.created_at = base + timedelta(seconds=i)
+ session.commit()
+
+ runner = SchedulerJobRunner(
+ job=Job(job_type=SchedulerJobRunner.job_type),
executors=[MockExecutor(do_update=False)]
+ )
+ runner._max_partition_dag_runs_per_loop = 2
+ caplog.set_level("DEBUG",
logger="airflow.jobs.scheduler_job_runner.SchedulerJobRunner")
+
+ runner._report_partition_cap_backlog() # run 1: 5 pending, cap 2
+ assert _get_cap_reached_levels(caplog) == ["warning"]
+
+ runner._report_partition_cap_backlog() # run 2: still 5 pending
+ assert _count_audit_log(session) == 1, (
+ "backlog persisted across two runs; audit rows must only be written
once"
+ )
+ assert [row.dag_id for row in _get_audit_logs(session)] ==
["cap-consumer-episode"]
+ assert _get_cap_reached_levels(caplog) == ["warning", "debug"]
+
+ runner._create_dagruns_for_partitioned_asset_dags(session=session) # 5 ->
3 pending
+ runner._create_dagruns_for_partitioned_asset_dags(session=session) # 3 ->
1 pending
+ runner._report_partition_cap_backlog() # run 3: 1 pending, below the cap
+ assert _count_audit_log(session) == 1, "draining below the cap must not
write any more audit rows"
+ assert runner._partition_cap_reported_dag_ids == set()
+ assert _get_cap_reached_levels(caplog) == ["warning", "debug"]
+
+ # A fresh backlog episode for the same consumer: three more satisfied
APDRs push
+ # pending count back above the cap.
+ new_apdrs = [
+ _produce_and_register_asset_event(
+ dag_id=f"asset-event-producer-episode-{i}",
+ asset=asset,
+ partition_key=key,
+ session=session,
+ dag_maker=dag_maker,
+ )
+ for i, key in enumerate(["k6", "k7", "k8"], start=1)
+ ]
+ base2 = timezone.utcnow()
+ for i, apdr in enumerate(new_apdrs):
+ apdr.created_at = base2 + timedelta(seconds=i)
+ session.commit()
+
+ runner._report_partition_cap_backlog() # run 4: new episode, 4 pending
+ assert _count_audit_log(session) == 2, (
+ "a fresh backlog episode after draining should write audit rows again"
+ )
+ assert _get_cap_reached_levels(caplog) == ["warning", "debug", "warning"]
+
+
[email protected]_serialized_dag
[email protected]("clear_asset_partition_rows", "clear_audit_log_rows")
+def test_partition_cap_dag_joining_mid_episode_gets_audit_row(dag_maker:
DagMaker, session: Session, caplog):
+ """
+ A Dag that enters the backlog after the episode started still gets its own
audit row.
+
+ The edge trigger is per Dag: a Dag already recorded in this episode is not
written again,
+ but one that first appears in the displayed Dag ids on a later run is
written on that run,
+ so its audit page is not left empty for the rest of the episode.
+ """
+ asset = Asset(name="asset-cap-mid-episode")
+ apdrs = _make_n_satisfied_apdrs(
+ consumer_dag_id="cap-consumer-mid-episode",
+ asset=asset,
+ partition_keys=["k1", "k2", "k3", "k4", "k5", "k6"],
+ session=session,
+ dag_maker=dag_maker,
+ )
+ base = timezone.utcnow()
+ for i, apdr in enumerate(apdrs):
+ apdr.created_at = base + timedelta(seconds=i)
+ session.commit()
+
+ runner = SchedulerJobRunner(
+ job=Job(job_type=SchedulerJobRunner.job_type),
executors=[MockExecutor(do_update=False)]
+ )
+ runner._max_partition_dag_runs_per_loop = 2
+ caplog.set_level("DEBUG",
logger="airflow.jobs.scheduler_job_runner.SchedulerJobRunner")
+
+ runner._report_partition_cap_backlog() # run 1: 6 pending, cap 2
+ assert _get_cap_reached_levels(caplog) == ["warning"]
+ assert [row.dag_id for row in _get_audit_logs(session)] ==
["cap-consumer-mid-episode"]
+
+ _make_n_consumer_dags_each_with_one_pending_apdr(
+ name_prefix="mid-episode-joiner", n=1, session=session,
dag_maker=dag_maker
+ )
+
+ runner._report_partition_cap_backlog() # run 2: 6 + 1 pending, over cap
+ assert _get_cap_reached_levels(caplog) == ["warning", "warning"]
+ audit_rows = sorted(_get_audit_logs(session), key=lambda row: row.dag_id
or "")
+ assert [row.dag_id for row in audit_rows] == [
+ "cap-consumer-mid-episode",
+ "cap-consumer-mid-episode-joiner-01",
+ ]
+ assert "This Dag has 1 pending partitioned Dag runs" in
(audit_rows[1].extra or "")
+
+ runner._report_partition_cap_backlog() # run 3: no new Dag joined
+ assert _get_cap_reached_levels(caplog) == ["warning", "warning", "debug"]
+ assert _count_audit_log(session) == 2, "no new Dag joined, so no further
audit rows"
+
+
[email protected]_serialized_dag
[email protected]("clear_asset_partition_rows", "clear_audit_log_rows")
+def test_partition_cap_backlog_reset_on_full_drain_to_empty(dag_maker:
DagMaker, session: Session, caplog):
+ """
+ Regression test: the reported Dag ids must clear even when the backlog
drains to
+ literally zero pending APDRs, not just when it drops to some
smaller-than-cap-but-still
+ -nonzero count.
+
+ Otherwise a brand new backlog that re-crosses the cap is wrongly logged at
``debug`` and
+ skips writing audit rows again.
+ """
+ asset = Asset(name="asset-cap-drain-empty")
+ apdrs = _make_n_satisfied_apdrs(
+ consumer_dag_id="cap-consumer-drain-empty",
+ asset=asset,
+ partition_keys=["k1", "k2", "k3"],
+ session=session,
+ dag_maker=dag_maker,
+ )
+ base = timezone.utcnow()
+ for i, apdr in enumerate(apdrs):
+ apdr.created_at = base + timedelta(seconds=i)
+ session.commit()
+
+ runner = SchedulerJobRunner(
+ job=Job(job_type=SchedulerJobRunner.job_type),
executors=[MockExecutor(do_update=False)]
+ )
+ runner._max_partition_dag_runs_per_loop = 2
+ caplog.set_level("DEBUG",
logger="airflow.jobs.scheduler_job_runner.SchedulerJobRunner")
+
+ apdr_ids = [apdr.id for apdr in apdrs]
+ runner._report_partition_cap_backlog() # run 1: 3 pending, cap 2 -> over
cap
+ assert _get_cap_reached_levels(caplog) == ["warning"]
+ assert _count_audit_log(session) == 1
+ assert runner._partition_cap_reported_dag_ids ==
{"cap-consumer-drain-empty"}
+
+ runner._create_dagruns_for_partitioned_asset_dags(session=session) #
fires the oldest two
+ # The callback's own `create_session()` closed the shared scoped session,
detaching the
+ # earlier instances.
+ apdrs =
list(session.scalars(select(AssetPartitionDagRun).where(AssetPartitionDagRun.id.in_(apdr_ids))))
+ apdrs.sort(key=lambda apdr: apdr.created_at)
+ assert apdrs[0].created_dag_run_id is not None
+ assert apdrs[1].created_dag_run_id is not None
+ assert apdrs[2].created_dag_run_id is None
+ # Resolve the one leftover APDR so the backlog is literally empty.
+ apdrs[2].created_dag_run_id = apdrs[0].created_dag_run_id
+ session.commit()
+
+ runner._report_partition_cap_backlog() # run 2: 0 pending
+ assert _count_audit_log(session) == 1, "the zero-pending run itself must
not write any more audit rows"
+ assert runner._partition_cap_reported_dag_ids == set()
+
+ # A fresh backlog episode for the same consumer Dag: three more satisfied
APDRs push the
+ # pending count back above the cap.
+ new_apdrs = [
+ _produce_and_register_asset_event(
+ dag_id=f"asset-event-producer-drain-empty-{i}",
+ asset=asset,
+ partition_key=key,
+ session=session,
+ dag_maker=dag_maker,
+ )
+ for i, key in enumerate(["k4", "k5", "k6"], start=1)
+ ]
+ base2 = timezone.utcnow()
+ for i, apdr in enumerate(new_apdrs):
+ apdr.created_at = base2 + timedelta(seconds=i)
+ session.commit()
+
+ runner._report_partition_cap_backlog() # run 3: new episode, 3 pending
+ assert _get_cap_reached_levels(caplog) == ["warning", "warning"], (
+ "a new episode after a full drain-to-zero must warn, not silently log
at debug"
+ )
+ assert _count_audit_log(session) == 2, (
+ "a new episode after a full drain-to-zero must write audit rows again"
+ )
+
+
[email protected]_serialized_dag
[email protected]("clear_asset_partition_rows", "clear_audit_log_rows")
[email protected](
+ "audit_write_error",
+ [
+ pytest.param(IntegrityError("INSERT INTO log", {},
Exception("duplicate key")), id="dbapi_error"),
+ pytest.param(InvalidRequestError("session already flushing"),
id="non_dbapi_sqlalchemy_error"),
+ ],
+)
+def test_partition_cap_audit_write_failure_leaves_dag_unreported(
+ dag_maker: DagMaker, session: Session, audit_write_error: Exception
+):
+ """
+ A failed audit write must not mark the Dag as reported, so the next run
retries it.
+
+ The set is only updated after the session block commits; the failure
propagates to
+ ``call_regular_interval(..., non_fatal=True)``, which swallows it.
+ """
+ _make_n_satisfied_apdrs(
+ consumer_dag_id="cap-consumer-audit-failure",
+ asset=Asset(name="asset-cap-audit-failure"),
+ partition_keys=["k1", "k2", "k3"],
+ session=session,
+ dag_maker=dag_maker,
+ )
+
+ runner = SchedulerJobRunner(
+ job=Job(job_type=SchedulerJobRunner.job_type),
executors=[MockExecutor(do_update=False)]
+ )
+ runner._max_partition_dag_runs_per_loop = 1
+
+ failing_session_cm = MagicMock()
+ failing_session_cm.__enter__.return_value = MagicMock(spec=session)
+ failing_session_cm.__enter__.return_value.execute.return_value =
[("cap-consumer-audit-failure", 3)]
+ failing_session_cm.__exit__.side_effect = audit_write_error
+
+ with mock.patch("airflow.jobs.scheduler_job_runner.create_session",
return_value=failing_session_cm):
+ with pytest.raises(type(audit_write_error)):
+ runner._report_partition_cap_backlog()
+
+ assert runner._partition_cap_reported_dag_ids == set()
+ assert _count_audit_log(session) == 0
+
+ runner._report_partition_cap_backlog()
+
+ assert runner._partition_cap_reported_dag_ids ==
{"cap-consumer-audit-failure"}
+ assert [row.dag_id for row in _get_audit_logs(session)] ==
["cap-consumer-audit-failure"]
def _set_asset_active(*, name: str, uri: str, session: Session, active: bool)
-> None: