jason810496 commented on code in PR #71072:
URL: https://github.com/apache/airflow/pull/71072#discussion_r3754876346
##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -2267,8 +2277,71 @@ def _create_dagruns_for_partitioned_asset_dags(self,
session: Session) -> set[st
)
).all()
if not pending_apdrs:
+ self._partition_cap_backlog_reported = False
return set()
+ if len(pending_apdrs) >= self._max_partition_dag_runs_per_loop:
+ backlog_total = (
+ session.scalar(
+ select(func.count())
+ .select_from(AssetPartitionDagRun)
+ .join(DagModel, DagModel.dag_id ==
AssetPartitionDagRun.target_dag_id)
+ .where(
+ AssetPartitionDagRun.created_dag_run_id.is_(None),
+ DagModel.is_stale.is_(False),
+ )
+ )
+ or 0
+ )
+ else:
+ backlog_total = len(pending_apdrs)
Review Comment:
IIUC, it seems equivalent. Would moving the resting part forward be more
more clear?
```suggestion
backlog_total = len(pending_apdrs)
self._partition_cap_backlog_reported = False
```
##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -2267,8 +2277,71 @@ def _create_dagruns_for_partitioned_asset_dags(self,
session: Session) -> set[st
)
).all()
if not pending_apdrs:
+ self._partition_cap_backlog_reported = False
return set()
+ if len(pending_apdrs) >= self._max_partition_dag_runs_per_loop:
+ backlog_total = (
+ session.scalar(
+ select(func.count())
+ .select_from(AssetPartitionDagRun)
+ .join(DagModel, DagModel.dag_id ==
AssetPartitionDagRun.target_dag_id)
+ .where(
+ AssetPartitionDagRun.created_dag_run_id.is_(None),
+ DagModel.is_stale.is_(False),
+ )
+ )
+ or 0
+ )
+ else:
+ backlog_total = len(pending_apdrs)
+ if backlog_total > self._max_partition_dag_runs_per_loop:
+ affected_dag_ids = {apdr.target_dag_id for apdr in pending_apdrs}
+ self.log.warning(
+ (
+ "Reached the per-tick cap on pending partitioned Dag runs;
the remaining backlog "
+ "will be evaluated over subsequent scheduler ticks"
+ ),
+ cap=self._max_partition_dag_runs_per_loop,
+ backlog_total=backlog_total,
+ dag_ids=affected_dag_ids,
+ )
+ # Edge-trigger the audit row: a persistent backlog re-hits this
branch every tick,
+ # and writing a `Log` row that often would flood the audit table.
Write it once per
+ # backlog episode; `_partition_cap_backlog_reported` is cleared
below once the
+ # backlog drains.
+ if not self._partition_cap_backlog_reported:
+ # A separate, independently-committed session is required
here: the caller
+ # (`_create_dagruns_for_dags`) runs under
`@retry_db_transaction`, which rolls
+ # back *session* on a `DBAPIError` — sharing that transaction
would silently
+ # discard this audit row along with the rest of the tick's
work.
+ # Invariant: this must run before *session* makes any writes
this tick, or the
+ # new connection's commit can lock-contend with it on SQLite.
+ try:
+ dag_ids_note = f"Affected dag_ids: {',
'.join(affected_dag_ids)}."
Review Comment:
Should we log those Dag IDs in the backlog? Instead of logging the pending
Dag IDs that we're going to create the DagRun.
##########
airflow-core/src/airflow/jobs/scheduler_job_runner.py:
##########
@@ -2267,8 +2277,71 @@ def _create_dagruns_for_partitioned_asset_dags(self,
session: Session) -> set[st
)
).all()
if not pending_apdrs:
+ self._partition_cap_backlog_reported = False
return set()
+ if len(pending_apdrs) >= self._max_partition_dag_runs_per_loop:
+ backlog_total = (
+ session.scalar(
+ select(func.count())
+ .select_from(AssetPartitionDagRun)
+ .join(DagModel, DagModel.dag_id ==
AssetPartitionDagRun.target_dag_id)
+ .where(
+ AssetPartitionDagRun.created_dag_run_id.is_(None),
+ DagModel.is_stale.is_(False),
+ )
+ )
+ or 0
+ )
+ else:
+ backlog_total = len(pending_apdrs)
+ if backlog_total > self._max_partition_dag_runs_per_loop:
+ affected_dag_ids = {apdr.target_dag_id for apdr in pending_apdrs}
+ self.log.warning(
+ (
+ "Reached the per-tick cap on pending partitioned Dag runs;
the remaining backlog "
+ "will be evaluated over subsequent scheduler ticks"
+ ),
+ cap=self._max_partition_dag_runs_per_loop,
+ backlog_total=backlog_total,
+ dag_ids=affected_dag_ids,
+ )
+ # Edge-trigger the audit row: a persistent backlog re-hits this
branch every tick,
+ # and writing a `Log` row that often would flood the audit table.
Write it once per
+ # backlog episode; `_partition_cap_backlog_reported` is cleared
below once the
+ # backlog drains.
+ if not self._partition_cap_backlog_reported:
+ # A separate, independently-committed session is required
here: the caller
+ # (`_create_dagruns_for_dags`) runs under
`@retry_db_transaction`, which rolls
+ # back *session* on a `DBAPIError` — sharing that transaction
would silently
+ # discard this audit row along with the rest of the tick's
work.
+ # Invariant: this must run before *session* makes any writes
this tick, or the
+ # new connection's commit can lock-contend with it on SQLite.
+ try:
+ dag_ids_note = f"Affected dag_ids: {',
'.join(affected_dag_ids)}."
+ with create_session(scoped=False) as audit_session:
+ audit_session.add(
+ Log(
+ event="partition dag run cap reached",
+ extra=(
+ f"The scheduler evaluated
{len(pending_apdrs)} pending partitioned Dag "
+ f"runs this tick, reaching the internal
per-tick cap of "
+ f"{self._max_partition_dag_runs_per_loop}.
The remaining backlog will "
+ f"be evaluated over subsequent ticks. A
total of {backlog_total} "
+ f"partitioned Dag runs are currently
eligible and pending across "
+ f"ticks. {dag_ids_note}"
+ ),
+ )
+ )
+ except (DBAPIError, exc.TimeoutError):
+ # Purely observational — must not fail or retry the tick's
actual DagRun
+ # creation work. Leave the flag False so the next tick
retries the write.
+ self.log.warning("Failed to write the partition Dag run
cap audit Log row", exc_info=True)
+ else:
+ self._partition_cap_backlog_reported = True
+ else:
+ self._partition_cap_backlog_reported = False
Review Comment:
```suggestion
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
