xBis7 commented on code in PR #77152:
URL: https://github.com/apache/airflow/pull/77152#discussion_r4616056393
##########
airflow-core/src/airflow/timetables/simple.py:
##########
@@ -304,6 +304,62 @@ def next_dagrun_info(
DEFAULT_PARTITION_MAPPER = IdentityMapper()
+def _BACKFILL_GRAIN_STEP() -> dict[type, timedelta]:
+ """Rollup grains that a backfill can walk, with a step that always lands
in the next period."""
+ from airflow.partition_mappers.temporal import (
+ StartOfDayMapper,
+ StartOfMonthMapper,
+ StartOfQuarterMapper,
+ StartOfWeekMapper,
+ StartOfYearMapper,
+ )
+
+ return {
+ StartOfDayMapper: timedelta(days=1),
+ StartOfWeekMapper: timedelta(days=7),
+ StartOfMonthMapper: timedelta(days=32),
+ StartOfQuarterMapper: timedelta(days=93),
+ StartOfYearMapper: timedelta(days=367),
+ }
+
+
+def _iter_rollup_periods(mapper: Any, earliest: datetime, latest: datetime) ->
Iterable[tuple[str, datetime]]:
+ """Walk the periods of a rollup mapper and yield (key, partition date) for
those in the bounds."""
+ grain = mapper.upstream_mapper
+ step = _BACKFILL_GRAIN_STEP()[type(grain)]
+ # The walk is on local wall-clock values; each key's date is then resolved
by the mapper.
+ current =
grain.normalize(earliest.astimezone(grain._timezone).replace(tzinfo=None))
Review Comment:
In `trigger.py`
https://github.com/shahar1/airflow/blob/a00006b0c6b6a95eff5950886e403ca8347d030a/airflow-core/src/airflow/timetables/trigger.py#L498-L502
it gets the bound in UTC and then strips it and attaches the timezone from
the cron-timetable.
This one here, gets it in UTC and keeps it in UTC while below it compares
against partition dates in local timezone. If it changes and attaches the
mapper's timezone, the behavior stays consistent.
```suggestion
naive_earliest = earliest.replace(tzinfo=None)
earliest = timezone.make_aware(naive_earliest, grain._timezone)
latest = timezone.make_aware(latest.replace(tzinfo=None),
grain._timezone)
current = grain.normalize(naive_earliest)
```
##########
airflow-core/src/airflow/models/backfill.py:
##########
@@ -367,30 +399,379 @@ def _validate_backfill_params(
raise InvalidBackfillConf(str(e)) from e
-def _do_dry_run(
+def _is_asset_consumer(dag: SerializedDAG) -> bool:
+ from airflow.timetables.simple import PartitionedAssetTimetable
+
+ return isinstance(dag.timetable, PartitionedAssetTimetable)
+
+
+def _check_backfillable(dag: SerializedDAG, dag_id: str) -> None:
+ """Raise when the Dag's schedule cannot be backfilled."""
+ from airflow.timetables.simple import PartitionedAssetTimetable
+
+ timetable = dag.timetable
+ if timetable.periodic:
+ return
+ if not isinstance(timetable, PartitionedAssetTimetable):
+ raise DagNonPeriodicScheduleException(
+ f"{dag_id} has a non-periodic schedule that does not support
backfills"
+ )
+ from airflow.configuration import conf
+
+ if conf.getboolean("core", "multi_team", fallback=False):
+ # Which consumers may receive an event is decided when the event
arrives, with data
+ # that is not all stored. A backfill cannot repeat that decision for
old events.
+ raise ConsumerBackfillNotSupported(
+ f"{dag_id} is scheduled on partitioned assets; such a Dag cannot
be backfilled "
+ "when [core] multi_team is enabled"
+ )
+ if reason := timetable.backfill_unsupported_reason():
+ raise ConsumerBackfillNotSupported(
+ f"{dag_id} is scheduled on partitioned assets and cannot be
backfilled: {reason}"
+ )
+
+
+def _consumer_infos_for_keys(
+ dag: SerializedDAG, partition_keys: list[str], session: Session
+) -> list[DagRunInfo]:
+ """Resolve explicit partition keys of a Dag scheduled on partitioned
assets."""
+ from airflow.timetables.simple import PartitionedAssetTimetable
+
+ timetable = dag.timetable
+ if not isinstance(timetable, PartitionedAssetTimetable):
+ raise InvalidBackfillPartitionKeys(
+ f"partition_keys is only accepted for a Dag scheduled on
partitioned assets, not for {dag.dag_id}"
+ )
+ if timetable.backfill_selects_keys_from_events():
+ return _consumer_infos_from_events(dag, session,
partition_keys=partition_keys)
+ if not partition_keys:
+ raise InvalidBackfillPartitionKeys("partition_keys must not be empty")
+ try:
+ return _consumer_info_list(dag,
timetable.partition_dagrun_infos_for_keys(partition_keys))
+ except ValueError as e:
+ raise InvalidBackfillPartitionKeys(str(e)) from e
+
+
+def _resolve_bounds(
+ dag: SerializedDAG,
+ from_date: datetime | None,
+ to_date: datetime | None,
+ partition_keys: list[str] | None,
+ session: Session,
+) -> tuple[datetime, datetime]:
+ """
+ Return the date bounds of a backfill request.
+
+ A request has a date range or partition keys. For partition keys, the
bounds are the
+ earliest and the latest partition date of the keys.
+ """
+ if partition_keys is not None:
+ if from_date is not None or to_date is not None:
+ raise InvalidBackfillPartitionKeys("Give a date range or
partition_keys, not both.")
+ infos = _consumer_infos_for_keys(dag, partition_keys, session)
+ dates = [info.partition_date for info in infos if info.partition_date
is not None]
+ if not dates:
+ raise InvalidBackfillPartitionKeys("None of the partition keys has
a single partition date.")
+ return min(dates), max(dates)
+ if from_date is None or to_date is None:
+ raise InvalidBackfillDateRange("from_date and to_date are required
when partition_keys is not given.")
+ return from_date, to_date
+
+
+def _consumer_info_list(dag: SerializedDAG, infos: Iterable[DagRunInfo]) ->
list[DagRunInfo]:
+ """List the partitions of a Dag scheduled on partitioned assets, up to the
configured limit."""
+ from airflow.configuration import conf
+
+ limit = conf.getint("scheduler", "max_consumer_backfill_partitions",
fallback=1000)
+ dagrun_info_list: list[DagRunInfo] = []
+ try:
+ for info in infos:
+ if len(dagrun_info_list) >= limit:
+ raise TooManyBackfillPartitions(
+ f"The range selects more than {limit} partitions of Dag
{dag.dag_id}. "
+ "Use a shorter range."
+ )
+ dagrun_info_list.append(info)
+ except ValueError as e:
+ raise ConsumerBackfillNotSupported(
+ f"{dag.dag_id} is scheduled on partitioned assets and cannot be
backfilled: {e}"
+ ) from e
+ return dagrun_info_list
+
+
+_EVIDENCE_OF_THIS_REQUEST: ContextVar[
+ dict[tuple[str, str], tuple[list[Any], list[tuple[str, str]]]] | None
+] = ContextVar("consumer_backfill_evidence", default=None)
+
+
+@contextmanager
+def _one_evidence_lookup() -> Iterator[None]:
+ """
+ Look up the evidence of each partition once inside the block, and reuse it.
+
+ A request reads the evidence of a key more than once: for the date of the
key, for the
+ plan, and when the run is created. An event that arrives in between must
not give the
+ run other events than the ones its date was taken from.
+ """
+ token = _EVIDENCE_OF_THIS_REQUEST.set({})
+ try:
+ yield
+ finally:
+ _EVIDENCE_OF_THIS_REQUEST.reset(token)
+
+
+def _consumer_evidence(
+ dag: SerializedDAG, partition_key: str, session: Session
+) -> tuple[list[Any], list[tuple[str, str]]]:
+ """
+ Find the newest existing asset event for each input of a partition.
+
+ Returns the events, and the (asset name, input key) pairs that have no
event.
+ """
+ from airflow.models.asset import AssetEvent, AssetModel
+ from airflow.partition_mappers.base import is_rollup
+
+ cache = _EVIDENCE_OF_THIS_REQUEST.get()
+ if cache is not None and (dag.dag_id, partition_key) in cache:
+ return cache[dag.dag_id, partition_key]
+ timetable: Any = dag.timetable
+ events: list[Any] = []
+ missing: list[tuple[str, str]] = []
+ for unique_key, _ in timetable.asset_condition.iter_assets():
+ mapper = timetable.get_partition_mapper(name=unique_key.name,
uri=unique_key.uri)
+ asset_id = session.scalar(
+ select(AssetModel.id).where(
+ AssetModel.name == unique_key.name,
+ AssetModel.uri == unique_key.uri,
+ AssetModel.active.has(),
+ )
+ )
+ input_keys = sorted(mapper.to_upstream(partition_key)) if
is_rollup(mapper) else [partition_key]
+ for input_key in input_keys:
+ event = None
+ if asset_id is not None:
+ event = session.scalar(
Review Comment:
This is 1 query per `input_key` per `asset_id` and it's running inside one
API request. We could end up with hundreds of queries here.
If possible, it would be better to do one batch query per asset, that brings
the newest event for every key in one go.
##########
airflow-core/src/airflow/models/backfill.py:
##########
@@ -126,6 +137,22 @@ class InvalidBackfillDateRange(AirflowException):
"""
+class TooManyBackfillPartitions(InvalidBackfillDateRange):
Review Comment:
These new errors extend classes that seem unrelated. I'm assuming that this
was done to avoid modifying the part of the API server that handles them.
Wouldn't it be better to create a new generic class and make all of them
extend it? E.g. `InvalidConsumerBackfill`?
##########
airflow-core/src/airflow/models/backfill.py:
##########
@@ -367,30 +399,379 @@ def _validate_backfill_params(
raise InvalidBackfillConf(str(e)) from e
-def _do_dry_run(
+def _is_asset_consumer(dag: SerializedDAG) -> bool:
+ from airflow.timetables.simple import PartitionedAssetTimetable
+
+ return isinstance(dag.timetable, PartitionedAssetTimetable)
+
+
+def _check_backfillable(dag: SerializedDAG, dag_id: str) -> None:
+ """Raise when the Dag's schedule cannot be backfilled."""
+ from airflow.timetables.simple import PartitionedAssetTimetable
+
+ timetable = dag.timetable
+ if timetable.periodic:
+ return
+ if not isinstance(timetable, PartitionedAssetTimetable):
+ raise DagNonPeriodicScheduleException(
+ f"{dag_id} has a non-periodic schedule that does not support
backfills"
+ )
+ from airflow.configuration import conf
+
+ if conf.getboolean("core", "multi_team", fallback=False):
+ # Which consumers may receive an event is decided when the event
arrives, with data
+ # that is not all stored. A backfill cannot repeat that decision for
old events.
+ raise ConsumerBackfillNotSupported(
+ f"{dag_id} is scheduled on partitioned assets; such a Dag cannot
be backfilled "
+ "when [core] multi_team is enabled"
+ )
+ if reason := timetable.backfill_unsupported_reason():
+ raise ConsumerBackfillNotSupported(
+ f"{dag_id} is scheduled on partitioned assets and cannot be
backfilled: {reason}"
+ )
+
+
+def _consumer_infos_for_keys(
+ dag: SerializedDAG, partition_keys: list[str], session: Session
+) -> list[DagRunInfo]:
+ """Resolve explicit partition keys of a Dag scheduled on partitioned
assets."""
+ from airflow.timetables.simple import PartitionedAssetTimetable
+
+ timetable = dag.timetable
+ if not isinstance(timetable, PartitionedAssetTimetable):
+ raise InvalidBackfillPartitionKeys(
+ f"partition_keys is only accepted for a Dag scheduled on
partitioned assets, not for {dag.dag_id}"
+ )
+ if timetable.backfill_selects_keys_from_events():
+ return _consumer_infos_from_events(dag, session,
partition_keys=partition_keys)
+ if not partition_keys:
+ raise InvalidBackfillPartitionKeys("partition_keys must not be empty")
+ try:
+ return _consumer_info_list(dag,
timetable.partition_dagrun_infos_for_keys(partition_keys))
+ except ValueError as e:
+ raise InvalidBackfillPartitionKeys(str(e)) from e
+
+
+def _resolve_bounds(
+ dag: SerializedDAG,
+ from_date: datetime | None,
+ to_date: datetime | None,
+ partition_keys: list[str] | None,
+ session: Session,
+) -> tuple[datetime, datetime]:
+ """
+ Return the date bounds of a backfill request.
+
+ A request has a date range or partition keys. For partition keys, the
bounds are the
+ earliest and the latest partition date of the keys.
+ """
+ if partition_keys is not None:
+ if from_date is not None or to_date is not None:
+ raise InvalidBackfillPartitionKeys("Give a date range or
partition_keys, not both.")
+ infos = _consumer_infos_for_keys(dag, partition_keys, session)
+ dates = [info.partition_date for info in infos if info.partition_date
is not None]
+ if not dates:
+ raise InvalidBackfillPartitionKeys("None of the partition keys has
a single partition date.")
+ return min(dates), max(dates)
+ if from_date is None or to_date is None:
+ raise InvalidBackfillDateRange("from_date and to_date are required
when partition_keys is not given.")
+ return from_date, to_date
+
+
+def _consumer_info_list(dag: SerializedDAG, infos: Iterable[DagRunInfo]) ->
list[DagRunInfo]:
+ """List the partitions of a Dag scheduled on partitioned assets, up to the
configured limit."""
+ from airflow.configuration import conf
+
+ limit = conf.getint("scheduler", "max_consumer_backfill_partitions",
fallback=1000)
+ dagrun_info_list: list[DagRunInfo] = []
+ try:
+ for info in infos:
+ if len(dagrun_info_list) >= limit:
+ raise TooManyBackfillPartitions(
+ f"The range selects more than {limit} partitions of Dag
{dag.dag_id}. "
+ "Use a shorter range."
+ )
+ dagrun_info_list.append(info)
+ except ValueError as e:
+ raise ConsumerBackfillNotSupported(
+ f"{dag.dag_id} is scheduled on partitioned assets and cannot be
backfilled: {e}"
+ ) from e
+ return dagrun_info_list
+
+
+_EVIDENCE_OF_THIS_REQUEST: ContextVar[
+ dict[tuple[str, str], tuple[list[Any], list[tuple[str, str]]]] | None
+] = ContextVar("consumer_backfill_evidence", default=None)
+
+
+@contextmanager
+def _one_evidence_lookup() -> Iterator[None]:
+ """
+ Look up the evidence of each partition once inside the block, and reuse it.
+
+ A request reads the evidence of a key more than once: for the date of the
key, for the
+ plan, and when the run is created. An event that arrives in between must
not give the
+ run other events than the ones its date was taken from.
+ """
+ token = _EVIDENCE_OF_THIS_REQUEST.set({})
+ try:
+ yield
+ finally:
+ _EVIDENCE_OF_THIS_REQUEST.reset(token)
+
+
+def _consumer_evidence(
+ dag: SerializedDAG, partition_key: str, session: Session
+) -> tuple[list[Any], list[tuple[str, str]]]:
+ """
+ Find the newest existing asset event for each input of a partition.
+
+ Returns the events, and the (asset name, input key) pairs that have no
event.
+ """
+ from airflow.models.asset import AssetEvent, AssetModel
+ from airflow.partition_mappers.base import is_rollup
+
+ cache = _EVIDENCE_OF_THIS_REQUEST.get()
+ if cache is not None and (dag.dag_id, partition_key) in cache:
+ return cache[dag.dag_id, partition_key]
+ timetable: Any = dag.timetable
+ events: list[Any] = []
+ missing: list[tuple[str, str]] = []
+ for unique_key, _ in timetable.asset_condition.iter_assets():
+ mapper = timetable.get_partition_mapper(name=unique_key.name,
uri=unique_key.uri)
+ asset_id = session.scalar(
+ select(AssetModel.id).where(
+ AssetModel.name == unique_key.name,
+ AssetModel.uri == unique_key.uri,
+ AssetModel.active.has(),
Review Comment:
This will make the query skip inactive assets and report them as missing
which will be misleading for users.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]