uranusjr commented on code in PR #75751:
URL: https://github.com/apache/airflow/pull/75751#discussion_r4597998048
##########
task-sdk/src/airflow/sdk/execution_time/context.py:
##########
@@ -1132,44 +1134,44 @@ class AssetStateStoreAccessors:
For tasks with exactly one concrete inlet or outlet, the accessor methods
(``get``,
``set``, ``delete``, ``clear``, and their async counterparts ``aget``,
``aset``,
``adelete``, ``aclear``) can be called directly without subscripting.
+
+ Alias inlets are resolved to their concrete assets through the supervisor
on
+ first access. This is a blocking call, even if the first access is from an
+ async method.
"""
- def __init__(
- self, inlets: list, outlets: list | None = None, stats_tags: dict[str,
str] | None = None
- ) -> None:
- self._by_name: dict[str, AssetStateStoreAccessor] = {}
- self._by_uri: dict[str, AssetStateStoreAccessor] = {}
- self._stats_tags = stats_tags
+ _inlets: list
+ _outlets: list = attrs.field(factory=list)
+ _stats_tags: dict[str, str] | None = None
- for inlet in inlets:
+ @functools.cached_property
+ def _by_name(self) -> dict[str, AssetStateStoreAccessor]:
+ by_name: dict[str, AssetStateStoreAccessor] = {}
+ for inlet in self._inlets:
if isinstance(inlet, (Asset, AssetNameRef)):
- self._by_name[inlet.name] = AssetStateStoreAccessor(
- name=inlet.name, stats_tags=self._stats_tags
- )
- elif isinstance(inlet, AssetUriRef):
- self._by_uri[inlet.uri] =
AssetStateStoreAccessor(uri=inlet.uri, stats_tags=self._stats_tags)
+ by_name[inlet.name] = AssetStateStoreAccessor(name=inlet.name,
stats_tags=self._stats_tags)
elif isinstance(inlet, AssetAlias):
from airflow.sdk.execution_time.task_runner import
SUPERVISOR_COMMS
resp =
SUPERVISOR_COMMS.send(GetAssetsByAlias(alias_name=inlet.name))
Review Comment:
Good catch. I tweaked around but ultimately lazy-resolution is simply at
odds with async I/O methods, so I decided to entirely redo the whole thing with
another approach (PR description also updated to match). Instead of resolving
on-demand, the accessors now simply try to resolve at the right time and
refresh things accordingly. This addresses the KeyError issue we set out to
fix, but still keeps things eagerly enough to not conflict with async
requirements.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]