jason810496 commented on code in PR #76825:
URL: https://github.com/apache/airflow/pull/76825#discussion_r4585452636
##########
airflow-core/tests/unit/api_fastapi/execution_api/versions/head/test_xcoms.py:
##########
@@ -1380,6 +1381,36 @@ def
test_parsing_attempt_id_is_not_treated_as_an_archived_task(
assert "XCom" in response.json()["detail"]["message"]
+class TestTaskSdkClientGetXCom:
+ """The Task SDK client reads XComs from this tree's Execution API as both
sides intend."""
+
+ @pytest.fixture(autouse=True)
+ def dag_run(self, dag_maker, session):
+ with dag_maker(dag_id="dag", serialized=True):
+ EmptyOperator(task_id="plain")
+ PythonOperator.partial(task_id="mapped",
python_callable=str).expand(op_args=[[1], [2]])
+ dag_run = dag_maker.create_dagrun(run_id="runid")
+ for ti in dag_run.get_task_instances(session=session):
+ value = "plain" if ti.task_id == "plain" else ti.region_index
+ session.add(XComModelV2(key=XCOM_RETURN_KEY, value=value,
task_instance_id=ti.id))
+ session.commit()
+ return dag_run
+
+ @pytest.mark.parametrize(
+ ("task_id", "map_index", "expected"),
+ [
+ pytest.param("mapped", None, [0, 1], id="mapped-omitted"),
+ pytest.param("mapped", 1, 1, id="mapped-one"),
+ pytest.param("plain", None, "plain", id="unmapped-omitted"),
+ ],
+ )
+ def test_get(self, task_id, map_index, expected):
+ client = Client(base_url=None, token="", dry_run=True,
transport=InProcessExecutionAPI().transport)
+ client.base_url = "http://in-process.invalid/"
Review Comment:
Optional nit: Maybe we could switch to `task_sdk_client` fixture as
https://github.com/apache/airflow/pull/77146 just got merged before this one.
##########
task-sdk/src/airflow/sdk/api/client.py:
##########
@@ -668,35 +668,80 @@ def get(
*,
previous_iteration: bool = False,
) -> XComResponse:
- """Get a XCom value from the API server."""
+ """
+ Get a XCom value from the API server.
+
+ Any *map_index* is sent as given. An omitted one reads the task's
result: its value, or for a
+ mapped task the values of all its instances as a list.
+ """
params: dict[str, str | int] = {}
if previous_iteration:
params["previous_iteration"] = True
- if map_index is not None and map_index >= 0:
- params.update({"map_index": map_index})
+ if map_index is not None:
+ params["map_index"] = map_index
if include_prior_dates:
params.update({"include_prior_dates": include_prior_dates})
try:
resp = self.client.get(_path("xcoms", dag_id, run_id, task_id,
tail=key), params=params)
except ServerResponseError as e:
- if e.response.status_code == HTTPStatus.NOT_FOUND:
- log.debug(
- "XCom not found",
- dag_id=dag_id,
- run_id=run_id,
- task_id=task_id,
- key=key,
- map_index=map_index,
- detail=e.detail,
- status_code=e.response.status_code,
- )
- # Airflow 2.x just ignores the absence of an XCom and moves on
with a return value of None
- # Hence returning with key as `key` and value as `None`, so
that the message is sent back to task runner
- # and the default value of None in xcom_pull is used.
- return XComResponse(key=key, value=None)
- raise
+ if e.response.status_code != HTTPStatus.NOT_FOUND:
+ raise
+ log.debug(
+ "XCom not found",
+ dag_id=dag_id,
+ run_id=run_id,
+ task_id=task_id,
+ key=key,
+ map_index=map_index,
+ detail=e.detail,
+ status_code=e.response.status_code,
+ )
+ value = self._get_mapped_values_or_none(
+ dag_id,
+ run_id,
+ task_id,
+ key,
+ map_index,
+ include_prior_dates=include_prior_dates,
+ previous_iteration=previous_iteration,
+ )
Review Comment:
Nit: this logs "XCom not found" even when the slice then finds the mapped
values.
How about logging it only on a real miss?
```suggestion
value = self._get_mapped_values_or_none(
dag_id,
run_id,
task_id,
key,
map_index,
include_prior_dates=include_prior_dates,
previous_iteration=previous_iteration,
)
if value is None:
log.debug(
"XCom not found",
dag_id=dag_id,
run_id=run_id,
task_id=task_id,
key=key,
map_index=map_index,
detail=e.detail,
status_code=e.response.status_code,
)
```
##########
task-sdk/src/airflow/sdk/api/client.py:
##########
@@ -668,35 +668,80 @@ def get(
*,
previous_iteration: bool = False,
) -> XComResponse:
- """Get a XCom value from the API server."""
+ """
+ Get a XCom value from the API server.
+
+ Any *map_index* is sent as given. An omitted one reads the task's
result: its value, or for a
+ mapped task the values of all its instances as a list.
+ """
params: dict[str, str | int] = {}
if previous_iteration:
params["previous_iteration"] = True
- if map_index is not None and map_index >= 0:
- params.update({"map_index": map_index})
+ if map_index is not None:
+ params["map_index"] = map_index
if include_prior_dates:
params.update({"include_prior_dates": include_prior_dates})
try:
resp = self.client.get(_path("xcoms", dag_id, run_id, task_id,
tail=key), params=params)
except ServerResponseError as e:
- if e.response.status_code == HTTPStatus.NOT_FOUND:
- log.debug(
- "XCom not found",
- dag_id=dag_id,
- run_id=run_id,
- task_id=task_id,
- key=key,
- map_index=map_index,
- detail=e.detail,
- status_code=e.response.status_code,
- )
- # Airflow 2.x just ignores the absence of an XCom and moves on
with a return value of None
- # Hence returning with key as `key` and value as `None`, so
that the message is sent back to task runner
- # and the default value of None in xcom_pull is used.
- return XComResponse(key=key, value=None)
- raise
+ if e.response.status_code != HTTPStatus.NOT_FOUND:
+ raise
+ log.debug(
+ "XCom not found",
+ dag_id=dag_id,
+ run_id=run_id,
+ task_id=task_id,
+ key=key,
+ map_index=map_index,
+ detail=e.detail,
+ status_code=e.response.status_code,
+ )
+ value = self._get_mapped_values_or_none(
+ dag_id,
+ run_id,
+ task_id,
+ key,
+ map_index,
+ include_prior_dates=include_prior_dates,
+ previous_iteration=previous_iteration,
+ )
+ # A missing XCom is not an error, as in Airflow 2; xcom_pull then
returns its default.
+ return XComResponse(key=key, value=value)
return XComResponse.model_validate_json(resp.read())
+ def _get_mapped_values_or_none(
+ self,
+ dag_id: str,
+ run_id: str,
+ task_id: str,
+ key: str,
+ map_index: int | None,
+ *,
+ include_prior_dates: bool,
+ previous_iteration: bool,
+ ) -> list[JsonValue] | None:
+ """
+ Read a mapped task's values from the slice endpoint.
+
+ This is for a server that reads map index -1 for an omitted map index,
+ and so finds nothing for a mapped task. A server that resolves it by
+ how the task is defined never answers a mapped task as not found.
+ """
+ if map_index is not None:
+ return None
Review Comment:
With `include_prior_dates`, the slice returns rows from every earlier run,
ordered by map index only. Runs interleave: `[run1#0, run2#0, run1#1, ...]`.
The docstring still says the latest matching XCom is returned. Should the
fallback skip prior-date reads?
```suggestion
if map_index is not None or include_prior_dates:
return None
```
##########
task-sdk/src/airflow/sdk/bases/xcom.py:
##########
@@ -502,8 +503,10 @@ def get_one(
:param dag_id: Dag ID to pull the XCom from.
:param task_id: Task ID to pull the XCom from.
:param map_index: Map index of the task instance to pull the XCom from.
- *None* (default) pulls the XCom of a non-mapped task, which has
- map index ``-1``.
+ *None* (default) pulls the task's result: the XCom of a non-mapped
+ task, or for a mapped task the XComs of all its instances as a
+ list, the same values ``get_all`` returns. Airflow 3.4 returns
+ *None* instead of an empty list.
Review Comment:
`get_all` loads each value with `_load_many`, but `get_one` and `aget_one`
still pass the whole list to `_load`.
https://github.com/apache/airflow/blob/cba3af8c8958733e84ed280d178ee151d5f7e0e4/task-sdk/src/airflow/sdk/bases/xcom.py#L532-L533
https://github.com/apache/airflow/blob/cba3af8c8958733e84ed280d178ee151d5f7e0e4/task-sdk/src/airflow/sdk/bases/xcom.py#L744-L753
A backend that stores one reference per value then gets a list.
`XComObjectStorageBackend` hits `TypeError` in `_get_full_path` and returns the
raw paths.
The coordinator has the same gap: `_handle_get_xcom` in
`coordinators/_subprocess.py` calls the helper with `hook="load"`.
`XComResult` can't tell a mapped list from an unmapped list value. Could the
fallback set a flag, so both paths use `load_many`?
The server-side read in #76826 has the same gap:
https://github.com/apache/airflow/pull/76826#discussion_r4595964265
##########
kubernetes-tests/lang_sdk/go_example/main.go:
##########
@@ -60,12 +63,43 @@ func goExtract(actx airflow.Context) (any, error) {
}
// goTransform reads the my_variable Airflow variable through the coordinator,
-// exercising a GetVariable round-trip over the Execution API.
+// exercising a GetVariable round-trip over the Execution API, and the result
of the
+// mapped python_fanout task, which is every instance's value in a list.
func goTransform(actx airflow.Context) error {
val, err := actx.Client().GetVariable(actx, "my_variable")
if err != nil {
return err
}
actx.Logger().InfoContext(actx, "go_transform obtained variable",
"my_variable", val)
+
+ ti := actx.TaskInstance()
+ fanout, err := actx.Client().GetXCom(
+ actx, ti.DagID, ti.RunID, "python_fanout", nil,
sdk.XComReturnValueKey, nil,
+ )
+ if err != nil {
+ return err
+ }
+ values, ok := fanout.([]any)
+ if !ok || len(values) != 3 {
+ return fmt.Errorf("python_fanout result is %#v, want []any{1,
2, 3}", fanout)
+ }
+ for i, v := range values {
+ if n, ok := asInt64(v); !ok || n != int64(i+1) {
+ return fmt.Errorf("python_fanout result is %#v, want
[]any{1, 2, 3}", fanout)
+ }
+ }
return nil
}
+
+// asInt64 returns v as an int64 if it is an integer. An integer XCom value
arrives
+// as a Go integer type whose width depends on the value.
+func asInt64(v any) (int64, bool) {
+ switch n := reflect.ValueOf(v); {
+ case n.CanInt():
+ return n.Int(), true
+ case n.CanUint():
+ return int64(n.Uint()), true
+ default:
+ return 0, false
+ }
+}
Review Comment:
The Go SDK already turns decoded integers into `int64`
(`normalizeDecodedValue` in `pkg/execution/messages.go`), the PR just merged a
few days ago.
So `asInt64` isn't needed:
```suggestion
want := []any{int64(1), int64(2), int64(3)}
if !reflect.DeepEqual(fanout, want) {
return fmt.Errorf("python_fanout result is %#v, want %#v",
fanout, want)
}
return nil
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]