This is an automated email from the ASF dual-hosted git repository.
husseinawala pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new cdd1a4824d Support multiple XCom output in the BaseOperator (#37297)
cdd1a4824d is described below
commit cdd1a4824d96e4fab3d9742aefb5c3660bd079a0
Author: Hussein Awala <[email protected]>
AuthorDate: Mon Feb 12 09:58:29 2024 +0100
Support multiple XCom output in the BaseOperator (#37297)
* Support multiple XCom output in the BaseOperator
* consolidate task_flow and normal operator multiple_outputs
* revert sftp provider change
---
airflow/decorators/base.py | 20 -------
airflow/decorators/sensor.py | 1 -
airflow/models/baseoperator.py | 5 ++
airflow/models/taskinstance.py | 16 +++++-
docs/apache-airflow/core-concepts/xcoms.rst | 2 +
.../pre_commit_base_operator_partial_arguments.py | 1 +
tests/models/test_taskinstance.py | 61 ++++++++++++++++++++++
tests/serialization/test_dag_serialization.py | 1 +
8 files changed, 85 insertions(+), 22 deletions(-)
diff --git a/airflow/decorators/base.py b/airflow/decorators/base.py
index 2518fe1b53..51ebbce29c 100644
--- a/airflow/decorators/base.py
+++ b/airflow/decorators/base.py
@@ -41,7 +41,6 @@ import re2
import typing_extensions
from airflow.datasets import Dataset
-from airflow.exceptions import AirflowException
from airflow.models.abstractoperator import DEFAULT_RETRIES,
DEFAULT_RETRY_DELAY
from airflow.models.baseoperator import (
BaseOperator,
@@ -195,7 +194,6 @@ class DecoratedOperator(BaseOperator):
task_id: str,
op_args: Collection[Any] | None = None,
op_kwargs: Mapping[str, Any] | None = None,
- multiple_outputs: bool = False,
kwargs_to_upstream: dict[str, Any] | None = None,
**kwargs,
) -> None:
@@ -227,7 +225,6 @@ class DecoratedOperator(BaseOperator):
else:
signature.bind(*op_args, **op_kwargs)
- self.multiple_outputs = multiple_outputs
self.op_args = op_args
self.op_kwargs = op_kwargs
super().__init__(task_id=task_id, **kwargs_to_upstream, **kwargs)
@@ -257,23 +254,6 @@ class DecoratedOperator(BaseOperator):
for item in return_value:
if isinstance(item, Dataset):
self.outlets.append(item)
- if not self.multiple_outputs or return_value is None:
- return return_value
- if isinstance(return_value, dict):
- for key in return_value.keys():
- if not isinstance(key, str):
- raise AirflowException(
- "Returned dictionary keys must be strings when using "
- f"multiple_outputs, found {key} ({type(key)}) instead"
- )
- for key, value in return_value.items():
- if isinstance(value, Dataset):
- self.outlets.append(value)
- xcom_push(context, key, value)
- else:
- raise AirflowException(
- f"Returned output was type {type(return_value)} expected
dictionary for multiple_outputs"
- )
return return_value
def _hook_apply_defaults(self, *args, **kwargs):
diff --git a/airflow/decorators/sensor.py b/airflow/decorators/sensor.py
index 1d6ab86c9f..c332a78f95 100644
--- a/airflow/decorators/sensor.py
+++ b/airflow/decorators/sensor.py
@@ -56,7 +56,6 @@ class DecoratedSensorOperator(PythonSensor):
task_id: str,
**kwargs,
) -> None:
- kwargs.pop("multiple_outputs")
kwargs["task_id"] = get_unique_task_id(task_id, kwargs.get("dag"),
kwargs.get("task_group"))
super().__init__(**kwargs)
diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py
index e2406776d8..7e8be8de03 100644
--- a/airflow/models/baseoperator.py
+++ b/airflow/models/baseoperator.py
@@ -643,6 +643,8 @@ class BaseOperator(AbstractOperator,
metaclass=BaseOperatorMeta):
:param do_xcom_push: if True, an XCom is pushed containing the Operator's
result
+ :param multiple_outputs: if True and do_xcom_push is True, pushes multiple
XComs, one for each
+ key in the returned dictionary result. If False and do_xcom_push is
True, pushes a single XCom.
:param task_group: The TaskGroup to which the task should belong. This is
typically provided when not
using a TaskGroup as a context manager.
:param doc: Add documentation or notes to your Task objects that is
visible in
@@ -713,6 +715,7 @@ class BaseOperator(AbstractOperator,
metaclass=BaseOperatorMeta):
"on_retry_callback",
"on_skipped_callback",
"do_xcom_push",
+ "multiple_outputs",
}
# Defines if the operator supports lineage without manual definitions
@@ -782,6 +785,7 @@ class BaseOperator(AbstractOperator,
metaclass=BaseOperatorMeta):
max_active_tis_per_dagrun: int | None = None,
executor_config: dict | None = None,
do_xcom_push: bool = True,
+ multiple_outputs: bool = False,
inlets: Any | None = None,
outlets: Any | None = None,
task_group: TaskGroup | None = None,
@@ -929,6 +933,7 @@ class BaseOperator(AbstractOperator,
metaclass=BaseOperatorMeta):
self.max_active_tis_per_dag: int | None = max_active_tis_per_dag
self.max_active_tis_per_dagrun: int | None = max_active_tis_per_dagrun
self.do_xcom_push: bool = do_xcom_push
+ self.multiple_outputs: bool = multiple_outputs
self.doc_md = doc_md
self.doc_json = doc_json
diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py
index 01a84fc883..4720361504 100644
--- a/airflow/models/taskinstance.py
+++ b/airflow/models/taskinstance.py
@@ -30,7 +30,7 @@ import warnings
from collections import defaultdict
from datetime import timedelta
from enum import Enum
-from typing import TYPE_CHECKING, Any, Callable, Collection, Generator,
Iterable, Tuple
+from typing import TYPE_CHECKING, Any, Callable, Collection, Generator,
Iterable, Mapping, Tuple
from urllib.parse import quote
import dill
@@ -450,6 +450,20 @@ def _execute_task(task_instance: TaskInstance |
TaskInstancePydantic, context: C
else:
xcom_value = None
if xcom_value is not None: # If the task returns a result, push an
XCom containing it.
+ if task_to_execute.multiple_outputs:
+ if not isinstance(xcom_value, Mapping):
+ raise AirflowException(
+ f"Returned output was type {type(xcom_value)} "
+ "expected dictionary for multiple_outputs"
+ )
+ for key in xcom_value.keys():
+ if not isinstance(key, str):
+ raise AirflowException(
+ "Returned dictionary keys must be strings when
using "
+ f"multiple_outputs, found {key} ({type(key)})
instead"
+ )
+ for key, value in xcom_value.items():
+ task_instance.xcom_push(key=key, value=value,
session=session)
task_instance.xcom_push(key=XCOM_RETURN_KEY, value=xcom_value,
session=session)
_record_task_map_for_downstreams(
task_instance=task_instance, task=task_orig, value=xcom_value,
session=session
diff --git a/docs/apache-airflow/core-concepts/xcoms.rst
b/docs/apache-airflow/core-concepts/xcoms.rst
index b2dc2b665c..02051def6e 100644
--- a/docs/apache-airflow/core-concepts/xcoms.rst
+++ b/docs/apache-airflow/core-concepts/xcoms.rst
@@ -52,6 +52,8 @@ You can also use XComs in :ref:`templates
<concepts:jinja-templating>`::
XComs are a relative of :doc:`variables`, with the main difference being that
XComs are per-task-instance and designed for communication within a DAG run,
while Variables are global and designed for overall configuration and value
sharing.
+If you want to push multiple XComs at once or rename the pushed XCom key, you
can use set ``do_xcom_push`` and ``multiple_outputs`` arguments to ``True``,
and then return a dictionary of values.
+
.. note::
If the first task run is not succeeded then on every retry task XComs will
be cleared to make the task run idempotent.
diff --git
a/scripts/ci/pre_commit/pre_commit_base_operator_partial_arguments.py
b/scripts/ci/pre_commit/pre_commit_base_operator_partial_arguments.py
index ff03efc86c..14999e034e 100755
--- a/scripts/ci/pre_commit/pre_commit_base_operator_partial_arguments.py
+++ b/scripts/ci/pre_commit/pre_commit_base_operator_partial_arguments.py
@@ -36,6 +36,7 @@ IGNORED = {
"email_on_retry",
"post_execute",
"pre_execute",
+ "multiple_outputs",
# Doesn't matter, not used anywhere.
"default_args",
# Deprecated and is aliased to max_active_tis_per_dag.
diff --git a/tests/models/test_taskinstance.py
b/tests/models/test_taskinstance.py
index 16ac978807..8f5536cab6 100644
--- a/tests/models/test_taskinstance.py
+++ b/tests/models/test_taskinstance.py
@@ -1698,6 +1698,67 @@ class TestTaskInstance:
ti.run()
assert ti.xcom_pull(task_ids=task_id, key=XCOM_RETURN_KEY) is None
+ def test_xcom_without_multiple_outputs(self, dag_maker):
+ """
+ Tests the option for Operators to push XComs without multiple outputs
+ """
+ value = {"key1": "value1", "key2": "value2"}
+ task_id = "test_xcom_push_without_multiple_outputs"
+
+ with dag_maker(dag_id="test_xcom"):
+ task = PythonOperator(
+ task_id=task_id,
+ python_callable=lambda: value,
+ do_xcom_push=True,
+ )
+ ti =
dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0]
+ ti.task = task
+ ti.run()
+ assert ti.xcom_pull(task_ids=task_id, key=XCOM_RETURN_KEY) == value
+
+ def test_xcom_with_multiple_outputs(self, dag_maker):
+ """
+ Tests the option for Operators to push XComs with multiple outputs
+ """
+ value = {"key1": "value1", "key2": "value2"}
+ task_id = "test_xcom_push_with_multiple_outputs"
+
+ with dag_maker(dag_id="test_xcom"):
+ task = PythonOperator(
+ task_id=task_id,
+ python_callable=lambda: value,
+ do_xcom_push=True,
+ multiple_outputs=True,
+ )
+ ti =
dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0]
+ ti.task = task
+ ti.run()
+ assert ti.xcom_pull(task_ids=task_id, key=XCOM_RETURN_KEY) == value
+ assert ti.xcom_pull(task_ids=task_id, key="key1") == "value1"
+ assert ti.xcom_pull(task_ids=task_id, key="key2") == "value2"
+
+ def test_xcom_with_multiple_outputs_and_no_mapping_result(self, dag_maker):
+ """
+ Tests the option for Operators to push XComs with multiple outputs and
no mapping result
+ """
+ value = "value"
+ task_id = "test_xcom_push_with_multiple_outputs"
+
+ with dag_maker(dag_id="test_xcom"):
+ task = PythonOperator(
+ task_id=task_id,
+ python_callable=lambda: value,
+ do_xcom_push=True,
+ multiple_outputs=True,
+ )
+ ti =
dag_maker.create_dagrun(execution_date=timezone.utcnow()).task_instances[0]
+ ti.task = task
+ with pytest.raises(AirflowException) as ctx:
+ ti.run()
+ assert f"Returned output was type {type(value)} expected dictionary
for multiple_outputs" in str(
+ ctx.value
+ )
+
def test_post_execute_hook(self, dag_maker):
"""
Test that post_execute hook is called with the Operator's result.
diff --git a/tests/serialization/test_dag_serialization.py
b/tests/serialization/test_dag_serialization.py
index 83d40886c0..8a122592fd 100644
--- a/tests/serialization/test_dag_serialization.py
+++ b/tests/serialization/test_dag_serialization.py
@@ -1256,6 +1256,7 @@ class TestStringifiedDAGs:
"wait_for_downstream": False,
"wait_for_past_depends_before_skipping": False,
"weight_rule": "downstream",
+ "multiple_outputs": False,
}, """
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!