This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new c2bd43a2c37 Check for task concurrency parameters inside encoded
object & partial_kwargs to handle dynamic tasks. (#49708)
c2bd43a2c37 is described below
commit c2bd43a2c37f1d9578fa861e5fb2466659b9169e
Author: Karthikeyan Singaravelan <[email protected]>
AuthorDate: Thu Apr 24 18:02:57 2025 +0530
Check for task concurrency parameters inside encoded object &
partial_kwargs to handle dynamic tasks. (#49708)
Closes #49696
---
.../airflow/serialization/serialized_objects.py | 6 +++-
.../unit/serialization/test_serialized_objects.py | 39 ++++++++++++++++++++--
2 files changed, 42 insertions(+), 3 deletions(-)
diff --git a/airflow-core/src/airflow/serialization/serialized_objects.py
b/airflow-core/src/airflow/serialization/serialized_objects.py
index 8428765e2be..948111b2c17 100644
--- a/airflow-core/src/airflow/serialization/serialized_objects.py
+++ b/airflow-core/src/airflow/serialization/serialized_objects.py
@@ -2098,7 +2098,11 @@ class LazyDeserializedDAG(pydantic.BaseModel):
@property
def has_task_concurrency_limits(self) -> bool:
return any(
- task[Encoding.VAR].get("max_active_tis_per_dag") is not None for
task in self.data["dag"]["tasks"]
+ task[Encoding.VAR].get("max_active_tis_per_dag") is not None
+ or task[Encoding.VAR].get("max_active_tis_per_dagrun") is not None
+ or task[Encoding.VAR].get("partial_kwargs",
{}).get("max_active_tis_per_dag") is not None
+ or task[Encoding.VAR].get("partial_kwargs",
{}).get("max_active_tis_per_dagrun") is not None
+ for task in self.data["dag"]["tasks"]
)
@property
diff --git a/airflow-core/tests/unit/serialization/test_serialized_objects.py
b/airflow-core/tests/unit/serialization/test_serialized_objects.py
index 3f666249428..6f287289673 100644
--- a/airflow-core/tests/unit/serialization/test_serialized_objects.py
+++ b/airflow-core/tests/unit/serialization/test_serialized_objects.py
@@ -44,6 +44,7 @@ from airflow.providers.standard.operators.empty import
EmptyOperator
from airflow.providers.standard.operators.python import PythonOperator
from airflow.providers.standard.triggers.file import FileDeleteTrigger
from airflow.sdk.definitions.asset import Asset, AssetAlias, AssetAliasEvent,
AssetUniqueKey, AssetWatcher
+from airflow.sdk.definitions.decorators import task
from airflow.sdk.definitions.param import Param
from airflow.sdk.execution_time.context import OutletEventAccessor,
OutletEventAccessors
from airflow.serialization.enums import DagAttributeTypes as DAT, Encoding
@@ -448,10 +449,44 @@ def
test_serialized_dag_to_dict_and_from_dict_gives_same_result_in_tasks(dag_mak
assert dag2["dag"]["tasks"][0]["__var"].keys() ==
dag1["dag"]["tasks"][0]["__var"].keys()
[email protected](
+ "concurrency_parameter",
+ [
+ "max_active_tis_per_dag",
+ "max_active_tis_per_dagrun",
+ ],
+)
[email protected]_test
+def test_serialized_dag_has_task_concurrency_limits(dag_maker,
concurrency_parameter):
+ with dag_maker() as dag:
+ BashOperator(task_id="task1", bash_command="echo 1",
**{concurrency_parameter: 1})
+
+ ser_dict = SerializedDAG.to_dict(dag)
+ lazy_serialized_dag = LazyDeserializedDAG(data=ser_dict)
+
+ assert lazy_serialized_dag.has_task_concurrency_limits
+
+
[email protected](
+ "concurrency_parameter",
+ [
+ "max_active_tis_per_dag",
+ "max_active_tis_per_dagrun",
+ ],
+)
@pytest.mark.db_test
-def test_serialized_dag_has_task_concurrency_limits(dag_maker):
+def test_serialized_dag_mapped_task_has_task_concurrency_limits(dag_maker,
concurrency_parameter):
with dag_maker() as dag:
- BashOperator(task_id="task1", bash_command="echo 1",
max_active_tis_per_dag=1)
+
+ @task
+ def my_task():
+ return [1, 2, 3, 4, 5, 6, 7]
+
+ @task(**{concurrency_parameter: 1})
+ def map_me_but_slowly(a):
+ pass
+
+ map_me_but_slowly.expand(a=my_task())
ser_dict = SerializedDAG.to_dict(dag)
lazy_serialized_dag = LazyDeserializedDAG(data=ser_dict)