This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 57b12fe0ce Resolve dag serialization deprecations in tests (#40006)
57b12fe0ce is described below
commit 57b12fe0ce14487188da2c12a7bbfa1108008301
Author: Gopal Dirisala <39794726+dir...@users.noreply.github.com>
AuthorDate: Sun Jun 2 13:49:42 2024 +0530
Resolve dag serialization deprecations in tests (#40006)
---
tests/deprecations_ignore.yml | 3 ---
tests/serialization/test_dag_serialization.py | 36 ++++++++++++++++++++++-----
tests/serialization/test_pydantic_models.py | 4 ++-
3 files changed, 33 insertions(+), 10 deletions(-)
diff --git a/tests/deprecations_ignore.yml b/tests/deprecations_ignore.yml
index b89b41355a..541d89898f 100644
--- a/tests/deprecations_ignore.yml
+++ b/tests/deprecations_ignore.yml
@@ -136,9 +136,6 @@
# Serialization
-
tests/serialization/test_dag_serialization.py::TestStringifiedDAGs::test_custom_dep_detector
--
tests/serialization/test_dag_serialization.py::TestStringifiedDAGs::test_dag_params_roundtrip
--
tests/serialization/test_dag_serialization.py::TestStringifiedDAGs::test_task_params_roundtrip
--
tests/serialization/test_pydantic_models.py::test_serializing_pydantic_dataset_event
# WWW
diff --git a/tests/serialization/test_dag_serialization.py
b/tests/serialization/test_dag_serialization.py
index fbefe71d28..093b7fba76 100644
--- a/tests/serialization/test_dag_serialization.py
+++ b/tests/serialization/test_dag_serialization.py
@@ -45,7 +45,7 @@ import airflow
from airflow.datasets import Dataset
from airflow.decorators import teardown
from airflow.decorators.base import DecoratedOperator
-from airflow.exceptions import AirflowException, SerializationError
+from airflow.exceptions import AirflowException, RemovedInAirflow3Warning,
SerializationError
from airflow.hooks.base import BaseHook
from airflow.models.baseoperator import BaseOperator
from airflow.models.connection import Connection
@@ -934,7 +934,14 @@ class TestStringifiedDAGs:
"""
Test that params work both on Serialized DAGs & Tasks
"""
- dag = DAG(dag_id="simple_dag", params=val)
+ if val and any([True for k, v in val.items() if isinstance(v, set)]):
+ with pytest.warns(
+ RemovedInAirflow3Warning,
+ match="The use of non-json-serializable params is deprecated
and will be removed in a future release",
+ ):
+ dag = DAG(dag_id="simple_dag", params=val)
+ else:
+ dag = DAG(dag_id="simple_dag", params=val)
BaseOperator(task_id="simple_task", dag=dag, start_date=datetime(2019,
8, 1))
serialized_dag_json = SerializedDAG.to_json(dag)
@@ -943,7 +950,15 @@ class TestStringifiedDAGs:
assert "params" in serialized_dag["dag"]
- deserialized_dag = SerializedDAG.from_dict(serialized_dag)
+ if val and any([True for k, v in val.items() if isinstance(v, set)]):
+ with pytest.warns(
+ RemovedInAirflow3Warning,
+ match="The use of non-json-serializable params is deprecated
and will be removed in a future release",
+ ):
+ deserialized_dag = SerializedDAG.from_dict(serialized_dag)
+
+ else:
+ deserialized_dag = SerializedDAG.from_dict(serialized_dag)
deserialized_simple_task = deserialized_dag.task_dict["simple_task"]
assert expected_val == deserialized_dag.params.dump()
assert expected_val == deserialized_simple_task.params.dump()
@@ -1011,15 +1026,24 @@ class TestStringifiedDAGs:
Test that params work both on Serialized DAGs & Tasks
"""
dag = DAG(dag_id="simple_dag")
- BaseOperator(task_id="simple_task", dag=dag, params=val,
start_date=datetime(2019, 8, 1))
+ if val and any([True for k, v in val.items() if isinstance(v, set)]):
+ with pytest.warns(
+ RemovedInAirflow3Warning,
+ match="The use of non-json-serializable params is deprecated
and will be removed in a future release",
+ ):
+ BaseOperator(task_id="simple_task", dag=dag, params=val,
start_date=datetime(2019, 8, 1))
+ serialized_dag = SerializedDAG.to_dict(dag)
+ deserialized_dag = SerializedDAG.from_dict(serialized_dag)
+ else:
+ BaseOperator(task_id="simple_task", dag=dag, params=val,
start_date=datetime(2019, 8, 1))
+ serialized_dag = SerializedDAG.to_dict(dag)
+ deserialized_dag = SerializedDAG.from_dict(serialized_dag)
- serialized_dag = SerializedDAG.to_dict(dag)
if val:
assert "params" in serialized_dag["dag"]["tasks"][0]["__var"]
else:
assert "params" not in serialized_dag["dag"]["tasks"][0]["__var"]
- deserialized_dag = SerializedDAG.from_dict(serialized_dag)
deserialized_simple_task = deserialized_dag.task_dict["simple_task"]
assert expected_val == deserialized_simple_task.params.dump()
diff --git a/tests/serialization/test_pydantic_models.py
b/tests/serialization/test_pydantic_models.py
index 522399d1fc..048faebf54 100644
--- a/tests/serialization/test_pydantic_models.py
+++ b/tests/serialization/test_pydantic_models.py
@@ -215,12 +215,14 @@ def test_serializing_pydantic_dataset_event(session,
create_task_instance, creat
with_dagrun_type=DagRunType.MANUAL,
session=session,
)
+ execution_date = timezone.utcnow()
dr = dag.create_dagrun(
run_id="test2",
run_type=DagRunType.DATASET_TRIGGERED,
- execution_date=timezone.utcnow(),
+ execution_date=execution_date,
state=None,
session=session,
+ data_interval=(execution_date, execution_date),
)
ds1_event = DatasetEvent(dataset_id=1)
ds2_event_1 = DatasetEvent(dataset_id=2)
