Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
github-actions[bot] commented on PR #63871: URL: https://github.com/apache/airflow/pull/63871#issuecomment-4453487557 ### Backport failed to create: v3-2-test. View the failure log Run details Note: As of [Merging PRs targeted for Airflow 3.X](https://github.com/apache/airflow/blob/main/dev/README_AIRFLOW3_DEV.md#merging-prs-targeted-for-airflow-3x) the committer who merges the PR is responsible for backporting the PRs that are bug fixes (generally speaking) to the maintenance branches. In matter of doubt please ask in [#release-management](https://apache-airflow.slack.com/archives/C03G9H97MM2) Slack channel. Status Branch Result ❌ v3-2-test https://github.com/apache/airflow/commit/90051561e721d24834f79d5e1335708c671a9be9";> You can attempt to backport this manually by running: ```bash cherry_picker 9005156 v3-2-test ``` This should apply the commit to the v3-2-test branch and leave the commit in conflict state marking the files that need manual conflict resolution. After you have resolved the conflicts, you can continue the backport process by running: ```bash cherry_picker --continue ``` If you don't have cherry-picker installed, see the [installation guide](https://github.com/apache/airflow/blob/main/dev/README.md#how-to-backport-pr-with-cherry-picker-cli). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
ephraimbuddy merged PR #63871: URL: https://github.com/apache/airflow/pull/63871 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
ephraimbuddy commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3241012344
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -19,87 +19,96 @@
from __future__ import annotations
import contextlib
+import inspect
from typing import TYPE_CHECKING, Any
from airflow._shared.module_loading import qualname
from airflow._shared.secrets_masker import redact
from airflow._shared.template_rendering import truncate_rendered_value
from airflow.configuration import conf
-from airflow.settings import json
if TYPE_CHECKING:
from airflow.partition_mappers.base import PartitionMapper
from airflow.timetables.base import Timetable as CoreTimetable
-def serialize_template_field(template_field: Any, name: str) -> str | dict |
list | int | float:
+def serialize_template_field(template_field: Any, name: str) -> str | dict |
list | int | float | bool | None:
"""
Return a serializable representation of the templated field.
-If ``templated_field`` is provided via a callable then
-return the following serialized value:
+The walk has two responsibilities:
-If ``templated_field`` contains a class or instance that requires recursive
-templating, store them as strings. Otherwise simply return the field as-is.
+1. **Make the template_field JSON-encodable** — every container is rebuilt
+ with primitive leaves (str/int/float/bool/None), tuples and sets are
+ flattened to lists, and unsupported objects fall through to ``str()``
+ so ``json.dumps`` never raises on the result.
+2. **Keep the output deterministic across parses** — callables are replaced
+ with their qualified name (never the default
+ repr), dicts are key-sorted, and (frozen)sets are sorted by element so
+ the same input always produces the same string.
"""
-def is_jsonable(x):
-try:
-json.dumps(x)
-except (TypeError, OverflowError):
-return False
-else:
-return True
-
-def translate_tuples_to_lists(obj: Any):
-"""Recursively convert tuples to lists."""
-if isinstance(obj, tuple):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, list):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, dict):
-return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
-return obj
+def normalize_dict_key(key) -> str:
+"""Normalize a dict key to a serialized string type."""
+# Serialized template_field keys must all be strings, not a mix of
types, so that
+# downstream json.dumps(..., sort_keys=True) does not raise on
mixed-type keys.
+return str(serialize_object(key))
+
+def serialize_object(obj):
+"""Recursively rewrite ``obj`` into a JSON-encodable, hash-stable
structure."""
+if obj is None or isinstance(obj, (str, int, float, bool)):
+return obj
-def sort_dict_recursively(obj: Any) -> Any:
-"""Recursively sort dictionaries to ensure consistent ordering."""
if isinstance(obj, dict):
-return {k: sort_dict_recursively(v) for k, v in
sorted(obj.items())}
-if isinstance(obj, list):
-return [sort_dict_recursively(item) for item in obj]
-if isinstance(obj, tuple):
-return tuple(sort_dict_recursively(item) for item in obj)
-return obj
+# Serialize keys/values first so each key is a string and the
output is hash-stable,
+# then sort by the serialized key to prevent hash inconsistencies
when dict ordering varies.
+serialized_pairs = [(normalize_dict_key(k), serialize_object(v))
for k, v in obj.items()]
+return dict(sorted(serialized_pairs, key=lambda kv: kv[0]))
+
+if isinstance(obj, (list, tuple)):
+return [serialize_object(item) for item in obj]
+
+if isinstance(obj, (set, frozenset)):
+# JSON has no set type → flatten to a list with deterministic
ordering
+# so hash randomization on element types cannot shift
cross-process iteration order.
+serialized_set = [serialize_object(e) for e in obj]
+return sorted(serialized_set, key=lambda x: (type(x).__name__,
str(x)))
+
+# Use inspect.getattr_static to bypass any custom __getattr__ /
metaclass magic
+if callable(inspect.getattr_static(obj, "serialize", None)):
+return serialize_object(obj.serialize())
+
+# Kubernetes client objects (V1Pod, V1Container, ...) expose their
content via to_dict().
+# Scope the branch to the kubernetes namespace so unrelated user
classes that happen to
+# define a to_dict() method fall through to str() instead of being
treated as K8s payloads.
+if getattr(type(obj), "__module
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
wjddn279 commented on PR #63871: URL: https://github.com/apache/airflow/pull/63871#issuecomment-4442941382 @ephraimbuddy Thanks for patient review.. I think we've covered all the cases we needed to address and hope so -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
wjddn279 commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3235690442
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -19,87 +19,88 @@
from __future__ import annotations
import contextlib
+import inspect
from typing import TYPE_CHECKING, Any
from airflow._shared.module_loading import qualname
from airflow._shared.secrets_masker import redact
from airflow._shared.template_rendering import truncate_rendered_value
from airflow.configuration import conf
-from airflow.settings import json
if TYPE_CHECKING:
from airflow.partition_mappers.base import PartitionMapper
from airflow.timetables.base import Timetable as CoreTimetable
-def serialize_template_field(template_field: Any, name: str) -> str | dict |
list | int | float:
+def serialize_template_field(template_field: Any, name: str) -> str | dict |
list | int | float | bool | None:
"""
Return a serializable representation of the templated field.
-If ``templated_field`` is provided via a callable then
-return the following serialized value:
+The walk has two responsibilities:
-If ``templated_field`` contains a class or instance that requires recursive
-templating, store them as strings. Otherwise simply return the field as-is.
+1. **Make the template_field JSON-encodable** — every container is rebuilt
+ with primitive leaves (str/int/float/bool/None), tuples and sets are
+ flattened to lists, and unsupported objects fall through to ``str()``
+ so ``json.dumps`` never raises on the result.
+2. **Keep the output deterministic across parses** — callables are replaced
+ with their qualified name (never the default
+ repr), dicts are key-sorted, and (frozen)sets are sorted by element so
+ the same input always produces the same string.
"""
-def is_jsonable(x):
-try:
-json.dumps(x)
-except (TypeError, OverflowError):
-return False
-else:
-return True
-
-def translate_tuples_to_lists(obj: Any):
-"""Recursively convert tuples to lists."""
-if isinstance(obj, tuple):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, list):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, dict):
-return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
-return obj
+def normalize_dict_key(key) -> str:
+"""Normalize a dict key to a serialized string type."""
+# Serialized template_field keys must all be strings, not a mix of
types, so that
+# downstream json.dumps(..., sort_keys=True) does not raise on
mixed-type keys.
+return str(serialize_object(key))
+
+def serialize_object(obj):
+"""Recursively rewrite ``obj`` into a JSON-encodable, hash-stable
structure."""
+if obj is None or isinstance(obj, (str, int, float, bool)):
+return obj
-def sort_dict_recursively(obj: Any) -> Any:
-"""Recursively sort dictionaries to ensure consistent ordering."""
if isinstance(obj, dict):
-return {k: sort_dict_recursively(v) for k, v in
sorted(obj.items())}
-if isinstance(obj, list):
-return [sort_dict_recursively(item) for item in obj]
-if isinstance(obj, tuple):
-return tuple(sort_dict_recursively(item) for item in obj)
-return obj
+# Serialize keys/values first so each key is a string and the
output is hash-stable,
+# then sort by the serialized key to prevents hash inconsistencies
when dict ordering varies.
+serialized_pairs = [(normalize_dict_key(k), serialize_object(v))
for k, v in obj.items()]
+return dict(sorted(serialized_pairs, key=lambda kv: kv[0]))
+
+if isinstance(obj, (list, tuple)):
+return [serialize_object(item) for item in obj]
+
+if isinstance(obj, (set, frozenset)):
+# JSON has no set type → flatten to a list with deterministic
ordering
+# so hash randomization on element types cannot shift
cross-process iteration order.
+return sorted(
+(serialize_object(item) for item in obj),
+key=lambda x: (type(x).__name__, str(x)),
+)
+
+# Use inspect.getattr_static to bypass any custom __getattr__ /
metaclass magic
+if callable(inspect.getattr_static(obj, "serialize", None)):
+return serialize_object(obj.serialize())
+
+# Kubernetes client objects (V1Pod, V1Container, ...) expose their
content via to_dict()
+if callable(inspect.getattr_static(obj, "to_dict", None)):
+return serialize_object(obj.to_dict())
Review Comment:
apply `if getattr(type(obj), "__module__", "").startswith("kubernetes.")`
--
This is an au
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
ephraimbuddy commented on PR #63871:
URL: https://github.com/apache/airflow/pull/63871#issuecomment-4439698483
The fix works for callables, but the same bug still exists for any object
that lacks a custom __repr__.
The PR's promise is "no memory addresses in the serialized output." It
achieves that for callables by replacing them with ``. But
the final fallback at `airflow-core/src/airflow/serialization/helpers.py:93` is
return `str(obj)` — and `str()` on any plain Python object hits the default
`` repr.
So:
```python
class Opaque: pass
serialize_template_field({Opaque(), Opaque()}, "f")
# → ['<__main__.Opaque object at 0x10abc>', '<__main__.Opaque object at
0x10def>']
```
Two parses → different addresses → different output → DAG hash flips.
Exactly the bug the PR is meant to fix, one type of object over.
Why the tests don't catch it: every "no address leak" test in the PR uses
either a callable (covered by the qualname branch) or an object with a custom
`__str__` (covered by user-provided text). Nothing exercises the plain-object
`str()` fallback.
Fix options:
Replace `str(obj)` with an address-free form like `qualname(type(obj),
True)`, or
Add a test with a vanilla class Foo: pass and assert "at 0x" not in result.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
ephraimbuddy commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3233165043
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -19,87 +19,88 @@
from __future__ import annotations
import contextlib
+import inspect
from typing import TYPE_CHECKING, Any
from airflow._shared.module_loading import qualname
from airflow._shared.secrets_masker import redact
from airflow._shared.template_rendering import truncate_rendered_value
from airflow.configuration import conf
-from airflow.settings import json
if TYPE_CHECKING:
from airflow.partition_mappers.base import PartitionMapper
from airflow.timetables.base import Timetable as CoreTimetable
-def serialize_template_field(template_field: Any, name: str) -> str | dict |
list | int | float:
+def serialize_template_field(template_field: Any, name: str) -> str | dict |
list | int | float | bool | None:
"""
Return a serializable representation of the templated field.
-If ``templated_field`` is provided via a callable then
-return the following serialized value:
+The walk has two responsibilities:
-If ``templated_field`` contains a class or instance that requires recursive
-templating, store them as strings. Otherwise simply return the field as-is.
+1. **Make the template_field JSON-encodable** — every container is rebuilt
+ with primitive leaves (str/int/float/bool/None), tuples and sets are
+ flattened to lists, and unsupported objects fall through to ``str()``
+ so ``json.dumps`` never raises on the result.
+2. **Keep the output deterministic across parses** — callables are replaced
+ with their qualified name (never the default
+ repr), dicts are key-sorted, and (frozen)sets are sorted by element so
+ the same input always produces the same string.
"""
-def is_jsonable(x):
-try:
-json.dumps(x)
-except (TypeError, OverflowError):
-return False
-else:
-return True
-
-def translate_tuples_to_lists(obj: Any):
-"""Recursively convert tuples to lists."""
-if isinstance(obj, tuple):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, list):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, dict):
-return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
-return obj
+def normalize_dict_key(key) -> str:
+"""Normalize a dict key to a serialized string type."""
+# Serialized template_field keys must all be strings, not a mix of
types, so that
+# downstream json.dumps(..., sort_keys=True) does not raise on
mixed-type keys.
+return str(serialize_object(key))
+
+def serialize_object(obj):
+"""Recursively rewrite ``obj`` into a JSON-encodable, hash-stable
structure."""
+if obj is None or isinstance(obj, (str, int, float, bool)):
+return obj
-def sort_dict_recursively(obj: Any) -> Any:
-"""Recursively sort dictionaries to ensure consistent ordering."""
if isinstance(obj, dict):
-return {k: sort_dict_recursively(v) for k, v in
sorted(obj.items())}
-if isinstance(obj, list):
-return [sort_dict_recursively(item) for item in obj]
-if isinstance(obj, tuple):
-return tuple(sort_dict_recursively(item) for item in obj)
-return obj
+# Serialize keys/values first so each key is a string and the
output is hash-stable,
+# then sort by the serialized key to prevents hash inconsistencies
when dict ordering varies.
+serialized_pairs = [(normalize_dict_key(k), serialize_object(v))
for k, v in obj.items()]
+return dict(sorted(serialized_pairs, key=lambda kv: kv[0]))
+
+if isinstance(obj, (list, tuple)):
+return [serialize_object(item) for item in obj]
+
+if isinstance(obj, (set, frozenset)):
+# JSON has no set type → flatten to a list with deterministic
ordering
+# so hash randomization on element types cannot shift
cross-process iteration order.
+return sorted(
+(serialize_object(item) for item in obj),
+key=lambda x: (type(x).__name__, str(x)),
+)
+
+# Use inspect.getattr_static to bypass any custom __getattr__ /
metaclass magic
+if callable(inspect.getattr_static(obj, "serialize", None)):
+return serialize_object(obj.serialize())
+
+# Kubernetes client objects (V1Pod, V1Container, ...) expose their
content via to_dict()
+if callable(inspect.getattr_static(obj, "to_dict", None)):
+return serialize_object(obj.to_dict())
Review Comment:
This looks too broad. Should we make it specifically for Kubernetes? For
example `inspect
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
vatsrahul1001 commented on PR #63871: URL: https://github.com/apache/airflow/pull/63871#issuecomment-4439044066 @ephraimbuddy @amoghrajesh can you review this looks all previous review comments have been addressed -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
wjddn279 commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3224986044
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -19,87 +19,84 @@
from __future__ import annotations
import contextlib
+import inspect
from typing import TYPE_CHECKING, Any
from airflow._shared.module_loading import qualname
from airflow._shared.secrets_masker import redact
from airflow._shared.template_rendering import truncate_rendered_value
from airflow.configuration import conf
-from airflow.settings import json
if TYPE_CHECKING:
from airflow.partition_mappers.base import PartitionMapper
from airflow.timetables.base import Timetable as CoreTimetable
-def serialize_template_field(template_field: Any, name: str) -> str | dict |
list | int | float:
+def serialize_template_field(template_field: Any, name: str) -> str | dict |
list | int | float | bool:
"""
Return a serializable representation of the templated field.
-If ``templated_field`` is provided via a callable then
-return the following serialized value:
+The walk has two responsibilities:
-If ``templated_field`` contains a class or instance that requires recursive
-templating, store them as strings. Otherwise simply return the field as-is.
+1. **Make the template_field JSON-encodable** — every container is rebuilt
+ with primitive leaves (str/int/float/bool/None), tuples and sets are
+ flattened to lists, and unsupported objects fall through to ``str()``
+ so ``json.dumps`` never raises on the result.
+2. **Keep the output deterministic across parses** — callables are replaced
+ with their qualified name (never the default
+ repr), dicts are key-sorted, and (frozen)sets are sorted by element so
+ the same input always produces the same string.
"""
-def is_jsonable(x):
-try:
-json.dumps(x)
-except (TypeError, OverflowError):
-return False
-else:
-return True
-
-def translate_tuples_to_lists(obj: Any):
-"""Recursively convert tuples to lists."""
-if isinstance(obj, tuple):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, list):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, dict):
-return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
-return obj
+def normalize_dict_key(key) -> str:
+"""Normalize a dict key to a serialized string type."""
+# Serialized template_field keys must all be strings, not a mix of
types, so that
+# downstream json.dumps(..., sort_keys=True) does not raise on
mixed-type keys.
+return str(serialize_object(key))
+
+def serialize_object(obj):
+"""Recursively rewrite ``obj`` into a JSON-encodable, hash-stable
structure."""
+if obj is None or isinstance(obj, (str, int, float, bool)):
+return obj
-def sort_dict_recursively(obj: Any) -> Any:
-"""Recursively sort dictionaries to ensure consistent ordering."""
if isinstance(obj, dict):
-return {k: sort_dict_recursively(v) for k, v in
sorted(obj.items())}
-if isinstance(obj, list):
-return [sort_dict_recursively(item) for item in obj]
-if isinstance(obj, tuple):
-return tuple(sort_dict_recursively(item) for item in obj)
-return obj
+# Serialize keys/values first so each key is a string and the
output is hash-stable,
+# then sort by the serialized key to prevents hash inconsistencies
when dict ordering varies.
+serialized_pairs = [(normalize_dict_key(k), serialize_object(v))
for k, v in obj.items()]
+return dict(sorted(serialized_pairs, key=lambda kv: kv[0]))
Review Comment:
Since keys are now fixed as strings by `normalize_dict_key`, the logic that
sorted by key type alongside the key value has been removed. Sorting is now
performed solely by the key value.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
wjddn279 commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3224968076
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -19,87 +19,89 @@
from __future__ import annotations
import contextlib
+import inspect
from typing import TYPE_CHECKING, Any
from airflow._shared.module_loading import qualname
from airflow._shared.secrets_masker import redact
from airflow._shared.template_rendering import truncate_rendered_value
from airflow.configuration import conf
-from airflow.settings import json
if TYPE_CHECKING:
from airflow.partition_mappers.base import PartitionMapper
from airflow.timetables.base import Timetable as CoreTimetable
-def serialize_template_field(template_field: Any, name: str) -> str | dict |
list | int | float:
+def serialize_template_field(template_field: Any, name: str) -> str | dict |
list | int | float | bool:
"""
Return a serializable representation of the templated field.
-If ``templated_field`` is provided via a callable then
-return the following serialized value:
+The walk has two responsibilities:
-If ``templated_field`` contains a class or instance that requires recursive
-templating, store them as strings. Otherwise simply return the field as-is.
+1. **Make the template_field JSON-encodable** — every container is rebuilt
+ with primitive leaves (str/int/float/bool/None), tuples and sets are
+ flattened to lists, and unsupported objects fall through to ``str()``
+ so ``json.dumps`` never raises on the result.
+2. **Keep the output deterministic across parses** — callables are replaced
+ with their qualified name (never the default
+ repr), dicts are key-sorted, and (frozen)sets are sorted by element so
+ the same input always produces the same string.
"""
-def is_jsonable(x):
-try:
-json.dumps(x)
-except (TypeError, OverflowError):
-return False
-else:
-return True
-
-def translate_tuples_to_lists(obj: Any):
-"""Recursively convert tuples to lists."""
-if isinstance(obj, tuple):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, list):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, dict):
-return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
-return obj
+def is_jsonable_primitive_type(k):
+"""Whether ``k`` is one of the primitive types json.dumps accepts as a
dict key or value."""
+return k is None or isinstance(k, (str, int, float, bool))
+
+def serialize_dict_key(key):
+"""Normalize a dict key to a JSON-encodable primitive."""
+serialized_key = serialize_object(key)
+return serialized_key if is_jsonable_primitive_type(serialized_key)
else str(serialized_key)
Review Comment:
fix by making key string
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
ephraimbuddy commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3220723394
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -19,87 +19,89 @@
from __future__ import annotations
import contextlib
+import inspect
from typing import TYPE_CHECKING, Any
from airflow._shared.module_loading import qualname
from airflow._shared.secrets_masker import redact
from airflow._shared.template_rendering import truncate_rendered_value
from airflow.configuration import conf
-from airflow.settings import json
if TYPE_CHECKING:
from airflow.partition_mappers.base import PartitionMapper
from airflow.timetables.base import Timetable as CoreTimetable
-def serialize_template_field(template_field: Any, name: str) -> str | dict |
list | int | float:
+def serialize_template_field(template_field: Any, name: str) -> str | dict |
list | int | float | bool:
"""
Return a serializable representation of the templated field.
-If ``templated_field`` is provided via a callable then
-return the following serialized value:
+The walk has two responsibilities:
-If ``templated_field`` contains a class or instance that requires recursive
-templating, store them as strings. Otherwise simply return the field as-is.
+1. **Make the template_field JSON-encodable** — every container is rebuilt
+ with primitive leaves (str/int/float/bool/None), tuples and sets are
+ flattened to lists, and unsupported objects fall through to ``str()``
+ so ``json.dumps`` never raises on the result.
+2. **Keep the output deterministic across parses** — callables are replaced
+ with their qualified name (never the default
+ repr), dicts are key-sorted, and (frozen)sets are sorted by element so
+ the same input always produces the same string.
"""
-def is_jsonable(x):
-try:
-json.dumps(x)
-except (TypeError, OverflowError):
-return False
-else:
-return True
-
-def translate_tuples_to_lists(obj: Any):
-"""Recursively convert tuples to lists."""
-if isinstance(obj, tuple):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, list):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, dict):
-return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
-return obj
+def is_jsonable_primitive_type(k):
+"""Whether ``k`` is one of the primitive types json.dumps accepts as a
dict key or value."""
+return k is None or isinstance(k, (str, int, float, bool))
+
+def serialize_dict_key(key):
+"""Normalize a dict key to a JSON-encodable primitive."""
+serialized_key = serialize_object(key)
+return serialized_key if is_jsonable_primitive_type(serialized_key)
else str(serialized_key)
+
+def serialize_object(obj):
+"""Recursively rewrite ``obj`` into a JSON-encodable, hash-stable
structure."""
+if is_jsonable_primitive_type(obj):
+return obj
-def sort_dict_recursively(obj: Any) -> Any:
-"""Recursively sort dictionaries to ensure consistent ordering."""
if isinstance(obj, dict):
-return {k: sort_dict_recursively(v) for k, v in
sorted(obj.items())}
-if isinstance(obj, list):
-return [sort_dict_recursively(item) for item in obj]
-if isinstance(obj, tuple):
-return tuple(sort_dict_recursively(item) for item in obj)
-return obj
+# Sort dictionaries recursively to ensure consistent string
representation
+# This prevents hash inconsistencies when dict ordering varies
+return {
+serialize_dict_key(k): serialize_object(v)
+for k, v in sorted(obj.items(), key=lambda kv:
(type(kv[0]).__name__, repr(kv[0])))
Review Comment:
The sort uses `(type(kv[0]).__name__, repr(kv[0]))`, but repr on a callable
embeds its memory address `()`. Concrete repro using the
new code:
```
v1 = {(lambda x: x): "a", (lambda y: y): "b"}
v2 = {(lambda y: y): "b", (lambda x: x): "a"}
serialize_template_field(v1, "k") != serialize_template_field(v2, "k") #
True
```
Both lambdas collapse to the same output key `>`,
and which value survives depends on the address-based sort. Same issue for any
two objects whose repr defaults to `<… at 0x…> (no custom __repr__)`. Fix: sort
on the serialized form of the key `((type_name, str(serialize_dict_key(k`
so callables sort by qualname, not address. Worth a regression test too.
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -19,87 +19,89 @@
from __future__ import annotations
import contextlib
+import inspect
from typing import TYPE_CHECKING, Any
from airflow._shared.module_loading import
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
wjddn279 commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3212524999
##
airflow-core/tests/unit/models/test_renderedtifields.py:
##
@@ -116,11 +116,11 @@ def teardown_method(self):
pytest.param([], [], id="list"),
pytest.param({}, {}, id="empty_dict"),
pytest.param((), [], id="empty_tuple"),
-pytest.param(set(), "set()", id="empty_set"),
+pytest.param(set(), [], id="empty_set"),
Review Comment:
Update tests to reflect logic change: `set` and `frozenset` are now
converted to `list` instead of being string-cast.
related https://github.com/apache/airflow/pull/63871/changes#r3212530026
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
wjddn279 commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3212530026
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -19,87 +19,89 @@
from __future__ import annotations
import contextlib
+import inspect
from typing import TYPE_CHECKING, Any
from airflow._shared.module_loading import qualname
from airflow._shared.secrets_masker import redact
from airflow._shared.template_rendering import truncate_rendered_value
from airflow.configuration import conf
-from airflow.settings import json
if TYPE_CHECKING:
from airflow.partition_mappers.base import PartitionMapper
from airflow.timetables.base import Timetable as CoreTimetable
-def serialize_template_field(template_field: Any, name: str) -> str | dict |
list | int | float:
+def serialize_template_field(template_field: Any, name: str) -> str | dict |
list | int | float | bool:
"""
Return a serializable representation of the templated field.
-If ``templated_field`` is provided via a callable then
-return the following serialized value:
+The walk has two responsibilities:
-If ``templated_field`` contains a class or instance that requires recursive
-templating, store them as strings. Otherwise simply return the field as-is.
+1. **Make the template_field JSON-encodable** — every container is rebuilt
+ with primitive leaves (str/int/float/bool/None), tuples and sets are
+ flattened to lists, and unsupported objects fall through to ``str()``
+ so ``json.dumps`` never raises on the result.
+2. **Keep the output deterministic across parses** — callables are replaced
+ with their qualified name (never the default
+ repr), dicts are key-sorted, and (frozen)sets are sorted by element so
+ the same input always produces the same string.
"""
-def is_jsonable(x):
-try:
-json.dumps(x)
-except (TypeError, OverflowError):
-return False
-else:
-return True
-
-def translate_tuples_to_lists(obj: Any):
-"""Recursively convert tuples to lists."""
-if isinstance(obj, tuple):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, list):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, dict):
-return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
-return obj
+def is_jsonable_primitive_type(k):
+"""Whether ``k`` is one of the primitive types json.dumps accepts as a
dict key or value."""
+return k is None or isinstance(k, (str, int, float, bool))
+
+def serialize_dict_key(key):
+"""Normalize a dict key to a JSON-encodable primitive."""
+serialized_key = serialize_object(key)
+return serialized_key if is_jsonable_primitive_type(serialized_key)
else str(serialized_key)
+
+def serialize_object(obj):
+"""Recursively rewrite ``obj`` into a JSON-encodable, hash-stable
structure."""
+if is_jsonable_primitive_type(obj):
+return obj
-def sort_dict_recursively(obj: Any) -> Any:
-"""Recursively sort dictionaries to ensure consistent ordering."""
if isinstance(obj, dict):
-return {k: sort_dict_recursively(v) for k, v in
sorted(obj.items())}
-if isinstance(obj, list):
-return [sort_dict_recursively(item) for item in obj]
-if isinstance(obj, tuple):
-return tuple(sort_dict_recursively(item) for item in obj)
-return obj
+# Sort dictionaries recursively to ensure consistent string
representation
+# This prevents hash inconsistencies when dict ordering varies
+return {
+serialize_dict_key(k): serialize_object(v)
+for k, v in sorted(obj.items(), key=lambda kv:
(type(kv[0]).__name__, repr(kv[0])))
+}
+
+if isinstance(obj, (list, tuple)):
+return [serialize_object(item) for item in obj]
+
+if isinstance(obj, (set, frozenset)):
Review Comment:
Previously `set/frozenset` values were converted via str(), which leaked
memory addresses for custom objects and destabilized the DAG hash on every
parse. The new logic recursively serializes each element and returns a sorted
list, producing deterministic, JSON-encodable output.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
wjddn279 commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3212525812
##
task-sdk/tests/task_sdk/execution_time/test_task_runner.py:
##
@@ -1117,7 +1117,7 @@ def test_basic_templated_dag(mocked_parse,
make_ti_context, mock_supervisor_comm
),
pytest.param(
{"my_tup": (1, 2), "my_set": {1, 2, 3}},
-{"my_tup": [1, 2], "my_set": "{1, 2, 3}"},
+{"my_tup": [1, 2], "my_set": [1, 2, 3]},
Review Comment:
same https://github.com/apache/airflow/pull/63871/changes#r3212524999
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
wjddn279 commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3212524999
##
airflow-core/tests/unit/models/test_renderedtifields.py:
##
@@ -116,11 +116,11 @@ def teardown_method(self):
pytest.param([], [], id="list"),
pytest.param({}, {}, id="empty_dict"),
pytest.param((), [], id="empty_tuple"),
-pytest.param(set(), "set()", id="empty_set"),
+pytest.param(set(), [], id="empty_set"),
Review Comment:
Update tests to reflect logic change: `set` and `frozenset` are now
converted to `list` instead of being string-cast.
##
airflow-core/tests/unit/models/test_renderedtifields.py:
##
@@ -116,11 +116,11 @@ def teardown_method(self):
pytest.param([], [], id="list"),
pytest.param({}, {}, id="empty_dict"),
pytest.param((), [], id="empty_tuple"),
-pytest.param(set(), "set()", id="empty_set"),
+pytest.param(set(), [], id="empty_set"),
pytest.param("test-string", "test-string", id="string"),
pytest.param({"foo": "bar"}, {"foo": "bar"}, id="dict"),
pytest.param(("foo", "bar"), ["foo", "bar"], id="tuple"),
-pytest.param({"foo"}, "{'foo'}", id="set"),
+pytest.param({"foo"}, ["foo"], id="set"),
Review Comment:
same https://github.com/apache/airflow/pull/63871/changes#r3212524999
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
wjddn279 commented on code in PR #63871: URL: https://github.com/apache/airflow/pull/63871#discussion_r3207320050 ## airflow-core/tests/unit/serialization/test_dag_serialization.py: ## @@ -704,6 +706,17 @@ def test_deserialization_across_process(self): for dag_id in stringified_dags: self.validate_deserialized_dag(stringified_dags[dag_id], dags[dag_id]) [email protected]_test +@conf_vars({("core", "load_examples"): "false"}) +def test_reserialize_should_make_equal_hash_with_dag_processor(self): +dagbag1 = DagBag(TEST_DAGS_FOLDER / "test_dag_decorator_version.py") +hash_result1 = LazyDeserializedDAG.from_dag(next(iter(dagbag1.dags.values(.hash + +dagbag2 = DagBag(TEST_DAGS_FOLDER / "test_dag_decorator_version.py") +hash_result2 = LazyDeserializedDAG.from_dag(next(iter(dagbag2.dags.values(.hash + +assert hash_result1 == hash_result2 Review Comment: migrate from https://github.com/apache/airflow/pull/65705 error when using AS-IS logic ``` FAILED airflow-core/tests/unit/serialization/test_dag_serialization.py::TestStringifiedDAGs::test_reserialize_should_make_equal_hash_with_dag_processor - AssertionError: assert equals failed 'c2b44f25dbfff98d131ccf67824081 '741b8de117c6d1c8a7cc7f6530c47d 3b' 1e' ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
wjddn279 commented on code in PR #63871: URL: https://github.com/apache/airflow/pull/63871#discussion_r3207321777 ## airflow-core/src/airflow/serialization/helpers.py: ## @@ -32,7 +32,7 @@ from airflow.timetables.base import Timetable as CoreTimetable -def serialize_template_field(template_field: Any, name: str) -> str | dict | list | int | float: +def serialize_template_field(template_field: Any, name: str) -> str | dict | list | int | float | bool: Review Comment: patched! -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
ephraimbuddy commented on code in PR #63871: URL: https://github.com/apache/airflow/pull/63871#discussion_r3200721429 ## airflow-core/tests/unit/serialization/test_dag_serialization.py: ## @@ -704,6 +706,17 @@ def test_deserialization_across_process(self): for dag_id in stringified_dags: self.validate_deserialized_dag(stringified_dags[dag_id], dags[dag_id]) [email protected]_test +@conf_vars({("core", "load_examples"): "false"}) +def test_reserialize_should_make_equal_hash_with_dag_processor(self): +dagbag1 = DagBag(TEST_DAGS_FOLDER / "test_dag_decorator_version.py") +hash_result1 = LazyDeserializedDAG.from_dag(next(iter(dagbag1.dags.values(.hash + +dagbag2 = DagBag(TEST_DAGS_FOLDER / "test_dag_decorator_version.py") +hash_result2 = LazyDeserializedDAG.from_dag(next(iter(dagbag2.dags.values(.hash + +assert hash_result1 == hash_result2 Review Comment: Looks like this will still pass with or without the fix? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
ephraimbuddy commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3200697520
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -51,55 +52,43 @@ def is_jsonable(x):
else:
return True
-def translate_tuples_to_lists(obj: Any):
-"""Recursively convert tuples to lists."""
-if isinstance(obj, tuple):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, list):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, dict):
-return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
-return obj
+def serialize_object(obj):
+if obj is None or isinstance(obj, (str, int, float, bool)):
+return obj
-def sort_dict_recursively(obj: Any) -> Any:
-"""Recursively sort dictionaries to ensure consistent ordering."""
if isinstance(obj, dict):
-return {k: sort_dict_recursively(v) for k, v in
sorted(obj.items())}
-if isinstance(obj, list):
-return [sort_dict_recursively(item) for item in obj]
-if isinstance(obj, tuple):
-return tuple(sort_dict_recursively(item) for item in obj)
-return obj
+# Sort dictionaries recursively to ensure consistent string
representation
+# This prevents hash inconsistencies when dict ordering varies
+return {
+k: serialize_object(v)
+for k, v in sorted(obj.items(), key=lambda kv:
(type(kv[0]).__name__, repr(kv[0])))
+}
+
+if isinstance(obj, (list, tuple)):
+return [serialize_object(item) for item in obj]
+
+# Use inspect.getattr_static to bypass any custom __getattr__ /
metaclass magic
+if callable(inspect.getattr_static(obj, "serialize", None)):
+return serialize_object(obj.serialize())
+
+if callable(obj):
+# Use qualified name; default repr embeds memory addresses, which
would change the DAG hash on every parse
+full_qualified_name = qualname(obj, True)
+return f""
+
+# Non-primitive objects without a serialize attribute are converted to
str
+# So they don't break json.dumps downstream
+return str(obj)
max_length = conf.getint("core", "max_templated_field_length")
-if not is_jsonable(template_field):
-try:
-serialized = template_field.serialize()
-except AttributeError:
-if callable(template_field):
-full_qualified_name = qualname(template_field, True)
-serialized = f""
-else:
-serialized = str(template_field)
-if len(serialized) > max_length:
-rendered = redact(serialized, name)
-return truncate_rendered_value(str(rendered), max_length)
-return serialized
-if not template_field and not isinstance(template_field, tuple):
-# Avoid unnecessary serialization steps for empty fields unless they
are tuples
-# and need to be converted to lists
-return template_field
-template_field = translate_tuples_to_lists(template_field)
-# Sort dictionaries recursively to ensure consistent string representation
-# This prevents hash inconsistencies when dict ordering varies
-if isinstance(template_field, dict):
-template_field = sort_dict_recursively(template_field)
-serialized = str(template_field)
-if len(serialized) > max_length:
+serialized = serialize_object(template_field)
+
+if len(str(serialized)) > max_length:
rendered = redact(serialized, name)
Review Comment:
`redact()` is now called on a structured `dict/list/bool` instead of the
`str` it previously always received. The next line then runs
`truncate_rendered_value(str(rendered), max_length)`, which stringifies the
redacted container — producing a `repr`-shape truncated value rather than the
previous string-truncation. Repro: `serialize_template_field({"password": "x" *
5000}, "op_kwargs")`.
```python
if len(str(serialized)) > max_length:
rendered = redact(str(serialized), name)
return truncate_rendered_value(rendered, max_length)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
ephraimbuddy commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3200705762
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -51,55 +52,43 @@ def is_jsonable(x):
else:
return True
-def translate_tuples_to_lists(obj: Any):
-"""Recursively convert tuples to lists."""
-if isinstance(obj, tuple):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, list):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, dict):
-return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
-return obj
+def serialize_object(obj):
+if obj is None or isinstance(obj, (str, int, float, bool)):
+return obj
-def sort_dict_recursively(obj: Any) -> Any:
-"""Recursively sort dictionaries to ensure consistent ordering."""
if isinstance(obj, dict):
-return {k: sort_dict_recursively(v) for k, v in
sorted(obj.items())}
-if isinstance(obj, list):
-return [sort_dict_recursively(item) for item in obj]
-if isinstance(obj, tuple):
-return tuple(sort_dict_recursively(item) for item in obj)
-return obj
+# Sort dictionaries recursively to ensure consistent string
representation
+# This prevents hash inconsistencies when dict ordering varies
+return {
+k: serialize_object(v)
+for k, v in sorted(obj.items(), key=lambda kv:
(type(kv[0]).__name__, repr(kv[0])))
+}
+
+if isinstance(obj, (list, tuple)):
+return [serialize_object(item) for item in obj]
+
+# Use inspect.getattr_static to bypass any custom __getattr__ /
metaclass magic
+if callable(inspect.getattr_static(obj, "serialize", None)):
+return serialize_object(obj.serialize())
+
+if callable(obj):
+# Use qualified name; default repr embeds memory addresses, which
would change the DAG hash on every parse
+full_qualified_name = qualname(obj, True)
+return f""
+
+# Non-primitive objects without a serialize attribute are converted to
str
+# So they don't break json.dumps downstream
+return str(obj)
max_length = conf.getint("core", "max_templated_field_length")
-if not is_jsonable(template_field):
-try:
-serialized = template_field.serialize()
-except AttributeError:
-if callable(template_field):
-full_qualified_name = qualname(template_field, True)
-serialized = f""
-else:
-serialized = str(template_field)
-if len(serialized) > max_length:
-rendered = redact(serialized, name)
-return truncate_rendered_value(str(rendered), max_length)
-return serialized
-if not template_field and not isinstance(template_field, tuple):
-# Avoid unnecessary serialization steps for empty fields unless they
are tuples
-# and need to be converted to lists
-return template_field
-template_field = translate_tuples_to_lists(template_field)
-# Sort dictionaries recursively to ensure consistent string representation
-# This prevents hash inconsistencies when dict ordering varies
-if isinstance(template_field, dict):
-template_field = sort_dict_recursively(template_field)
-serialized = str(template_field)
-if len(serialized) > max_length:
+serialized = serialize_object(template_field)
+
+if len(str(serialized)) > max_length:
rendered = redact(serialized, name)
return truncate_rendered_value(str(rendered), max_length)
-return template_field
+
+return serialized if is_jsonable(serialized) else str(serialized)
Review Comment:
After `serialize_object`, every leaf is already a JSON primitive — so this
`is_jsonable` check is dead except for one case: dicts with non-JSON-encodable
keys (tuple/frozenset keys), which `json.dumps` rejects. In that case the
entire dict collapses to `str()`, defeating the recursive walk and
re-introducing the repr-with-memory-addresses problem. Either drop the check
entirely (recommended — `serialize_object `should guarantee `jsonable output)`
or handle exotic keys explicitly inside `serialize_object`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
ephraimbuddy commented on PR #63871: URL: https://github.com/apache/airflow/pull/63871#issuecomment-4396257991 > @ephraimbuddy > > I've written logic that handles those issues all at once. (with #65705). Could you take another look? This now regressed well from earlier. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
ephraimbuddy commented on code in PR #63871: URL: https://github.com/apache/airflow/pull/63871#discussion_r3200599672 ## airflow-core/src/airflow/serialization/helpers.py: ## @@ -32,7 +32,7 @@ from airflow.timetables.base import Timetable as CoreTimetable -def serialize_template_field(template_field: Any, name: str) -> str | dict | list | int | float: +def serialize_template_field(template_field: Any, name: str) -> str | dict | list | int | float | bool: Review Comment: The task-sdk side is a parallel implementation and should be patched too. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
wjddn279 commented on PR #63871: URL: https://github.com/apache/airflow/pull/63871#issuecomment-4395232905 ping @ephraimbuddy ! -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
wjddn279 commented on PR #63871: URL: https://github.com/apache/airflow/pull/63871#issuecomment-4377983800 @ephraimbuddy I've written logic that handles those issues all at once. (with https://github.com/apache/airflow/pull/65705). Could you take another look? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
wjddn279 commented on code in PR #63871: URL: https://github.com/apache/airflow/pull/63871#discussion_r3182616382 ## airflow-core/src/airflow/serialization/helpers.py: ## @@ -32,7 +32,7 @@ from airflow.timetables.base import Timetable as CoreTimetable -def serialize_template_field(template_field: Any, name: str) -> str | dict | list | int | float: +def serialize_template_field(template_field: Any, name: str) -> str | dict | list | int | float | bool: Review Comment: I don't have a clear understanding of how it's actually used in task_runner, so it would be difficult for me to give a proper answer. wdyt? @ephraimbuddy -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
wjddn279 commented on code in PR #63871: URL: https://github.com/apache/airflow/pull/63871#discussion_r3182616382 ## airflow-core/src/airflow/serialization/helpers.py: ## @@ -32,7 +32,7 @@ from airflow.timetables.base import Timetable as CoreTimetable -def serialize_template_field(template_field: Any, name: str) -> str | dict | list | int | float: +def serialize_template_field(template_field: Any, name: str) -> str | dict | list | int | float | bool: Review Comment: wdyt? @ephraimbuddy -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
wjddn279 commented on code in PR #63871: URL: https://github.com/apache/airflow/pull/63871#discussion_r3182600745 ## airflow-core/tests/unit/dags/test_dag_decorator_version.py: ## @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations Review Comment: it is automatically added by prek -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
amoghrajesh commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3182120340
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -32,7 +32,7 @@
from airflow.timetables.base import Timetable as CoreTimetable
-def serialize_template_field(template_field: Any, name: str) -> str | dict |
list | int | float:
+def serialize_template_field(template_field: Any, name: str) -> str | dict |
list | int | float | bool:
Review Comment:
There is a version of this function present in `task_runner.py` too called:
`_serialize_template_field`, do we need this change there as well to keep them
in sync?
##
airflow-core/tests/unit/serialization/test_dag_serialization.py:
##
@@ -704,6 +706,16 @@ def test_deserialization_across_process(self):
for dag_id in stringified_dags:
self.validate_deserialized_dag(stringified_dags[dag_id],
dags[dag_id])
+@conf_vars({("core", "load_examples"): "false"})
Review Comment:
`@pytest.mark.db_test` is needed too?
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -51,55 +51,42 @@ def is_jsonable(x):
else:
return True
-def translate_tuples_to_lists(obj: Any):
-"""Recursively convert tuples to lists."""
-if isinstance(obj, tuple):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, list):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, dict):
-return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
-return obj
+def serialize_object(obj):
+if obj is None or isinstance(obj, (str, int, float, bool)):
+return obj
-def sort_dict_recursively(obj: Any) -> Any:
-"""Recursively sort dictionaries to ensure consistent ordering."""
if isinstance(obj, dict):
-return {k: sort_dict_recursively(v) for k, v in
sorted(obj.items())}
-if isinstance(obj, list):
-return [sort_dict_recursively(item) for item in obj]
-if isinstance(obj, tuple):
-return tuple(sort_dict_recursively(item) for item in obj)
-return obj
+# Sort dictionaries recursively to ensure consistent string
representation
+# This prevents hash inconsistencies when dict ordering varies
+return {
+k: serialize_object(v)
+for k, v in sorted(obj.items(), key=lambda kv:
(type(kv[0]).__name__, repr(kv[0])))
+}
+
+if isinstance(obj, (list, tuple)):
+return [serialize_object(item) for item in obj]
+
+if (serialize_attr := getattr(type(obj), "serialize", None)) and
callable(serialize_attr):
+return serialize_object(obj.serialize())
+
+if callable(obj):
+# to make callable object (e.g. lambda function) static
Review Comment:
This comment just restates what the line does, maybe add the reasoning why
stable serialising is needed instead?
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -51,55 +51,42 @@ def is_jsonable(x):
else:
return True
-def translate_tuples_to_lists(obj: Any):
-"""Recursively convert tuples to lists."""
-if isinstance(obj, tuple):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, list):
-return [translate_tuples_to_lists(item) for item in obj]
-if isinstance(obj, dict):
-return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
-return obj
+def serialize_object(obj):
+if obj is None or isinstance(obj, (str, int, float, bool)):
+return obj
-def sort_dict_recursively(obj: Any) -> Any:
-"""Recursively sort dictionaries to ensure consistent ordering."""
if isinstance(obj, dict):
-return {k: sort_dict_recursively(v) for k, v in
sorted(obj.items())}
-if isinstance(obj, list):
-return [sort_dict_recursively(item) for item in obj]
-if isinstance(obj, tuple):
-return tuple(sort_dict_recursively(item) for item in obj)
-return obj
+# Sort dictionaries recursively to ensure consistent string
representation
+# This prevents hash inconsistencies when dict ordering varies
+return {
+k: serialize_object(v)
+for k, v in sorted(obj.items(), key=lambda kv:
(type(kv[0]).__name__, repr(kv[0])))
+}
+
+if isinstance(obj, (list, tuple)):
+return [serialize_object(item) for item in obj]
+
+if (serialize_attr := getattr(type(obj), "serialize", None)) and
callable(serialize_attr):
Review Comment:
```suggestion
if (_ := getattr(type(obj), "serialize", None)) and
callable(ser
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
ephraimbuddy commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3166962618
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -61,25 +65,32 @@ def translate_tuples_to_lists(obj: Any):
return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
return obj
-def sort_dict_recursively(obj: Any) -> Any:
-"""Recursively sort dictionaries to ensure consistent ordering."""
+def sort_and_make_static_dict_recursively(obj: Any) -> Any:
Review Comment:
I don't think the name change is necessary. This is now mouthful. The
previous name can accommodate the changes.
##
airflow-core/tests/unit/serialization/test_helpers.py:
##
@@ -23,6 +23,8 @@
def test_serialize_template_field_with_very_small_max_length(monkeypatch):
"""Test that truncation message is prioritized even for very small
max_length."""
+from airflow.serialization.helpers import serialize_template_field
Review Comment:
```suggestion
```
Redundant
##
airflow-core/tests/unit/serialization/test_helpers.py:
##
@@ -51,3 +53,34 @@ def test_argnotset_repr_and_str():
assert str(NOTSET) == "NOTSET"
assert repr(SET_DURING_EXECUTION) == "DYNAMIC (set during execution)"
assert str(SET_DURING_EXECUTION) == "DYNAMIC (set during execution)"
+
+
+def test_serialize_template_field_with_dict_value_callable():
+from airflow.serialization.helpers import serialize_template_field
Review Comment:
```suggestion
```
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -61,25 +65,32 @@ def translate_tuples_to_lists(obj: Any):
return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
return obj
-def sort_dict_recursively(obj: Any) -> Any:
-"""Recursively sort dictionaries to ensure consistent ordering."""
+def sort_and_make_static_dict_recursively(obj: Any) -> Any:
+"""Recursively sort dictionaries and make callable value static to
ensure consistent ordering."""
+if callable(obj):
+return make_callable_static(obj)
if isinstance(obj, dict):
-return {k: sort_dict_recursively(v) for k, v in
sorted(obj.items())}
+return {k: sort_and_make_static_dict_recursively(v) for k, v in
sorted(obj.items())}
if isinstance(obj, list):
-return [sort_dict_recursively(item) for item in obj]
+return [sort_and_make_static_dict_recursively(item) for item in
obj]
if isinstance(obj, tuple):
-return tuple(sort_dict_recursively(item) for item in obj)
+return tuple(sort_and_make_static_dict_recursively(item) for item
in obj)
return obj
max_length = conf.getint("core", "max_templated_field_length")
if not is_jsonable(template_field):
try:
-serialized = template_field.serialize()
+# If the callable objects are in dict values, it makes is_jsonable
False.
+# So, exclude dicts here; they are handled separately in the logic
below.
Review Comment:
```suggestion
# Dicts may be non-jsonable because they contain callables;
normalize them here so str() is stable.
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
wjddn279 commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3166260701
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -61,25 +65,32 @@ def translate_tuples_to_lists(obj: Any):
return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
return obj
-def sort_dict_recursively(obj: Any) -> Any:
-"""Recursively sort dictionaries to ensure consistent ordering."""
+def sort_and_make_static_dict_recursively(obj: Any) -> Any:
+"""Recursively sort dictionaries and make callable value static to
ensure consistent ordering."""
+if callable(obj):
+return make_callable_static(obj)
if isinstance(obj, dict):
-return {k: sort_dict_recursively(v) for k, v in
sorted(obj.items())}
+return {k: sort_and_make_static_dict_recursively(v) for k, v in
sorted(obj.items())}
if isinstance(obj, list):
-return [sort_dict_recursively(item) for item in obj]
+return [sort_and_make_static_dict_recursively(item) for item in
obj]
if isinstance(obj, tuple):
-return tuple(sort_dict_recursively(item) for item in obj)
+return tuple(sort_and_make_static_dict_recursively(item) for item
in obj)
return obj
max_length = conf.getint("core", "max_templated_field_length")
if not is_jsonable(template_field):
try:
-serialized = template_field.serialize()
+# If the callable objects are in dict values, it makes is_jsonable
False.
+# So, exclude dicts here; they are handled separately in the logic
below.
+if isinstance(template_field, dict):
+template_field =
sort_and_make_static_dict_recursively(template_field)
+serialized = str(template_field)
Review Comment:
if `str()`is not applied to `serialized` out of dict
```
/usr/python/lib/python3.10/site-packages/sqlalchemy/orm/state.py:596: in
_initialize_instance
with util.safe_reraise():
/usr/python/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:121:
in __exit__
raise exc_value.with_traceback(exc_tb)
/usr/python/lib/python3.10/site-packages/sqlalchemy/orm/state.py:594: in
_initialize_instance
manager.original_init(*mixed[1:], **kwargs)
airflow-core/src/airflow/models/serialized_dag.py:349: in __init__
self.dag_hash = SerializedDagModel.hash(dag_data)
airflow-core/src/airflow/models/serialized_dag.py:378: in hash
data_json = json.dumps(data_, sort_keys=True).encode("utf-8")
/usr/python/lib/python3.10/json/__init__.py:238: in dumps
**kw).encode(obj)
/usr/python/lib/python3.10/json/encoder.py:199: in encode
chunks = self.iterencode(o, _one_shot=True)
/usr/python/lib/python3.10/json/encoder.py:257: in iterencode
return _iterencode(o, 0)
/usr/python/lib/python3.10/json/encoder.py:179: in default
raise TypeError(f'Object of type {o.__class__.__name__} '
E TypeError: Object of type PlainXComArg is not JSON serializable
```
the values has object type (e.g. `PlainXComArg`) must be converted to
`string`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
Copilot commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3066484773
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -61,25 +65,32 @@ def translate_tuples_to_lists(obj: Any):
return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
return obj
-def sort_dict_recursively(obj: Any) -> Any:
-"""Recursively sort dictionaries to ensure consistent ordering."""
+def sort_and_make_static_dict_recursively(obj: Any) -> Any:
+"""Recursively sort dictionaries and make callable value static to
ensure consistent ordering."""
+if callable(obj):
+return make_callable_static(obj)
if isinstance(obj, dict):
-return {k: sort_dict_recursively(v) for k, v in
sorted(obj.items())}
+return {k: sort_and_make_static_dict_recursively(v) for k, v in
sorted(obj.items())}
if isinstance(obj, list):
-return [sort_dict_recursively(item) for item in obj]
+return [sort_and_make_static_dict_recursively(item) for item in
obj]
if isinstance(obj, tuple):
-return tuple(sort_dict_recursively(item) for item in obj)
+return tuple(sort_and_make_static_dict_recursively(item) for item
in obj)
return obj
max_length = conf.getint("core", "max_templated_field_length")
if not is_jsonable(template_field):
try:
-serialized = template_field.serialize()
+# If the callable objects are in dict values, it makes is_jsonable
False.
+# So, exclude dicts here; they are handled separately in the logic
below.
+if isinstance(template_field, dict):
+template_field =
sort_and_make_static_dict_recursively(template_field)
+serialized = str(template_field)
Review Comment:
`sort_and_make_static_dict_recursively()` uses `sorted(obj.items())` when
serializing *non-JSONable* dicts (the new callable-in-values case). This can
raise `TypeError` for dicts with mixed / non-orderable key types (e.g. `{1:
"a", "b": }`) where previously this path fell back to
`str(template_field)` and would not error. Consider guarding the sort with
`try/except TypeError` (fallback to unsorted iteration) or sorting with a
stable key function (e.g. based on `str(key)`) to avoid breaking DAG
serialization on such inputs.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
wjddn279 commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3051440848
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -61,25 +65,28 @@ def translate_tuples_to_lists(obj: Any):
return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
return obj
-def sort_dict_recursively(obj: Any) -> Any:
-"""Recursively sort dictionaries to ensure consistent ordering."""
+def sort_and_make_static_dict_recursively(obj: Any) -> Any:
+"""Recursively sort dictionaries and make callable value static to
ensure consistent ordering."""
+if callable(obj):
+return make_callable_static(obj)
if isinstance(obj, dict):
-return {k: sort_dict_recursively(v) for k, v in
sorted(obj.items())}
+return {k: sort_and_make_static_dict_recursively(v) for k, v in
sorted(obj.items())}
if isinstance(obj, list):
-return [sort_dict_recursively(item) for item in obj]
+return [sort_and_make_static_dict_recursively(item) for item in
obj]
if isinstance(obj, tuple):
-return tuple(sort_dict_recursively(item) for item in obj)
-return obj
+return tuple(sort_and_make_static_dict_recursively(item) for item
in obj)
+return str(obj)
max_length = conf.getint("core", "max_templated_field_length")
-if not is_jsonable(template_field):
+# If the callable objects is in dict value, it makes is_jsonable False.
+# So, filter the case in here and handle it downside
+if not is_jsonable(template_field) and not isinstance(template_field,
dict):
Review Comment:
don't need to affect to another logic
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
wjddn279 commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3050308908
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -61,25 +65,28 @@ def translate_tuples_to_lists(obj: Any):
return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
return obj
-def sort_dict_recursively(obj: Any) -> Any:
-"""Recursively sort dictionaries to ensure consistent ordering."""
+def sort_and_make_static_dict_recursively(obj: Any) -> Any:
+"""Recursively sort dictionaries and make callable value static to
ensure consistent ordering."""
+if callable(obj):
+return make_callable_static(obj)
if isinstance(obj, dict):
-return {k: sort_dict_recursively(v) for k, v in
sorted(obj.items())}
+return {k: sort_and_make_static_dict_recursively(v) for k, v in
sorted(obj.items())}
if isinstance(obj, list):
-return [sort_dict_recursively(item) for item in obj]
+return [sort_and_make_static_dict_recursively(item) for item in
obj]
if isinstance(obj, tuple):
-return tuple(sort_dict_recursively(item) for item in obj)
-return obj
+return tuple(sort_and_make_static_dict_recursively(item) for item
in obj)
+return str(obj)
Review Comment:
make sense
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Re: [PR] fix serialize_template_field handling callable value in dict [airflow]
Copilot commented on code in PR #63871:
URL: https://github.com/apache/airflow/pull/63871#discussion_r3025333515
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -61,25 +65,28 @@ def translate_tuples_to_lists(obj: Any):
return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
return obj
-def sort_dict_recursively(obj: Any) -> Any:
-"""Recursively sort dictionaries to ensure consistent ordering."""
+def sort_and_make_static_dict_recursively(obj: Any) -> Any:
+"""Recursively sort dictionaries and make callable value static to
ensure consistent ordering."""
+if callable(obj):
+return make_callable_static(obj)
if isinstance(obj, dict):
-return {k: sort_dict_recursively(v) for k, v in
sorted(obj.items())}
+return {k: sort_and_make_static_dict_recursively(v) for k, v in
sorted(obj.items())}
if isinstance(obj, list):
-return [sort_dict_recursively(item) for item in obj]
+return [sort_and_make_static_dict_recursively(item) for item in
obj]
if isinstance(obj, tuple):
-return tuple(sort_dict_recursively(item) for item in obj)
-return obj
+return tuple(sort_and_make_static_dict_recursively(item) for item
in obj)
+return str(obj)
max_length = conf.getint("core", "max_templated_field_length")
-if not is_jsonable(template_field):
+# If the callable objects is in dict value, it makes is_jsonable False.
+# So, filter the case in here and handle it downside
+if not is_jsonable(template_field) and not isinstance(template_field,
dict):
Review Comment:
This special-casing only skips the fallback serialization path for `dict`,
but the same instability can occur for other containers (e.g., a top-level
`list`/`tuple` containing a callable) because `is_jsonable` will be false and
the code will fall back to `str(template_field)`, which includes the function
object's memory address. Suggestion: extend the container handling to include
`list`/`tuple` (and potentially nested containers generally), so callables
anywhere in the templated structure get stabilized.
##
airflow-core/src/airflow/serialization/helpers.py:
##
@@ -61,25 +65,28 @@ def translate_tuples_to_lists(obj: Any):
return {key: translate_tuples_to_lists(value) for key, value in
obj.items()}
return obj
-def sort_dict_recursively(obj: Any) -> Any:
-"""Recursively sort dictionaries to ensure consistent ordering."""
+def sort_and_make_static_dict_recursively(obj: Any) -> Any:
+"""Recursively sort dictionaries and make callable value static to
ensure consistent ordering."""
+if callable(obj):
+return make_callable_static(obj)
if isinstance(obj, dict):
-return {k: sort_dict_recursively(v) for k, v in
sorted(obj.items())}
+return {k: sort_and_make_static_dict_recursively(v) for k, v in
sorted(obj.items())}
if isinstance(obj, list):
-return [sort_dict_recursively(item) for item in obj]
+return [sort_and_make_static_dict_recursively(item) for item in
obj]
if isinstance(obj, tuple):
-return tuple(sort_dict_recursively(item) for item in obj)
-return obj
+return tuple(sort_and_make_static_dict_recursively(item) for item
in obj)
+return str(obj)
Review Comment:
Returning `str(obj)` for all leaf values changes the data types of *every*
non-container value inside dicts/lists/tuples (e.g., ints/bools/None become
strings). This is a behavior change unrelated to callable stabilization and can
alter downstream consumers that expect preserved types. Suggestion: only
convert callables to the static string, and otherwise return the original `obj`
(or only stringify non-JSON-serializable leaf objects), so numbers/bools/None
remain unchanged.
```suggestion
return obj
```
##
airflow-core/tests/unit/serialization/test_helpers.py:
##
@@ -16,16 +16,38 @@
# under the License.
from __future__ import annotations
+from airflow.serialization.helpers import serialize_template_field
+
def test_serialize_template_field_with_very_small_max_length(monkeypatch):
"""Test that truncation message is prioritized even for very small
max_length."""
monkeypatch.setenv("AIRFLOW__CORE__MAX_TEMPLATED_FIELD_LENGTH", "1")
-from airflow.serialization.helpers import serialize_template_field
-
result = serialize_template_field("This is a long string", "test")
# The truncation message should be shown even if it exceeds max_length
# This ensures users always see why content is truncated
assert result
assert "Truncated. You can change this behaviour" in result
+
+
+def test_serialize_template_field_with_dict_value_callable():
+def fn_returns_callable():
