amoghrajesh commented on code in PR #74421:
URL: https://github.com/apache/airflow/pull/74421#discussion_r4534139658
##########
task-sdk/tests/task_sdk/execution_time/test_context.py:
##########
@@ -2272,6 +2354,48 @@ def
test_get_with_custom_backend_removes_decoration_marker(self, mock_supervisor
"s3://bucket/assets/orders/watermark"
)
+ def test_set_with_custom_backend_returning_none_stores_value_inline(self,
mock_supervisor_comms):
+ """A backend that returns None keeps the value inline, with no marker
around it."""
+ mock_supervisor_comms.send.return_value = OKResponse(ok=True)
+
+ backend = MagicMock(spec=BaseStoreBackend)
+ backend.serialize_asset_state_store_to_ref.return_value = None
+
+ with patch(
+
"airflow.sdk.execution_time.context._get_worker_state_store_backend",
return_value=backend
+ ):
+ AssetStateStoreAccessor(name=self.ASSET_NAME).set("watermark",
"2026-05-01")
+
+ mock_supervisor_comms.send.assert_called_once_with(
+ SetAssetStateStoreByName(name=self.ASSET_NAME, key="watermark",
value="2026-05-01")
+ )
+
+ def
test_get_with_custom_backend_returns_unmarked_value_without_deserializing(
Review Comment:
Same as above
##########
task-sdk/tests/task_sdk/execution_time/test_context.py:
##########
@@ -1962,6 +1963,87 @@ def
test_get_with_custom_backend_removes_decoration_marker(self, mock_supervisor
assert result == {"rows": 123}
backend.deserialize_task_state_store_from_ref.assert_called_once_with("s3://bucket/ti_123/job_id")
+ def test_set_with_custom_backend_returning_none_stores_value_inline(self,
mock_supervisor_comms):
+ """A backend that returns None keeps the value inline, with no marker
around it."""
+ mock_supervisor_comms.send.return_value = OKResponse(ok=True)
+
+ backend = MagicMock(spec=BaseStoreBackend)
+ backend.serialize_task_state_store_to_ref.return_value = None
+
+ with (
+
patch("airflow.sdk.execution_time.context._get_worker_state_store_backend",
return_value=backend),
+ conf_vars({("state_store", "default_retention_days"): "0"}),
+ ):
+ TaskStateStoreAccessor(ti_id=self.TI_ID,
scope=self.SCOPE).set("progress", {"count": 5})
+
+ mock_supervisor_comms.send.assert_called_once_with(
+ SetTaskStateStore(ti_id=self.TI_ID, key="progress",
value={"count": 5}, expires_at=None)
+ )
+
+ def
test_get_with_custom_backend_returns_unmarked_value_without_deserializing(
Review Comment:
This will fail on main because of the warning. It patches the module's
logger to assert no warning. Use structured caplog instead
##########
task-sdk/src/airflow/sdk/execution_time/context.py:
##########
@@ -163,8 +163,23 @@ def _wrap_external_ref(ref: str) -> dict[str, JsonValue]:
return {_EXTERNAL_STATE_REF_KEY: ref}
-def _unwrap_external_ref(stored: dict) -> str | None:
- return stored.get(_EXTERNAL_STATE_REF_KEY)
+def _unwrap_external_ref(stored: JsonValue) -> str | None:
+ """Return the reference inside a ``{marker: ref}`` wrapper, or ``None``
for any other value."""
+ if isinstance(stored, dict) and stored.keys() == {_EXTERNAL_STATE_REF_KEY}:
+ ref = stored[_EXTERNAL_STATE_REF_KEY]
+ if isinstance(ref, str):
+ return ref
+ return None
+
+
+def _validate_state_store_value(value: JsonValue) -> None:
+ if value is None:
+ raise ValueError("Cannot set value as None")
+ # get() would read such a value back as a backend reference
Review Comment:
```suggestion
# Older sdk's read any dict holding this key as a backend reference.
```
##########
task-sdk/tests/task_sdk/execution_time/test_context.py:
##########
@@ -1962,6 +1963,87 @@ def
test_get_with_custom_backend_removes_decoration_marker(self, mock_supervisor
assert result == {"rows": 123}
backend.deserialize_task_state_store_from_ref.assert_called_once_with("s3://bucket/ti_123/job_id")
+ def test_set_with_custom_backend_returning_none_stores_value_inline(self,
mock_supervisor_comms):
+ """A backend that returns None keeps the value inline, with no marker
around it."""
+ mock_supervisor_comms.send.return_value = OKResponse(ok=True)
+
+ backend = MagicMock(spec=BaseStoreBackend)
+ backend.serialize_task_state_store_to_ref.return_value = None
+
+ with (
+
patch("airflow.sdk.execution_time.context._get_worker_state_store_backend",
return_value=backend),
Review Comment:
Add autospec=True please
##########
task-sdk/tests/task_sdk/execution_time/test_context.py:
##########
@@ -1962,6 +1963,87 @@ def
test_get_with_custom_backend_removes_decoration_marker(self, mock_supervisor
assert result == {"rows": 123}
backend.deserialize_task_state_store_from_ref.assert_called_once_with("s3://bucket/ti_123/job_id")
+ def test_set_with_custom_backend_returning_none_stores_value_inline(self,
mock_supervisor_comms):
+ """A backend that returns None keeps the value inline, with no marker
around it."""
+ mock_supervisor_comms.send.return_value = OKResponse(ok=True)
+
+ backend = MagicMock(spec=BaseStoreBackend)
+ backend.serialize_task_state_store_to_ref.return_value = None
+
+ with (
+
patch("airflow.sdk.execution_time.context._get_worker_state_store_backend",
return_value=backend),
+ conf_vars({("state_store", "default_retention_days"): "0"}),
+ ):
+ TaskStateStoreAccessor(ti_id=self.TI_ID,
scope=self.SCOPE).set("progress", {"count": 5})
+
+ mock_supervisor_comms.send.assert_called_once_with(
+ SetTaskStateStore(ti_id=self.TI_ID, key="progress",
value={"count": 5}, expires_at=None)
+ )
+
+ def
test_get_with_custom_backend_returns_unmarked_value_without_deserializing(
+ self, mock_supervisor_comms
+ ):
+ """A value stored inline (or written through the REST API) is returned
as-is."""
+ mock_supervisor_comms.send.return_value =
TaskStateStoreResult(value={"count": 5})
+
+ backend = MagicMock(spec=BaseStoreBackend)
+
+ with (
+
patch("airflow.sdk.execution_time.context._get_worker_state_store_backend",
return_value=backend),
+ patch("airflow.sdk.execution_time.context.log",
spec=structlog.stdlib.BoundLogger) as mock_log,
+ ):
+ result = TaskStateStoreAccessor(ti_id=self.TI_ID,
scope=self.SCOPE).get("progress")
+
+ assert result == {"count": 5}
+ backend.deserialize_task_state_store_from_ref.assert_not_called()
+ mock_log.warning.assert_not_called()
+
+ @pytest.mark.parametrize(
+ "stored",
+ [
+ pytest.param({"__airflow_state_ref__": "/tmp/x", "other": 1},
id="marker-with-other-keys"),
+ pytest.param({"__airflow_state_ref__": 1},
id="non-string-marker-value"),
+ ],
+ )
+ def
test_get_with_custom_backend_returns_marker_lookalike_without_deserializing(
+ self, mock_supervisor_comms, stored
+ ):
+ """Only an exact ``{marker: str}`` wrapper is treated as a backend
reference."""
+ mock_supervisor_comms.send.return_value =
TaskStateStoreResult(value=stored)
+
+ backend = MagicMock(spec=BaseStoreBackend)
+
+ with patch(
+
"airflow.sdk.execution_time.context._get_worker_state_store_backend",
return_value=backend
+ ):
+ result = TaskStateStoreAccessor(ti_id=self.TI_ID,
scope=self.SCOPE).get("job_id")
+
+ assert result == stored
+ backend.deserialize_task_state_store_from_ref.assert_not_called()
+
+ def test_get_with_custom_backend_resolves_empty_string_ref(self,
mock_supervisor_comms):
+ """An empty reference is still a reference: set() wraps it, so get()
must resolve it."""
+ mock_supervisor_comms.send.return_value =
TaskStateStoreResult(value=_wrap_external_ref(""))
+
+ backend = MagicMock(spec=BaseStoreBackend)
+ backend.deserialize_task_state_store_from_ref.return_value = "resolved"
+
+ with patch(
+
"airflow.sdk.execution_time.context._get_worker_state_store_backend",
return_value=backend
+ ):
+ result = TaskStateStoreAccessor(ti_id=self.TI_ID,
scope=self.SCOPE).get("job_id")
+
+ assert result == "resolved"
+
backend.deserialize_task_state_store_from_ref.assert_called_once_with("")
+
+ def test_set_rejects_value_with_reserved_marker_key(self,
mock_supervisor_comms):
Review Comment:
It only tests the exact `{marker: "/tmp/x"}` shape. The broader rule (any
dict holding the key, which old readers misread) is untested. Parametrize with
`{"__airflow_state_ref__": "x", "other": 1}`.
##########
task-sdk/src/airflow/sdk/execution_time/context.py:
##########
@@ -163,8 +163,23 @@ def _wrap_external_ref(ref: str) -> dict[str, JsonValue]:
return {_EXTERNAL_STATE_REF_KEY: ref}
-def _unwrap_external_ref(stored: dict) -> str | None:
- return stored.get(_EXTERNAL_STATE_REF_KEY)
+def _unwrap_external_ref(stored: JsonValue) -> str | None:
+ """Return the reference inside a ``{marker: ref}`` wrapper, or ``None``
for any other value."""
+ if isinstance(stored, dict) and stored.keys() == {_EXTERNAL_STATE_REF_KEY}:
+ ref = stored[_EXTERNAL_STATE_REF_KEY]
+ if isinstance(ref, str):
+ return ref
+ return None
+
+
+def _validate_state_store_value(value: JsonValue) -> None:
Review Comment:
`set()` now raises `ValueError` for any value with a top level
`__airflow_state_ref__` key, even with no backend configured. That is a new
rejection of input that used to be accepted. List it in the same newsfragment
too
##########
shared/state/src/airflow_shared/state/__init__.py:
##########
Review Comment:
Since this is a behaviour change for custom backends, maybe we should add a
significant newsfragment?
##########
providers/common/io/tests/unit/common/io/state_store/test_backend.py:
##########
@@ -286,20 +286,20 @@ def test_asset_serialize_offloads_to_storage(self,
asset_scope, base_path):
ref = store.serialize_asset_state_store_to_ref(value={"x": 1},
key="k", scope=asset_scope)
assert ref.startswith("file://")
- def test_task_serialize_to_db_when_below_threshold(self, task_scope,
base_path):
- with conf_vars(
- {
- ("common.io", "state_store_objectstorage_path"): base_path,
- ("common.io", "state_store_objectstorage_threshold"): "10000",
- }
- ):
- backend._get_threshold.cache_clear()
- store = StateStoreObjectStorageBackend()
- ref = store.serialize_task_state_store_to_ref(value={"x": 1},
key="k", scope=task_scope)
- assert not ref.startswith("file://")
- assert store.deserialize_task_state_store_from_ref(ref) == {"x": 1}
-
- def test_asset_serialize_to_db_when_below_threshold(self, asset_scope,
base_path):
+ @pytest.mark.parametrize(
+ ("kind", "scope_fixture"),
+ [pytest.param("task", "task_scope", id="task"), pytest.param("asset",
"asset_scope", id="asset")],
+ )
+ @pytest.mark.parametrize(
+ "base_default",
+ [pytest.param(None, id="sdk-keeps-inline"), pytest.param('{"x": 1}',
id="sdk-wraps-every-return")],
+ )
+ def test_serialize_below_threshold_returns_base_default(
Review Comment:
```suggestion
def test_serialize_below_threshold_defers_to_base_class(
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
