This is an automated email from the ASF dual-hosted git repository.
amoghdesai pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new a6da8df0d7b AIP-72: Allow retrieving Variable from Task Context
(#45431)
a6da8df0d7b is described below
commit a6da8df0d7bb1958b7e3ab558e06d16ce72b5327
Author: Amogh Desai <[email protected]>
AuthorDate: Tue Jan 7 11:35:43 2025 +0530
AIP-72: Allow retrieving Variable from Task Context (#45431)
---
task_sdk/src/airflow/sdk/__init__.py | 1 +
task_sdk/src/airflow/sdk/api/client.py | 15 ++++-
task_sdk/src/airflow/sdk/definitions/variable.py | 41 ++++++++++++
task_sdk/src/airflow/sdk/execution_time/comms.py | 2 +-
task_sdk/src/airflow/sdk/execution_time/context.py | 63 +++++++++++++++++-
.../src/airflow/sdk/execution_time/supervisor.py | 8 ++-
.../src/airflow/sdk/execution_time/task_runner.py | 10 +--
task_sdk/tests/api/test_client.py | 36 +++++++---
task_sdk/tests/execution_time/test_context.py | 76 +++++++++++++++++++++-
task_sdk/tests/execution_time/test_task_runner.py | 58 ++++++++++++++++-
10 files changed, 285 insertions(+), 25 deletions(-)
diff --git a/task_sdk/src/airflow/sdk/__init__.py
b/task_sdk/src/airflow/sdk/__init__.py
index 13f2819a1c6..1117e946f8e 100644
--- a/task_sdk/src/airflow/sdk/__init__.py
+++ b/task_sdk/src/airflow/sdk/__init__.py
@@ -46,6 +46,7 @@ __lazy_imports: dict[str, str] = {
"EdgeModifier": ".definitions.edges",
"Label": ".definitions.edges",
"Connection": ".definitions.connection",
+ "Variable": ".definitions.variable",
}
diff --git a/task_sdk/src/airflow/sdk/api/client.py
b/task_sdk/src/airflow/sdk/api/client.py
index 4c6a02efe94..0ef3efe95f1 100644
--- a/task_sdk/src/airflow/sdk/api/client.py
+++ b/task_sdk/src/airflow/sdk/api/client.py
@@ -189,9 +189,20 @@ class VariableOperations:
def __init__(self, client: Client):
self.client = client
- def get(self, key: str) -> VariableResponse:
+ def get(self, key: str) -> VariableResponse | ErrorResponse:
"""Get a variable from the API server."""
- resp = self.client.get(f"variables/{key}")
+ try:
+ resp = self.client.get(f"variables/{key}")
+ except ServerResponseError as e:
+ if e.response.status_code == HTTPStatus.NOT_FOUND:
+ log.error(
+ "Variable not found",
+ key=key,
+ detail=e.detail,
+ status_code=e.response.status_code,
+ )
+ return ErrorResponse(error=ErrorType.VARIABLE_NOT_FOUND,
detail={"key": key})
+ raise
return VariableResponse.model_validate_json(resp.read())
def set(self, key: str, value: str | None, description: str | None = None):
diff --git a/task_sdk/src/airflow/sdk/definitions/variable.py
b/task_sdk/src/airflow/sdk/definitions/variable.py
new file mode 100644
index 00000000000..5f458580065
--- /dev/null
+++ b/task_sdk/src/airflow/sdk/definitions/variable.py
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from typing import Any
+
+import attrs
+
+
[email protected]
+class Variable:
+ """
+ A generic way to store and retrieve arbitrary content or settings as a
simple key/value store.
+
+ :param key: The variable key.
+ :param value: The variable value.
+ :param description: The variable description.
+
+ """
+
+ key: str
+ # keeping as any for supporting deserialize_json
+ value: Any | None = None
+ description: str | None = None
+
+ # TODO: Extend this definition for reading/writing variables without
context
diff --git a/task_sdk/src/airflow/sdk/execution_time/comms.py
b/task_sdk/src/airflow/sdk/execution_time/comms.py
index b90787ca4cf..e1007d1f2fc 100644
--- a/task_sdk/src/airflow/sdk/execution_time/comms.py
+++ b/task_sdk/src/airflow/sdk/execution_time/comms.py
@@ -121,7 +121,7 @@ class VariableResult(VariableResponse):
VariableResponse is autogenerated from the API schema, so we need to
convert it to VariableResult
for communication between the Supervisor and the task process.
"""
- return cls(**variable_response.model_dump())
+ return cls(**variable_response.model_dump(exclude_defaults=True),
type="VariableResult")
class ErrorResponse(BaseModel):
diff --git a/task_sdk/src/airflow/sdk/execution_time/context.py
b/task_sdk/src/airflow/sdk/execution_time/context.py
index 30295a84f9d..d96f5aeda2c 100644
--- a/task_sdk/src/airflow/sdk/execution_time/context.py
+++ b/task_sdk/src/airflow/sdk/execution_time/context.py
@@ -21,19 +21,31 @@ from typing import TYPE_CHECKING, Any
import structlog
from airflow.sdk.exceptions import AirflowRuntimeError, ErrorType
+from airflow.sdk.types import NOTSET
if TYPE_CHECKING:
from airflow.sdk.definitions.connection import Connection
- from airflow.sdk.execution_time.comms import ConnectionResult
+ from airflow.sdk.definitions.variable import Variable
+ from airflow.sdk.execution_time.comms import ConnectionResult,
VariableResult
-def _convert_connection_result_conn(conn_result: ConnectionResult):
+def _convert_connection_result_conn(conn_result: ConnectionResult) ->
Connection:
from airflow.sdk.definitions.connection import Connection
# `by_alias=True` is used to convert the `schema` field to `schema_` in
the Connection model
return Connection(**conn_result.model_dump(exclude={"type"},
by_alias=True))
+def _convert_variable_result_to_variable(var_result: VariableResult,
deserialize_json: bool) -> Variable:
+ from airflow.sdk.definitions.variable import Variable
+
+ if deserialize_json:
+ import json
+
+ var_result.value = json.loads(var_result.value) # type: ignore
+ return Variable(**var_result.model_dump(exclude={"type"}))
+
+
def _get_connection(conn_id: str) -> Connection:
# TODO: This should probably be moved to a separate module like
`airflow.sdk.execution_time.comms`
# or `airflow.sdk.execution_time.connection`
@@ -54,6 +66,26 @@ def _get_connection(conn_id: str) -> Connection:
return _convert_connection_result_conn(msg)
+def _get_variable(key: str, deserialize_json: bool) -> Variable:
+ # TODO: This should probably be moved to a separate module like
`airflow.sdk.execution_time.comms`
+ # or `airflow.sdk.execution_time.variable`
+ # A reason to not move it to `airflow.sdk.execution_time.comms` is that
it
+ # will make that module depend on Task SDK, which is not ideal because
we intend to
+ # keep Task SDK as a separate package than execution time mods.
+ from airflow.sdk.execution_time.comms import ErrorResponse, GetVariable
+ from airflow.sdk.execution_time.task_runner import SUPERVISOR_COMMS
+
+ log = structlog.get_logger(logger_name="task")
+ SUPERVISOR_COMMS.send_request(log=log, msg=GetVariable(key=key))
+ msg = SUPERVISOR_COMMS.get_message()
+ if isinstance(msg, ErrorResponse):
+ raise AirflowRuntimeError(msg)
+
+ if TYPE_CHECKING:
+ assert isinstance(msg, VariableResult)
+ return _convert_variable_result_to_variable(msg, deserialize_json)
+
+
class ConnectionAccessor:
"""Wrapper to access Connection entries in template."""
@@ -76,3 +108,30 @@ class ConnectionAccessor:
if e.error.error == ErrorType.CONNECTION_NOT_FOUND:
return default_conn
raise
+
+
+class VariableAccessor:
+ """Wrapper to access Variable values in template."""
+
+ def __init__(self, deserialize_json: bool) -> None:
+ self._deserialize_json = deserialize_json
+
+ def __eq__(self, other):
+ if not isinstance(other, VariableAccessor):
+ return False
+ # All instances of VariableAccessor are equal since it is a stateless
dynamic accessor
+ return True
+
+ def __repr__(self) -> str:
+ return "<VariableAccessor (dynamic access)>"
+
+ def __getattr__(self, key: str) -> Any:
+ return _get_variable(key, self._deserialize_json)
+
+ def get(self, key, default_var: Any = NOTSET) -> Any:
+ try:
+ return _get_variable(key, self._deserialize_json)
+ except AirflowRuntimeError as e:
+ if e.error.error == ErrorType.VARIABLE_NOT_FOUND:
+ return default_var
+ raise
diff --git a/task_sdk/src/airflow/sdk/execution_time/supervisor.py
b/task_sdk/src/airflow/sdk/execution_time/supervisor.py
index 811d1ce86a6..2363e927b46 100644
--- a/task_sdk/src/airflow/sdk/execution_time/supervisor.py
+++ b/task_sdk/src/airflow/sdk/execution_time/supervisor.py
@@ -58,6 +58,7 @@ from airflow.sdk.api.datamodels._generated import (
IntermediateTIState,
TaskInstance,
TerminalTIState,
+ VariableResponse,
)
from airflow.sdk.execution_time.comms import (
ConnectionResult,
@@ -722,8 +723,11 @@ class WatchedSubprocess:
resp = conn.model_dump_json().encode()
elif isinstance(msg, GetVariable):
var = self.client.variables.get(msg.key)
- var_result = VariableResult.from_variable_response(var)
- resp = var_result.model_dump_json().encode()
+ if isinstance(var, VariableResponse):
+ var_result = VariableResult.from_variable_response(var)
+ resp = var_result.model_dump_json(exclude_unset=True).encode()
+ else:
+ resp = var.model_dump_json().encode()
elif isinstance(msg, GetXCom):
xcom = self.client.xcoms.get(msg.dag_id, msg.run_id, msg.task_id,
msg.key, msg.map_index)
xcom_result = XComResult.from_xcom_response(xcom)
diff --git a/task_sdk/src/airflow/sdk/execution_time/task_runner.py
b/task_sdk/src/airflow/sdk/execution_time/task_runner.py
index effea9f6a7e..4f8a4f0045c 100644
--- a/task_sdk/src/airflow/sdk/execution_time/task_runner.py
+++ b/task_sdk/src/airflow/sdk/execution_time/task_runner.py
@@ -44,7 +44,7 @@ from airflow.sdk.execution_time.comms import (
ToTask,
XComResult,
)
-from airflow.sdk.execution_time.context import ConnectionAccessor
+from airflow.sdk.execution_time.context import ConnectionAccessor,
VariableAccessor
if TYPE_CHECKING:
from structlog.typing import FilteringBoundLogger as Logger
@@ -85,10 +85,10 @@ class RuntimeTaskInstance(TaskInstance):
# "prev_end_date_success": get_prev_end_date_success(),
# "test_mode": task_instance.test_mode,
# "triggering_asset_events":
lazy_object_proxy.Proxy(get_triggering_events),
- # "var": {
- # "json": VariableAccessor(deserialize_json=True),
- # "value": VariableAccessor(deserialize_json=False),
- # },
+ "var": {
+ "json": VariableAccessor(deserialize_json=True),
+ "value": VariableAccessor(deserialize_json=False),
+ },
"conn": ConnectionAccessor(),
}
if self._ti_context_from_server:
diff --git a/task_sdk/tests/api/test_client.py
b/task_sdk/tests/api/test_client.py
index ff686537a96..8315a121fc4 100644
--- a/task_sdk/tests/api/test_client.py
+++ b/task_sdk/tests/api/test_client.py
@@ -362,16 +362,32 @@ class TestVariableOperations:
client = make_client(transport=httpx.MockTransport(handle_request))
- with pytest.raises(ServerResponseError) as err:
- client.variables.get(key="non_existent_var")
-
- assert err.value.response.status_code == 404
- assert err.value.detail == {
- "detail": {
- "message": "Variable with key 'non_existent_var' not found",
- "reason": "not_found",
- }
- }
+ resp = client.variables.get(key="non_existent_var")
+
+ assert isinstance(resp, ErrorResponse)
+ assert resp.error == ErrorType.VARIABLE_NOT_FOUND
+ assert resp.detail == {"key": "non_existent_var"}
+
+ @mock.patch("time.sleep", return_value=None)
+ def test_variable_get_500_error(self, mock_sleep):
+ # Simulate a response from the server returning a 500 error
+ def handle_request(request: httpx.Request) -> httpx.Response:
+ if request.url.path == "/variables/test_key":
+ return httpx.Response(
+ status_code=500,
+ headers=[("content-Type", "application/json")],
+ json={
+ "reason": "internal_server_error",
+ "message": "Internal Server Error",
+ },
+ )
+ return httpx.Response(status_code=400, json={"detail": "Bad
Request"})
+
+ client = make_client(transport=httpx.MockTransport(handle_request))
+ with pytest.raises(ServerResponseError):
+ client.variables.get(
+ key="test_key",
+ )
def test_variable_set_success(self):
# Simulate a successful response from the server when putting a
variable
diff --git a/task_sdk/tests/execution_time/test_context.py
b/task_sdk/tests/execution_time/test_context.py
index 34502a1a917..4aff0da0775 100644
--- a/task_sdk/tests/execution_time/test_context.py
+++ b/task_sdk/tests/execution_time/test_context.py
@@ -18,9 +18,15 @@
from __future__ import annotations
from airflow.sdk.definitions.connection import Connection
+from airflow.sdk.definitions.variable import Variable
from airflow.sdk.exceptions import ErrorType
-from airflow.sdk.execution_time.comms import ConnectionResult, ErrorResponse
-from airflow.sdk.execution_time.context import ConnectionAccessor,
_convert_connection_result_conn
+from airflow.sdk.execution_time.comms import ConnectionResult, ErrorResponse,
VariableResult
+from airflow.sdk.execution_time.context import (
+ ConnectionAccessor,
+ VariableAccessor,
+ _convert_connection_result_conn,
+ _convert_variable_result_to_variable,
+)
def test_convert_connection_result_conn():
@@ -48,6 +54,31 @@ def test_convert_connection_result_conn():
)
+def test_convert_variable_result_to_variable():
+ """Test that the VariableResult is converted to a Variable object."""
+ var = VariableResult(
+ key="test_key",
+ value="test_value",
+ )
+ var = _convert_variable_result_to_variable(var, deserialize_json=False)
+ assert var == Variable(
+ key="test_key",
+ value="test_value",
+ )
+
+
+def test_convert_variable_result_to_variable_with_deserialize_json():
+ """Test that the VariableResult is converted to a Variable object with
deserialize_json set to True."""
+ var = VariableResult(
+ key="test_key",
+ value='{\r\n "key1": "value1",\r\n "key2": "value2",\r\n "enabled":
true,\r\n "threshold": 42\r\n}',
+ )
+ var = _convert_variable_result_to_variable(var, deserialize_json=True)
+ assert var == Variable(
+ key="test_key", value={"key1": "value1", "key2": "value2", "enabled":
True, "threshold": 42}
+ )
+
+
class TestConnectionAccessor:
def test_getattr_connection(self, mock_supervisor_comms):
"""
@@ -90,3 +121,44 @@ class TestConnectionAccessor:
conn = accessor.get("nonexistent_conn", default_conn=default_conn)
assert conn == default_conn
+
+
+class TestVariableAccessor:
+ def test_getattr_variable(self, mock_supervisor_comms):
+ """
+ Test that the variable is fetched when accessed via __getattr__.
+ """
+ accessor = VariableAccessor(deserialize_json=False)
+
+ # Variable from the supervisor / API Server
+ var_result = VariableResult(key="test_key", value="test_value")
+
+ mock_supervisor_comms.get_message.return_value = var_result
+
+ # Fetch the variable; triggers __getattr__
+ var = accessor.test_key
+
+ expected_var = Variable(key="test_key", value="test_value")
+ assert var == expected_var
+
+ def test_get_method_valid_variable(self, mock_supervisor_comms):
+ """Test that the get method returns the requested variable using
`var.get`."""
+ accessor = VariableAccessor(deserialize_json=False)
+ var_result = VariableResult(key="test_key", value="test_value")
+
+ mock_supervisor_comms.get_message.return_value = var_result
+
+ var = accessor.get("test_key")
+ assert var == Variable(key="test_key", value="test_value")
+
+ def test_get_method_with_default(self, mock_supervisor_comms):
+ """Test that the get method returns the default variable when the
requested variable is not found."""
+
+ accessor = VariableAccessor(deserialize_json=False)
+ default_var = {"default_key": "default_value"}
+ error_response = ErrorResponse(error=ErrorType.VARIABLE_NOT_FOUND,
detail={"test_key": "test_value"})
+
+ mock_supervisor_comms.get_message.return_value = error_response
+
+ var = accessor.get("nonexistent_var_key", default_var=default_var)
+ assert var == default_var
diff --git a/task_sdk/tests/execution_time/test_task_runner.py
b/task_sdk/tests/execution_time/test_task_runner.py
index ebf03d33233..8749bb2be10 100644
--- a/task_sdk/tests/execution_time/test_task_runner.py
+++ b/task_sdk/tests/execution_time/test_task_runner.py
@@ -35,15 +35,18 @@ from airflow.exceptions import (
)
from airflow.sdk import DAG, BaseOperator, Connection
from airflow.sdk.api.datamodels._generated import TaskInstance, TerminalTIState
+from airflow.sdk.definitions.variable import Variable
from airflow.sdk.execution_time.comms import (
ConnectionResult,
DeferTask,
GetConnection,
+ GetVariable,
SetRenderedFields,
StartupDetails,
TaskState,
+ VariableResult,
)
-from airflow.sdk.execution_time.context import ConnectionAccessor
+from airflow.sdk.execution_time.context import ConnectionAccessor,
VariableAccessor
from airflow.sdk.execution_time.task_runner import (
CommsDecoder,
RuntimeTaskInstance,
@@ -590,6 +593,10 @@ class TestRuntimeTaskInstance:
# Verify the context keys and values
assert context == {
+ "var": {
+ "json": VariableAccessor(deserialize_json=True),
+ "value": VariableAccessor(deserialize_json=False),
+ },
"conn": ConnectionAccessor(),
"dag": runtime_ti.task.dag,
"inlets": task.inlets,
@@ -623,6 +630,10 @@ class TestRuntimeTaskInstance:
context = runtime_ti.get_template_context()
assert context == {
+ "var": {
+ "json": VariableAccessor(deserialize_json=True),
+ "value": VariableAccessor(deserialize_json=False),
+ },
"conn": ConnectionAccessor(),
"dag": runtime_ti.task.dag,
"inlets": task.inlets,
@@ -695,6 +706,51 @@ class TestRuntimeTaskInstance:
extra='{"extra_key": "extra_value"}',
)
+ @pytest.mark.parametrize(
+ ["accessor_type", "var_value", "expected_value"],
+ [
+ pytest.param("value", "test_value", "test_value"),
+ pytest.param(
+ "json",
+ '{\r\n "key1": "value1",\r\n "key2": "value2",\r\n
"enabled": true,\r\n "threshold": 42\r\n}',
+ {"key1": "value1", "key2": "value2", "enabled": True,
"threshold": 42},
+ ),
+ ],
+ )
+ def test_get_variable_from_context(
+ self, mocked_parse, make_ti_context, mock_supervisor_comms,
accessor_type, var_value, expected_value
+ ):
+ """Test that the variable is fetched from the API server via the
Supervisor lazily when accessed"""
+
+ task = BaseOperator(task_id="hello")
+
+ ti_id = uuid7()
+ ti = TaskInstance(
+ id=ti_id, task_id=task.task_id, dag_id="basic_task",
run_id="test_run", try_number=1
+ )
+ var = VariableResult(key="test_key", value=var_value)
+
+ what = StartupDetails(ti=ti, file="", requests_fd=0,
ti_context=make_ti_context())
+ runtime_ti = mocked_parse(what, ti.dag_id, task)
+ mock_supervisor_comms.get_message.return_value = var
+
+ context = runtime_ti.get_template_context()
+
+ # Assert that the variable is not fetched from the API server yet!
+ # The variable should be only fetched connection is accessed
+ mock_supervisor_comms.send_request.assert_not_called()
+ mock_supervisor_comms.get_message.assert_not_called()
+
+ # Access the variable from the context
+ var_from_context = context["var"][accessor_type].test_key
+
+ mock_supervisor_comms.send_request.assert_called_once_with(
+ log=mock.ANY, msg=GetVariable(key="test_key")
+ )
+ mock_supervisor_comms.get_message.assert_called_once_with()
+
+ assert var_from_context == Variable(key="test_key",
value=expected_value)
+
class TestXComAfterTaskExecution:
@pytest.mark.parametrize(