kaxil commented on code in PR #66073: URL: https://github.com/apache/airflow/pull/66073#discussion_r3175794465
########## airflow-core/src/airflow/api_fastapi/execution_api/routes/task_state.py: ########## @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from typing import Annotated +from uuid import UUID + +from cadwyn import VersionedAPIRouter +from fastapi import HTTPException, Path, Query, Security, status +from sqlalchemy.orm import Session + +from airflow._shared.state import TaskScope +from airflow.api_fastapi.common.db.common import SessionDep +from airflow.api_fastapi.execution_api.datamodels.task_state import ( + TaskStatePutBody, + TaskStateResponse, +) +from airflow.api_fastapi.execution_api.security import require_auth +from airflow.models.taskinstance import TaskInstance as TI +from airflow.state import get_state_backend + +router = VersionedAPIRouter( Review Comment: This router and `asset_state.py:52` don't set `route_class=ExecutionAPIRoute`, but the other versioned routers in this module do (`hitl.py:36-37`, `task_instances.py:89-90`). The default at `security.py:170` makes `ti:self` work today, but as soon as someone wants to add a `Security(require_auth, scopes=["token:workload"])` route here, the workload-token branch won't be reachable because `allowed_token_types` falls back to `{"execution"}`. Worth aligning with the rest of the execution API now while these routers are new. ########## airflow-core/src/airflow/api_fastapi/execution_api/routes/task_state.py: ########## @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from typing import Annotated +from uuid import UUID + +from cadwyn import VersionedAPIRouter +from fastapi import HTTPException, Path, Query, Security, status +from sqlalchemy.orm import Session + +from airflow._shared.state import TaskScope +from airflow.api_fastapi.common.db.common import SessionDep +from airflow.api_fastapi.execution_api.datamodels.task_state import ( + TaskStatePutBody, + TaskStateResponse, +) +from airflow.api_fastapi.execution_api.security import require_auth +from airflow.models.taskinstance import TaskInstance as TI +from airflow.state import get_state_backend + +router = VersionedAPIRouter( + responses={ + status.HTTP_401_UNAUTHORIZED: {"description": "Unauthorized"}, + status.HTTP_403_FORBIDDEN: {"description": "Access denied"}, + status.HTTP_404_NOT_FOUND: {"description": "Not found"}, + }, + dependencies=[Security(require_auth, scopes=["ti:self"])], +) + + +def _get_task_scope_for_ti(task_instance_id: UUID, session: Session) -> TaskScope: + ti = session.get(TI, task_instance_id) + if ti is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={ + "reason": "not_found", + "message": f"Task instance {task_instance_id} not found", + }, + ) + return TaskScope(dag_id=ti.dag_id, run_id=ti.run_id, task_id=ti.task_id, map_index=ti.map_index) + + [email protected]("/{task_instance_id}/{key}") +def get_task_state( + task_instance_id: UUID, + key: Annotated[str, Path(min_length=1)], + session: SessionDep, +) -> TaskStateResponse: + """Get value for a task state.""" + scope = _get_task_scope_for_ti(task_instance_id, session) + value = get_state_backend().get(scope, key, session=session) + if value is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={ + "reason": "not_found", + "message": f"Task state key {key!r} not found", + }, + ) + return TaskStateResponse(value=value) + + [email protected]("/{task_instance_id}/{key}", status_code=status.HTTP_204_NO_CONTENT) +def set_task_state( + task_instance_id: UUID, + key: Annotated[str, Path(min_length=1)], + body: TaskStatePutBody, + session: SessionDep, +) -> None: + """Set a task state key, creating or updating the row.""" + scope = _get_task_scope_for_ti(task_instance_id, session) + get_state_backend().set(scope, key, body.value, session=session) + + [email protected]("/{task_instance_id}/{key}", status_code=status.HTTP_204_NO_CONTENT) +def delete_task_state( + task_instance_id: UUID, + key: Annotated[str, Path(min_length=1)], + session: SessionDep, +) -> None: + """Delete a single task state key.""" + scope = _get_task_scope_for_ti(task_instance_id, session) + get_state_backend().delete(scope, key, session=session) + + [email protected]("/{task_instance_id}", status_code=status.HTTP_204_NO_CONTENT) +def clear_task_state( + task_instance_id: UUID, + session: SessionDep, + all_map_indices: Annotated[bool, Query()] = False, +) -> None: + """ + Delete all task state keys for this task instance. + + By default, only keys for the requesting TI's exact ``map_index`` are + cleared — same isolation as DELETE endpoint above. + + Pass ``?all_map_indices=true`` to wipe state for every mapped sibling of + the task in the same DAG run. This is intentionally fleet-wide: the + ``ti:self`` JWT authentication scope authenticates that the caller is + a legitimate member of the mapped task group, and grants it authority + to reset shared task state on behalf of the whole group. + The SDK only forwards this flag when the user calls ``task_state.clear(all_map_indices=True)`` + explicitly, so the expanded scope is always an explicit opt-in by the task author. + + For non-mapped tasks (``map_index=-1``), there is only ever one index, so + ``?all_map_indices=true`` is functionally identical to the default and is + accepted without error. + """ + ti = session.get(TI, task_instance_id) + if ti is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={ + "reason": "not_found", + "message": f"Task instance {task_instance_id} not found", + }, + ) + scope = TaskScope(dag_id=ti.dag_id, run_id=ti.run_id, task_id=ti.task_id, map_index=ti.map_index) Review Comment: `clear_task_state` re-implements the TI lookup, 404 handling, and `TaskScope` build that already lives in `_get_task_scope_for_ti` (line 46). The two will drift: if anyone tweaks the 404 message, adds a permission check, or changes how the scope is derived in the helper, this branch silently misses it. Replacing lines 126-135 with `scope = _get_task_scope_for_ti(task_instance_id, session)` keeps the four route handlers in lockstep. ########## airflow-core/src/airflow/api_fastapi/execution_api/routes/asset_state.py: ########## @@ -0,0 +1,121 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Execution API routes for asset state. + +Asset state is keyed by asset *name* (not integer id) in the URL — asset names +are unique, and callers (task SDK accessors) have the name from their Asset +object without needing a DB lookup. The route resolves name → asset_id +internally for the state backend scope. + +Per-task asset registration checks are intentionally not implemented here +(deferred to AIP-93 — see TODO comment below). +""" + +from __future__ import annotations + +from typing import Annotated + +from cadwyn import VersionedAPIRouter +from fastapi import HTTPException, Path, status +from sqlalchemy import select + +from airflow._shared.state import AssetScope +from airflow.api_fastapi.common.db.common import SessionDep +from airflow.api_fastapi.execution_api.datamodels.asset_state import ( + AssetStatePutBody, + AssetStateResponse, +) +from airflow.models.asset import AssetModel +from airflow.state import get_state_backend + +# TODO(AIP-103): enforce that the requesting task is registered with the asset +# (via task_inlet_asset_reference or task_outlet_asset_reference) before +# allowing reads/writes. Currently any task with a valid execution token can +# access any asset's state — the same gap exists in /assets and /asset-events. +# Proper fix is a unified asset-registration check across all asset routes, +# not just here. +router = VersionedAPIRouter( + responses={ + status.HTTP_401_UNAUTHORIZED: {"description": "Unauthorized"}, + status.HTTP_404_NOT_FOUND: {"description": "Not found"}, + }, +) + + +def _resolve_asset_id(name: str, session: SessionDep) -> int: + """Resolve asset name → integer asset_id, 404 if not found.""" + asset_id = session.scalar(select(AssetModel.id).where(AssetModel.name == name, AssetModel.active.has())) + if asset_id is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"reason": "not_found", "message": f"Asset {name!r} not found"}, + ) + return asset_id + + [email protected]("/{name}/{key}") Review Comment: Asset names accept `/` (only `self` and `context` are prohibited; see `_validate_asset_name` in `task-sdk/.../asset/__init__.py:206-210`). A legitimate asset named `team/sales` is unreachable at `/state/asset/team/sales/{key}` because FastAPI matches `name=team`, `key=sales`, and the trailing `/key` falls off the route entirely. Same for keys with `/`. Two existing options in this codebase: `assets.py` uses `/by-name?name=...` query parameters precisely to side-step this, or you can use a path converter that consumes the rest (`name: Annotated[str, Path(min_length=1, regex=...)]`). Worth doing here too, otherwise users with grouped/namespaced assets silently can't use this API at all. ########## airflow-core/src/airflow/api_fastapi/execution_api/routes/asset_state.py: ########## @@ -0,0 +1,121 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Execution API routes for asset state. + +Asset state is keyed by asset *name* (not integer id) in the URL — asset names +are unique, and callers (task SDK accessors) have the name from their Asset +object without needing a DB lookup. The route resolves name → asset_id +internally for the state backend scope. + +Per-task asset registration checks are intentionally not implemented here +(deferred to AIP-93 — see TODO comment below). +""" + +from __future__ import annotations + +from typing import Annotated + +from cadwyn import VersionedAPIRouter +from fastapi import HTTPException, Path, status +from sqlalchemy import select + +from airflow._shared.state import AssetScope +from airflow.api_fastapi.common.db.common import SessionDep +from airflow.api_fastapi.execution_api.datamodels.asset_state import ( + AssetStatePutBody, + AssetStateResponse, +) +from airflow.models.asset import AssetModel +from airflow.state import get_state_backend + +# TODO(AIP-103): enforce that the requesting task is registered with the asset Review Comment: Now that the resolver filters by `AssetModel.active.has()` (good change), state rows for assets that go inactive (DAG stops referencing them) become permanently unreachable through the API. `asset_state.asset_id` only cascades on `asset` row delete (`asset_state.py:53` `ondelete="CASCADE"`), but Airflow's GC removes the `asset_active` row, not the `asset` row. Net effect: orphan state rows accumulate forever for assets that ever went inactive. Worth deciding now whether to (a) cascade-delete state rows when `AssetActive` is removed, (b) add a periodic GC, or (c) document this and offer an admin-only "clear-all-inactive" endpoint. Otherwise this'll quietly grow forever in any long-lived deployment. ########## airflow-core/src/airflow/api_fastapi/execution_api/routes/task_state.py: ########## @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from typing import Annotated +from uuid import UUID + +from cadwyn import VersionedAPIRouter +from fastapi import HTTPException, Path, Query, Security, status +from sqlalchemy.orm import Session + +from airflow._shared.state import TaskScope +from airflow.api_fastapi.common.db.common import SessionDep +from airflow.api_fastapi.execution_api.datamodels.task_state import ( + TaskStatePutBody, + TaskStateResponse, +) +from airflow.api_fastapi.execution_api.security import require_auth +from airflow.models.taskinstance import TaskInstance as TI +from airflow.state import get_state_backend + +router = VersionedAPIRouter( + responses={ + status.HTTP_401_UNAUTHORIZED: {"description": "Unauthorized"}, + status.HTTP_403_FORBIDDEN: {"description": "Access denied"}, + status.HTTP_404_NOT_FOUND: {"description": "Not found"}, + }, + dependencies=[Security(require_auth, scopes=["ti:self"])], Review Comment: Following up on the prior comment on this line. The `ti:self` enforcement here is the primary protection on these routes, but the conftest at `airflow-core/tests/unit/api_fastapi/execution_api/conftest.py:56-60` overrides `require_auth` with a mock that always builds a `TIToken` whose `id` matches `path_params["task_instance_id"]`. With that override in place the cross-TI mismatch branch in `security.py:185-191` is structurally untestable here. Could you add at least one regression test that uses the real `require_auth` (a token bound to a different TI hits a 403 against this router)? The all-map-indices fleet-wide path is now also explicitly relying on this scope by docstring, which makes coverage even more important. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
