This is an automated email from the ASF dual-hosted git repository.
dabla pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 466bfe33515 Add team-level authorization to Keycloak auth manager list
filters (#77007)
466bfe33515 is described below
commit 466bfe3351564011ba4798691bbe407cc34eb55b
Author: David Blain <[email protected]>
AuthorDate: Tue Nov 17 08:00:25 2026 +0100
Add team-level authorization to Keycloak auth manager list filters (#77007)
* Add team-level authorization to Keycloak auth manager list filters
Listing Dags asks Keycloak once per Dag, because a policy can read the id
of the
Dag. Since Airflow 3.4.0 the Dag list of the UI also authorises every asset
one by
one. Realms whose permissions are granted by role and group only get the
same
answer for every resource of a team, so deployments with many Dags and
assets wait
many seconds for an answer that one request would give.
The new opt-in setting lets such a deployment say so.
* Name the setting in the docs without a link and skip the team test before
Airflow 3.2
The docs build drops options whose version_added is newer than the released
provider, so a reference to the new option cannot resolve yet. DagDetails
has
no team_name before Airflow 3.2, which the compatibility run exercises.
* Fold the Keycloak list filters into one helper
The five filter_authorized_* methods each carried the same empty guard,
thread pool, cache lookup and, since team-level authorization, the same
branch that applies one answer to the whole list. That rule now lives
once, in _filter_authorized; each filter only says how an id becomes the
check of a single resource. Assets go through it by id as well, so the
resource the team-level request names is picked the same way for all
five types. Cache keys and the requests sent are unchanged.
* Cover the cached team-level decision for all five list filters
The parametrized test now calls each filter twice, so the cached decision
is checked for every resource type, granted and denied, instead of for
Dags only. The test of an empty list is dropped: the guard it exercised
predates team-level authorization and has its own tests.
* Key the Keycloak filter cache on the resource type
The cache of the list filters is shared by all resource types, and its
key held the user, the method, the team and the ids only. A pool and a
variable of the same name therefore shared one entry: the decision
cached for the pool was returned for the variable without asking
Keycloak. _filter_authorized now builds the key itself and starts it
with the resource type, so a filter cannot leave it out.
* Set team-level authorization with the conf_vars decorator in tests
The value is fixed for the whole test, so the decorator replaces the
with block in the two tests that enable the setting.
---
.../docs/auth-manager/manage/permissions.rst | 29 +++
providers/keycloak/provider.yaml | 12 ++
.../providers/keycloak/auth_manager/constants.py | 1 +
.../keycloak/auth_manager/keycloak_auth_manager.py | 218 ++++++++++-----------
.../providers/keycloak/get_provider_info.py | 7 +
.../auth_manager/test_keycloak_auth_manager.py | 87 ++++++++
6 files changed, 239 insertions(+), 115 deletions(-)
diff --git a/providers/keycloak/docs/auth-manager/manage/permissions.rst
b/providers/keycloak/docs/auth-manager/manage/permissions.rst
index 78f70899e64..c9a05b986c9 100644
--- a/providers/keycloak/docs/auth-manager/manage/permissions.rst
+++ b/providers/keycloak/docs/auth-manager/manage/permissions.rst
@@ -124,6 +124,35 @@ import errors for files with no registered Dag, reparsing
such files, and Dag bu
``AdminView`` resource, which the CLI grants to ``SuperAdmin`` only. The other
views are checked against
the ``View`` resource, which every team role can read.
+Listing resources when permissions are granted per team
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Whenever Airflow lists Dags, assets, connections, pools or variables, the auth
manager asks Keycloak which
+of them the user may access. By default it sends one request per resource,
with the id of the resource as a
+claim, so that a Keycloak policy can grant access to individual resources.
With many resources this makes
+pages that list them slow: the Dag list sends one request per Dag and, from
Airflow 3.4.0, one per asset.
+
+The permissions created by the CLI are granted by role and group only, and no
policy reads the id of the
+resource. Keycloak then gives the same answer for every Dag of a team. When
that holds for your realm, enable
+``team_level_authorization`` in the ``[keycloak_auth_manager]`` section:
+
+.. code-block:: ini
+
+ [keycloak_auth_manager]
+ team_level_authorization = True
+
+The auth manager then sends one request per team and resource type and applies
the answer to all resources of
+that type in the team. Assets do not belong to a team, so one request decides
for all assets. When multi-team
+is disabled, one request decides for each resource type.
+
+.. warning::
+
+ Do not enable this setting when a policy of the realm grants or denies
access to individual resources, for
+ example a JavaScript or regex policy that reads the ``resource_id`` claim.
Such a policy would then be
+ evaluated for one resource only and its answer applied to all the others.
+
+Checks of a single resource, such as opening or triggering one Dag, are not
affected by this setting.
+
More resources about permissions can be found in the official documentation of
Keycloak:
1- `Keycloak Authorization Process
<https://www.keycloak.org/docs/latest/authorization_services/index.html#the-authorization-process>`_
diff --git a/providers/keycloak/provider.yaml b/providers/keycloak/provider.yaml
index 8c169a43fe5..4d9b2657dd6 100644
--- a/providers/keycloak/provider.yaml
+++ b/providers/keycloak/provider.yaml
@@ -141,3 +141,15 @@ config:
version_added: 0.4.0
example: "3"
default: "3"
+ team_level_authorization:
+ description: |
+ Whether one authorization decision holds for every resource of a
type within a team.
+ By default the auth manager asks Keycloak once per Dag, asset,
connection, pool or variable
+ when listing them, because a Keycloak policy can read the id of the
resource. When no policy
+ of the realm does, set this to ``True``: the auth manager then asks
once per team (once in
+ total for assets, and for every type when multi-team is disabled)
and applies the answer to
+ all resources of that type. Do not enable it when a policy grants
access per resource.
+ type: boolean
+ version_added: 0.13.0
+ example: ~
+ default: "False"
diff --git
a/providers/keycloak/src/airflow/providers/keycloak/auth_manager/constants.py
b/providers/keycloak/src/airflow/providers/keycloak/auth_manager/constants.py
index 74d76861890..2b6dde15b6d 100644
---
a/providers/keycloak/src/airflow/providers/keycloak/auth_manager/constants.py
+++
b/providers/keycloak/src/airflow/providers/keycloak/auth_manager/constants.py
@@ -30,6 +30,7 @@ CONF_REQUESTS_RETRIES_KEY = "requests_retries"
# Comma-separated allow-list of federated Keycloak client ids (azp claim)
permitted
# to use the jwt-bearer grant. Empty/unset denies all callers.
CONF_JWT_FEDERATED_CLIENT_IDS_KEY = "jwt_federated_client_ids"
+CONF_TEAM_LEVEL_AUTHORIZATION_KEY = "team_level_authorization"
# Extra Cookie names
COOKIE_NAME_ACCESS_TOKEN = "_access_token"
diff --git
a/providers/keycloak/src/airflow/providers/keycloak/auth_manager/keycloak_auth_manager.py
b/providers/keycloak/src/airflow/providers/keycloak/auth_manager/keycloak_auth_manager.py
index f5bd0742262..524a2565ca8 100644
---
a/providers/keycloak/src/airflow/providers/keycloak/auth_manager/keycloak_auth_manager.py
+++
b/providers/keycloak/src/airflow/providers/keycloak/auth_manager/keycloak_auth_manager.py
@@ -22,7 +22,7 @@ import logging
import time
import warnings
from base64 import urlsafe_b64decode
-from collections.abc import Sequence
+from collections.abc import Callable, Collection, Sequence
from concurrent.futures import ThreadPoolExecutor
from typing import TYPE_CHECKING, Any, cast
from urllib.parse import urlencode, urljoin
@@ -62,6 +62,7 @@ from airflow.providers.keycloak.auth_manager.constants import
(
CONF_REQUESTS_RETRIES_KEY,
CONF_SECTION_NAME,
CONF_SERVER_URL_KEY,
+ CONF_TEAM_LEVEL_AUTHORIZATION_KEY,
)
from airflow.providers.keycloak.auth_manager.resources import KeycloakResource
from airflow.providers.keycloak.auth_manager.user import
KeycloakAuthManagerUser
@@ -567,34 +568,19 @@ class
KeycloakAuthManager(BaseAuthManager[KeycloakAuthManagerUser]):
method: ResourceMethod = "GET",
team_name: str | None = None,
) -> set[str]:
- cache_key = (user.get_id(), method, team_name, frozenset(dag_ids))
-
- def query_keycloak() -> set[str]:
- if not dag_ids:
- return set()
- # Cap workers at the HTTP connection pool size: each
is_authorized_dag() call
- # goes through the shared requests.Session, so extra threads would
just block
- # waiting for a free connection in urllib3's pool.
- max_workers = min(
- len(dag_ids), conf.getint(CONF_SECTION_NAME,
CONF_REQUESTS_POOL_SIZE_KEY, fallback=10)
+ def is_authorized(dag_id: str) -> bool:
+ details_kwargs: dict[str, Any] = {"id": dag_id}
+ if team_name is not None:
+ details_kwargs["team_name"] = team_name
+ return self.is_authorized_dag(
+ method=method,
+ user=user,
+ details=DagDetails(**details_kwargs),
)
- def check(dag_id: str) -> tuple[str, bool]:
- details_kwargs: dict[str, Any] = {"id": dag_id}
- if team_name is not None:
- details_kwargs["team_name"] = team_name
- return dag_id, self.is_authorized_dag(
- method=method,
- user=user,
- details=DagDetails(**details_kwargs),
- )
-
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- results = executor.map(check, dag_ids)
-
- return {dag_id for dag_id, authorized in results if authorized}
-
- return single_flight(cache_key, query_keycloak)
+ return self._filter_authorized(
+ KeycloakResource.DAG, dag_ids, is_authorized, user=user,
method=method, team_name=team_name
+ )
def batch_is_authorized_connection(
self,
@@ -696,30 +682,14 @@ class
KeycloakAuthManager(BaseAuthManager[KeycloakAuthManagerUser]):
user: KeycloakAuthManagerUser,
method: ResourceMethod = "GET",
) -> set[str]:
- candidates = [details for details in assets if details.id is not None]
- cache_key = (
- user.get_id(),
- method,
- frozenset(cast("str", details.id) for details in candidates),
- )
+ candidates = {details.id: details for details in assets if details.id
is not None}
- def query_keycloak() -> set[str]:
- if not candidates:
- return set()
- max_workers = min(
- len(candidates), conf.getint(CONF_SECTION_NAME,
CONF_REQUESTS_POOL_SIZE_KEY, fallback=10)
- )
-
- def check(details: AssetDetails) -> tuple[str, bool]:
- return cast("str", details.id), self.is_authorized_asset(
- method=method, user=user, details=details
- )
-
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- results = executor.map(check, candidates)
- return {asset_id for asset_id, authorized in results if authorized}
+ def is_authorized(asset_id: str) -> bool:
+ return self.is_authorized_asset(method=method, user=user,
details=candidates[asset_id])
- return single_flight(cache_key, query_keycloak)
+ return self._filter_authorized(
+ KeycloakResource.ASSET, candidates, is_authorized, user=user,
method=method
+ )
def filter_authorized_connections(
self,
@@ -729,30 +699,24 @@ class
KeycloakAuthManager(BaseAuthManager[KeycloakAuthManagerUser]):
method: ResourceMethod = "GET",
team_name: str | None = None,
) -> set[str]:
- cache_key = (user.get_id(), method, team_name, frozenset(conn_ids))
-
- def query_keycloak() -> set[str]:
- if not conn_ids:
- return set()
- max_workers = min(
- len(conn_ids), conf.getint(CONF_SECTION_NAME,
CONF_REQUESTS_POOL_SIZE_KEY, fallback=10)
+ def is_authorized(conn_id: str) -> bool:
+ details_kwargs: dict[str, Any] = {"conn_id": conn_id}
+ if team_name is not None:
+ details_kwargs["team_name"] = team_name
+ return self.is_authorized_connection(
+ method=method,
+ user=user,
+ details=ConnectionDetails(**details_kwargs),
)
- def check(conn_id: str) -> tuple[str, bool]:
- details_kwargs: dict[str, Any] = {"conn_id": conn_id}
- if team_name is not None:
- details_kwargs["team_name"] = team_name
- return conn_id, self.is_authorized_connection(
- method=method,
- user=user,
- details=ConnectionDetails(**details_kwargs),
- )
-
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- results = executor.map(check, conn_ids)
- return {conn_id for conn_id, authorized in results if authorized}
-
- return single_flight(cache_key, query_keycloak)
+ return self._filter_authorized(
+ KeycloakResource.CONNECTION,
+ conn_ids,
+ is_authorized,
+ user=user,
+ method=method,
+ team_name=team_name,
+ )
def filter_authorized_pools(
self,
@@ -762,30 +726,19 @@ class
KeycloakAuthManager(BaseAuthManager[KeycloakAuthManagerUser]):
method: ResourceMethod = "GET",
team_name: str | None = None,
) -> set[str]:
- cache_key = (user.get_id(), method, team_name, frozenset(pool_names))
-
- def query_keycloak() -> set[str]:
- if not pool_names:
- return set()
- max_workers = min(
- len(pool_names), conf.getint(CONF_SECTION_NAME,
CONF_REQUESTS_POOL_SIZE_KEY, fallback=10)
+ def is_authorized(pool_name: str) -> bool:
+ details_kwargs: dict[str, Any] = {"name": pool_name}
+ if team_name is not None:
+ details_kwargs["team_name"] = team_name
+ return self.is_authorized_pool(
+ method=method,
+ user=user,
+ details=PoolDetails(**details_kwargs),
)
- def check(pool_name: str) -> tuple[str, bool]:
- details_kwargs: dict[str, Any] = {"name": pool_name}
- if team_name is not None:
- details_kwargs["team_name"] = team_name
- return pool_name, self.is_authorized_pool(
- method=method,
- user=user,
- details=PoolDetails(**details_kwargs),
- )
-
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- results = executor.map(check, pool_names)
- return {pool_name for pool_name, authorized in results if
authorized}
-
- return single_flight(cache_key, query_keycloak)
+ return self._filter_authorized(
+ KeycloakResource.POOL, pool_names, is_authorized, user=user,
method=method, team_name=team_name
+ )
def filter_authorized_variables(
self,
@@ -795,30 +748,24 @@ class
KeycloakAuthManager(BaseAuthManager[KeycloakAuthManagerUser]):
method: ResourceMethod = "GET",
team_name: str | None = None,
) -> set[str]:
- cache_key = (user.get_id(), method, team_name,
frozenset(variable_keys))
-
- def query_keycloak() -> set[str]:
- if not variable_keys:
- return set()
- max_workers = min(
- len(variable_keys), conf.getint(CONF_SECTION_NAME,
CONF_REQUESTS_POOL_SIZE_KEY, fallback=10)
+ def is_authorized(variable_key: str) -> bool:
+ details_kwargs: dict[str, Any] = {"key": variable_key}
+ if team_name is not None:
+ details_kwargs["team_name"] = team_name
+ return self.is_authorized_variable(
+ method=method,
+ user=user,
+ details=VariableDetails(**details_kwargs),
)
- def check(variable_key: str) -> tuple[str, bool]:
- details_kwargs: dict[str, Any] = {"key": variable_key}
- if team_name is not None:
- details_kwargs["team_name"] = team_name
- return variable_key, self.is_authorized_variable(
- method=method,
- user=user,
- details=VariableDetails(**details_kwargs),
- )
-
- with ThreadPoolExecutor(max_workers=max_workers) as executor:
- results = executor.map(check, variable_keys)
- return {variable_key for variable_key, authorized in results if
authorized}
-
- return single_flight(cache_key, query_keycloak)
+ return self._filter_authorized(
+ KeycloakResource.VARIABLE,
+ variable_keys,
+ is_authorized,
+ user=user,
+ method=method,
+ team_name=team_name,
+ )
def _is_batch_authorized(
self,
@@ -872,6 +819,47 @@ class
KeycloakAuthManager(BaseAuthManager[KeycloakAuthManagerUser]):
return {r["name"][len(prefix) :] for r in resources_resp.json() if
r["name"].startswith(prefix)}
+ def _filter_authorized(
+ self,
+ resource_type: KeycloakResource,
+ ids: Collection[str],
+ is_authorized: Callable[[str], bool],
+ *,
+ user: KeycloakAuthManagerUser,
+ method: ResourceMethod,
+ team_name: str | None = None,
+ ) -> set[str]:
+ """
+ Return the ids Keycloak authorizes, asking once per id or, with
team-level authorization, once.
+
+ The auth manager sends the id of the resource as a claim, so that a
policy can decide per
+ resource. When no policy of the realm reads that claim
(``team_level_authorization``), Keycloak
+ answers the same for every Dag, connection, pool and variable of a
team, and for every asset:
+ the answer for one id of the list then holds for all of them.
+ """
+
+ def query_keycloak() -> set[str]:
+ if not ids:
+ return set()
+ if conf.getboolean(CONF_SECTION_NAME,
CONF_TEAM_LEVEL_AUTHORIZATION_KEY, fallback=False):
+ return set(ids) if is_authorized(min(ids)) else set()
+ # Cap workers at the HTTP connection pool size: each is_authorized
call goes through
+ # the shared requests.Session, so extra threads would just block
waiting for a free
+ # connection in urllib3's pool.
+ max_workers = min(
+ len(ids), conf.getint(CONF_SECTION_NAME,
CONF_REQUESTS_POOL_SIZE_KEY, fallback=10)
+ )
+ candidates = list(ids)
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ results = executor.map(is_authorized, candidates)
+ return {
+ candidate for candidate, authorized in zip(candidates,
results, strict=True) if authorized
+ }
+
+ # The cache is shared by all resource types, and a pool and a variable
can have the same name.
+ cache_key = (resource_type, user.get_id(), method, team_name,
frozenset(ids))
+ return single_flight(cache_key, query_keycloak)
+
@staticmethod
def _get_token_url(server_url, realm):
# Normalize server_url to avoid double slashes (required for Keycloak
26.4+ strict path validation).
diff --git
a/providers/keycloak/src/airflow/providers/keycloak/get_provider_info.py
b/providers/keycloak/src/airflow/providers/keycloak/get_provider_info.py
index f4b0ae3a890..0deaf841bb4 100644
--- a/providers/keycloak/src/airflow/providers/keycloak/get_provider_info.py
+++ b/providers/keycloak/src/airflow/providers/keycloak/get_provider_info.py
@@ -106,6 +106,13 @@ def get_provider_info():
"example": "3",
"default": "3",
},
+ "team_level_authorization": {
+ "description": "Whether one authorization decision
holds for every resource of a type within a team.\nBy default the auth manager
asks Keycloak once per Dag, asset, connection, pool or variable\nwhen listing
them, because a Keycloak policy can read the id of the resource. When no
policy\nof the realm does, set this to ``True``: the auth manager then asks
once per team (once in\ntotal for assets, and for every type when multi-team is
disabled) and applies the answe [...]
+ "type": "boolean",
+ "version_added": "0.13.0",
+ "example": None,
+ "default": "False",
+ },
},
}
},
diff --git
a/providers/keycloak/tests/unit/keycloak/auth_manager/test_keycloak_auth_manager.py
b/providers/keycloak/tests/unit/keycloak/auth_manager/test_keycloak_auth_manager.py
index c994c6071c5..920bcad894a 100644
---
a/providers/keycloak/tests/unit/keycloak/auth_manager/test_keycloak_auth_manager.py
+++
b/providers/keycloak/tests/unit/keycloak/auth_manager/test_keycloak_auth_manager.py
@@ -73,6 +73,7 @@ from airflow.providers.keycloak.auth_manager.constants import
(
CONF_REQUESTS_POOL_SIZE_KEY,
CONF_SECTION_NAME,
CONF_SERVER_URL_KEY,
+ CONF_TEAM_LEVEL_AUTHORIZATION_KEY,
)
from airflow.providers.keycloak.auth_manager.keycloak_auth_manager import (
RESOURCE_ID_ATTRIBUTE_NAME,
@@ -1634,5 +1635,91 @@ class TestKeycloakAuthManager:
mock_executor.assert_called_once_with(max_workers=expected_max_workers)
+ @pytest.mark.parametrize(
+ ("filter_name", "check_name", "resources", "expected"),
+ [
+ pytest.param(
+ "filter_authorized_dag_ids",
+ "is_authorized_dag",
+ {"dag_ids": {"dag_0", "dag_1", "dag_2"}},
+ {"dag_0", "dag_1", "dag_2"},
+ id="dags",
+ ),
+ pytest.param(
+ "filter_authorized_assets",
+ "is_authorized_asset",
+ {"assets": [AssetDetails(id="1"), AssetDetails(id="2"),
AssetDetails(id=None)]},
+ {"1", "2"},
+ id="assets",
+ ),
+ pytest.param(
+ "filter_authorized_connections",
+ "is_authorized_connection",
+ {"conn_ids": {"conn_0", "conn_1", "conn_2"}},
+ {"conn_0", "conn_1", "conn_2"},
+ id="connections",
+ ),
+ pytest.param(
+ "filter_authorized_pools",
+ "is_authorized_pool",
+ {"pool_names": {"pool_0", "pool_1", "pool_2"}},
+ {"pool_0", "pool_1", "pool_2"},
+ id="pools",
+ ),
+ pytest.param(
+ "filter_authorized_variables",
+ "is_authorized_variable",
+ {"variable_keys": {"var_0", "var_1", "var_2"}},
+ {"var_0", "var_1", "var_2"},
+ id="variables",
+ ),
+ ],
+ )
+ @pytest.mark.parametrize("authorized", [True, False])
+ @conf_vars({(CONF_SECTION_NAME, CONF_TEAM_LEVEL_AUTHORIZATION_KEY):
"True"})
+ def test_filter_with_team_level_authorization_asks_keycloak_once(
+ self, auth_manager, user, filter_name, check_name, resources,
expected, authorized
+ ):
+ """One decision of Keycloak applies to every resource of the list, and
is cached."""
+ with patch.object(
+ KeycloakAuthManager, check_name, autospec=True,
return_value=authorized
+ ) as mock_is_authorized:
+ first = getattr(auth_manager, filter_name)(**resources, user=user,
method="GET")
+ second = getattr(auth_manager, filter_name)(**resources,
user=user, method="GET")
+
+ assert first == second == (expected if authorized else set())
+ mock_is_authorized.assert_called_once()
+
+ @pytest.mark.skipif(not AIRFLOW_V_3_2_PLUS, reason="team_name not
supported before Airflow 3.2.0")
+ @conf_vars({(CONF_SECTION_NAME, CONF_TEAM_LEVEL_AUTHORIZATION_KEY):
"True"})
+ @patch.object(KeycloakAuthManager, "is_authorized_dag", autospec=True,
return_value=True)
+ def
test_filter_with_team_level_authorization_asks_for_a_listed_dag_of_the_team(
+ self, mock_is_authorized, auth_manager_multi_team, user
+ ):
+ """The request names a Dag of the list and its team, as a check of
that single Dag would."""
+ auth_manager_multi_team.filter_authorized_dag_ids(
+ dag_ids={"dag_1", "dag_0"}, user=user, method="PUT",
team_name="team_a"
+ )
+
+ mock_is_authorized.assert_called_once_with(
+ auth_manager_multi_team,
+ method="PUT",
+ user=user,
+ details=DagDetails(id="dag_0", team_name="team_a"),
+ )
+
+ @patch.object(KeycloakAuthManager, "is_authorized_variable",
autospec=True, return_value=False)
+ @patch.object(KeycloakAuthManager, "is_authorized_pool", autospec=True,
return_value=True)
+ def test_filter_authorized_caches_the_decision_per_resource_type(
+ self, mock_is_authorized_pool, mock_is_authorized_variable,
auth_manager, user
+ ):
+ """A pool and a variable of the same name do not share a cached
decision."""
+ pools =
auth_manager.filter_authorized_pools(pool_names={"default_pool"}, user=user)
+ variables =
auth_manager.filter_authorized_variables(variable_keys={"default_pool"},
user=user)
+
+ assert pools == {"default_pool"}
+ assert variables == set()
+ mock_is_authorized_variable.assert_called_once()
+
def test_get_fastapi_middleware(self, auth_manager):
assert auth_manager.get_fastapi_middlewares() ==
[(KeycloakJWTMiddleware, {})]