This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 2740c33fcc2 Allow secrets backend kwargs to be set via per-key env
vars (#63312)
2740c33fcc2 is described below
commit 2740c33fcc2b820e2485915d45e7d2e7552bbe6c
Author: Shubham Gondane <[email protected]>
AuthorDate: Wed Mar 11 17:28:55 2026 -0700
Allow secrets backend kwargs to be set via per-key env vars (#63312)
* Allow secrets backend kwargs to be set via per-key env vars
Adds support for AIRFLOW__SECRETS__BACKEND_KWARG__<KEY> environment
variables
as an alternative to the single AIRFLOW__SECRETS__BACKEND_KWARGS JSON blob.
* Rename newsfragment to match PR number
* Mask per-key backend kwarg env vars in logs at startup
* ci: retrigger CI
---
.../security/secrets/secrets-backend/index.rst | 23 +++++++
airflow-core/newsfragments/63312.feature.rst | 1 +
.../src/airflow/config_templates/config.yml | 8 +++
airflow-core/src/airflow/configuration.py | 12 ++++
airflow-core/tests/unit/always/test_secrets.py | 71 ++++++++++++++++++++++
airflow-core/tests/unit/core/test_configuration.py | 56 +++++++++++++++++
.../src/airflow_shared/configuration/parser.py | 32 ++++++++++
7 files changed, 203 insertions(+)
diff --git a/airflow-core/docs/security/secrets/secrets-backend/index.rst
b/airflow-core/docs/security/secrets/secrets-backend/index.rst
index a4ae1547a10..029dadcfb87 100644
--- a/airflow-core/docs/security/secrets/secrets-backend/index.rst
+++ b/airflow-core/docs/security/secrets/secrets-backend/index.rst
@@ -78,6 +78,29 @@ the example below.
$ airflow config get-value secrets backend
airflow.providers.google.cloud.secrets.secret_manager.CloudSecretManagerBackend
+Setting individual backend kwargs
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Instead of encoding all kwargs as a JSON blob, you can set each one as a
separate environment
+variable using the ``AIRFLOW__SECRETS__BACKEND_KWARG__<KEY>`` prefix:
+
+.. code-block:: bash
+
+ # These two are equivalent:
+ export AIRFLOW__SECRETS__BACKEND_KWARGS='{"role_id": "abc", "secret_id":
"xyz"}'
+
+ # or individually (useful for K8s Secrets):
+ export AIRFLOW__SECRETS__BACKEND_KWARG__ROLE_ID=abc
+ export AIRFLOW__SECRETS__BACKEND_KWARG__SECRET_ID=xyz
+
+Per-key variables override the same key from ``BACKEND_KWARGS``. Values are
raw strings
+(not JSON-parsed). For workers, use the
+``AIRFLOW__WORKERS__SECRETS_BACKEND_KWARG__<KEY>`` prefix.
+
+.. note::
+ These environment variables are masked in logs at startup, the same way
+ ``BACKEND_KWARGS`` is masked.
+
Worker Specific Configuration
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/airflow-core/newsfragments/63312.feature.rst
b/airflow-core/newsfragments/63312.feature.rst
new file mode 100644
index 00000000000..1af30e04ded
--- /dev/null
+++ b/airflow-core/newsfragments/63312.feature.rst
@@ -0,0 +1 @@
+Allow individual secrets backend kwargs to be set via
``AIRFLOW__SECRETS__BACKEND_KWARG__<KEY>`` environment variables
diff --git a/airflow-core/src/airflow/config_templates/config.yml
b/airflow-core/src/airflow/config_templates/config.yml
index d398f54ddfd..f931fedc473 100644
--- a/airflow-core/src/airflow/config_templates/config.yml
+++ b/airflow-core/src/airflow/config_templates/config.yml
@@ -1460,6 +1460,10 @@ secrets:
Example for AWS Systems Manager ParameterStore:
``{"connections_prefix": "/airflow/connections", "profile_name":
"default"}``
+
+ You can also set individual kwargs via
``AIRFLOW__SECRETS__BACKEND_KWARG__<KEY>=value``
+ environment variables. Per-key variables override the same key in this
JSON setting.
+ Values are raw strings (not JSON-parsed).
version_added: 1.10.10
type: string
sensitive: true
@@ -1820,6 +1824,10 @@ workers:
Example for AWS Systems Manager ParameterStore:
``{"connections_prefix": "/airflow/connections", "profile_name":
"default"}``
+
+ You can also set individual kwargs via
``AIRFLOW__WORKERS__SECRETS_BACKEND_KWARG__<KEY>=value``
+ environment variables. Per-key variables override the same key in this
JSON setting.
+ Values are raw strings (not JSON-parsed).
version_added: 3.0.0
type: string
sensitive: true
diff --git a/airflow-core/src/airflow/configuration.py
b/airflow-core/src/airflow/configuration.py
index 556821d71fb..69d936be01d 100644
--- a/airflow-core/src/airflow/configuration.py
+++ b/airflow-core/src/airflow/configuration.py
@@ -491,6 +491,7 @@ class AirflowConfigParser(_SharedAirflowConfigParser):
)
def mask_secrets(self):
+ from airflow._shared.configuration.parser import
_build_kwarg_env_prefix, _collect_kwarg_env_vars
from airflow._shared.secrets_masker import mask_secret as
mask_secret_core
from airflow.sdk.log import mask_secret as mask_secret_sdk
@@ -508,6 +509,17 @@ class AirflowConfigParser(_SharedAirflowConfigParser):
mask_secret_core(value)
mask_secret_sdk(value)
+ # Mask per-key backend kwarg env vars
(AIRFLOW__SECRETS__BACKEND_KWARG__* etc.).
+ # These are not in sensitive_config_values but may contain sensitive
values.
+ for _section, _kwargs_key in [
+ ("secrets", "backend_kwargs"),
+ ("workers", "secrets_backend_kwargs"),
+ ]:
+ _prefix = _build_kwarg_env_prefix(_section, _kwargs_key)
+ for _value in _collect_kwarg_env_vars(_prefix).values():
+ mask_secret_core(_value)
+ mask_secret_sdk(_value)
+
def load_test_config(self):
"""
Use test configuration rather than the configuration coming from
airflow defaults.
diff --git a/airflow-core/tests/unit/always/test_secrets.py
b/airflow-core/tests/unit/always/test_secrets.py
index dfa39352b92..d09f04df7df 100644
--- a/airflow-core/tests/unit/always/test_secrets.py
+++ b/airflow-core/tests/unit/always/test_secrets.py
@@ -225,3 +225,74 @@ class TestVariableFromSecrets:
)
def test_variable_env_var_do_not_access_team_specific(self):
assert Variable.get_variable_from_secrets(key="_team___myvar") is None
+
+
+@skip_if_force_lowest_dependencies_marker
+class TestSecretBackendKwargEnvVars:
+ """Test per-key env var overrides for secrets backend kwargs."""
+
+ def setup_method(self) -> None:
+ SecretCache.reset()
+
+ @conf_vars(
+ {
+ (
+ "secrets",
+ "backend",
+ ):
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
+ }
+ )
+ @mock.patch.dict(
+ "os.environ",
+ {"AIRFLOW__SECRETS__BACKEND_KWARG__CONNECTIONS_PREFIX":
"/airflow/connections"},
+ )
+ def test_backend_kwarg_env_vars_basic(self):
+ """Per-key env var is picked up when no JSON blob is set."""
+ backends = initialize_secrets_backends()
+ systems_manager = next(
+ b for b in backends if b.__class__.__name__ ==
"SystemsManagerParameterStoreBackend"
+ )
+ assert systems_manager.connections_prefix == "/airflow/connections"
+
+ @conf_vars(
+ {
+ (
+ "secrets",
+ "backend",
+ ):
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
+ ("secrets", "backend_kwargs"): '{"connections_prefix": "/old"}',
+ }
+ )
+ @mock.patch.dict(
+ "os.environ",
+ {"AIRFLOW__SECRETS__BACKEND_KWARG__CONNECTIONS_PREFIX": "/new"},
+ )
+ def test_backend_kwarg_env_vars_override_json(self):
+ """Per-key env var overrides the same key in the JSON blob."""
+ backends = initialize_secrets_backends()
+ systems_manager = next(
+ b for b in backends if b.__class__.__name__ ==
"SystemsManagerParameterStoreBackend"
+ )
+ assert systems_manager.connections_prefix == "/new"
+
+ @conf_vars(
+ {
+ (
+ "secrets",
+ "backend",
+ ):
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
+ ("secrets", "backend_kwargs"): '{"connections_prefix":
"/airflow"}',
+ }
+ )
+ @mock.patch.dict(
+ "os.environ",
+ {"AIRFLOW__SECRETS__BACKEND_KWARG__VARIABLES_PREFIX":
"/airflow/variables"},
+ )
+ def test_backend_kwarg_env_vars_merge_with_json(self):
+ """Per-key env var is merged with (not replacing) the JSON blob."""
+ backends = initialize_secrets_backends()
+ systems_manager = next(
+ b for b in backends if b.__class__.__name__ ==
"SystemsManagerParameterStoreBackend"
+ )
+ assert systems_manager.connections_prefix == "/airflow"
+ assert systems_manager.variables_prefix == "/airflow/variables"
diff --git a/airflow-core/tests/unit/core/test_configuration.py
b/airflow-core/tests/unit/core/test_configuration.py
index e591421e68b..be0f191f8c9 100644
--- a/airflow-core/tests/unit/core/test_configuration.py
+++ b/airflow-core/tests/unit/core/test_configuration.py
@@ -924,6 +924,62 @@ class TestConf:
for key, value in expected_backend_kwargs.items():
assert getattr(secrets_backend, key) == value
+ def test_build_kwarg_env_prefix(self):
+ """Test that _build_kwarg_env_prefix generates the correct prefixes."""
+ from airflow._shared.configuration.parser import
_build_kwarg_env_prefix
+
+ assert _build_kwarg_env_prefix("secrets", "backend_kwargs") ==
"AIRFLOW__SECRETS__BACKEND_KWARG__"
+ assert (
+ _build_kwarg_env_prefix("workers", "secrets_backend_kwargs")
+ == "AIRFLOW__WORKERS__SECRETS_BACKEND_KWARG__"
+ )
+
+ @mock.patch.dict(
+ "os.environ",
+ {
+ "AIRFLOW__SECRETS__BACKEND_KWARG__ROLE_ID": "abc",
+ "AIRFLOW__SECRETS__BACKEND_KWARG__": "ignored", # empty key —
must be ignored
+ "OTHER_VAR": "irrelevant",
+ },
+ )
+ def test_collect_kwarg_env_vars(self):
+ """Test that _collect_kwarg_env_vars collects matching vars and
ignores empty keys."""
+ from airflow._shared.configuration.parser import
_collect_kwarg_env_vars
+
+ result = _collect_kwarg_env_vars("AIRFLOW__SECRETS__BACKEND_KWARG__")
+ assert result == {"role_id": "abc"}
+
+ @conf_vars(
+ {
+ (
+ "workers",
+ "secrets_backend",
+ ):
"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
+ }
+ )
+ @mock.patch.dict(
+ "os.environ",
+ {"AIRFLOW__WORKERS__SECRETS_BACKEND_KWARG__CONNECTIONS_PREFIX":
"/worker/connections"},
+ )
+ def test_worker_backend_kwarg_env_vars(self):
+ """Per-key env var is picked up for the workers secrets backend."""
+ backends = ensure_secrets_loaded(DEFAULT_SECRETS_SEARCH_PATH_WORKERS)
+ secrets_backend = backends[0]
+ assert secrets_backend.__class__.__name__ ==
"SystemsManagerParameterStoreBackend"
+ assert secrets_backend.connections_prefix == "/worker/connections"
+
+ @mock.patch("airflow._shared.secrets_masker.mask_secret")
+ @mock.patch("airflow.sdk.log.mask_secret")
+ @mock.patch.dict(
+ "os.environ",
+ {"AIRFLOW__SECRETS__BACKEND_KWARG__ROLE_ID": "super-secret-role"},
+ )
+ def test_mask_secrets_includes_backend_kwarg_env_vars(self, mock_sdk_mask,
mock_core_mask):
+ """Per-key BACKEND_KWARG__* env var values are registered with the
masker at startup."""
+ conf.mask_secrets()
+ all_core_masked = [call.args[0] for call in
mock_core_mask.call_args_list]
+ assert "super-secret-role" in all_core_masked
+
def test_lookup_sequence_override_excludes_env_vars(self, monkeypatch):
"""Test that overriding lookup sequence to exclude env vars means env
vars are not respected."""
diff --git a/shared/configuration/src/airflow_shared/configuration/parser.py
b/shared/configuration/src/airflow_shared/configuration/parser.py
index 455a6282924..b06f9611a98 100644
--- a/shared/configuration/src/airflow_shared/configuration/parser.py
+++ b/shared/configuration/src/airflow_shared/configuration/parser.py
@@ -43,6 +43,34 @@ from .exceptions import AirflowConfigException
log = logging.getLogger(__name__)
+def _build_kwarg_env_prefix(section: str, kwargs_key: str) -> str:
+ """
+ Build env prefix for per-key backend kwargs.
+
+ ("secrets", "backend_kwargs") ->
"AIRFLOW__SECRETS__BACKEND_KWARG__"
+ ("workers", "secrets_backend_kwargs") ->
"AIRFLOW__WORKERS__SECRETS_BACKEND_KWARG__"
+ """
+ singular_key = kwargs_key.replace("_kwargs", "_kwarg")
+ return f"{ENV_VAR_PREFIX}{section.upper()}__{singular_key.upper()}__"
+
+
+def _collect_kwarg_env_vars(prefix: str) -> dict[str, str]:
+ """
+ Scan os.environ for per-key secrets backend kwargs.
+
+ AIRFLOW__SECRETS__BACKEND_KWARG__ROLE_ID -> {"role_id": value}
+ Values are raw strings (not JSON-parsed).
+ Empty keys (trailing __ with no suffix) are ignored.
+ """
+ overrides: dict[str, str] = {}
+ for env_var, value in os.environ.items():
+ if env_var.startswith(prefix):
+ kwarg_key = env_var[len(prefix) :].lower()
+ if kwarg_key:
+ overrides[kwarg_key] = value
+ return overrides
+
+
ConfigType = str | int | float | bool
ConfigOptionsDictType = dict[str, ConfigType]
ConfigSectionSourcesType = dict[str, str | tuple[str, str]]
@@ -409,6 +437,10 @@ class AirflowConfigParser(ConfigParser):
log.warning("Failed to parse [%s] %s into a dict, defaulting to no
kwargs.", section, kwargs_key)
backend_kwargs = {}
+ # Collect per-key overrides; they take precedence over the JSON blob.
+ env_prefix = _build_kwarg_env_prefix(section, kwargs_key)
+ backend_kwargs.update(_collect_kwarg_env_vars(env_prefix))
+
return secrets_backend_cls(**backend_kwargs)
def _get_config_value_from_secret_backend(self, config_key: str) -> str |
None:
