This is an automated email from the ASF dual-hosted git repository.
amoghdesai pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 807bfd670a0 Decouple secrets_masker project from airflow configuration
(#55259)
807bfd670a0 is described below
commit 807bfd670a00b7e8769e2db1dea4f41d6dd58f9f
Author: Amogh Desai <[email protected]>
AuthorDate: Sat Sep 6 01:56:45 2025 +0530
Decouple secrets_masker project from airflow configuration (#55259)
---
airflow-core/src/airflow/settings.py | 31 +++++
airflow-core/src/airflow/utils/cli.py | 7 +-
.../airflow/providers/openlineage/utils/utils.py | 14 +-
.../tests/unit/openlineage/plugins/test_utils.py | 17 ++-
.../src/airflow_shared/secrets_masker/__init__.py | 4 -
.../secrets_masker/secrets_masker.py | 71 ++++------
.../tests/secrets_masker/test_secrets_masker.py | 144 ++++++++++++++-------
7 files changed, 180 insertions(+), 108 deletions(-)
diff --git a/airflow-core/src/airflow/settings.py
b/airflow-core/src/airflow/settings.py
index 895acf6feab..733a539b344 100644
--- a/airflow-core/src/airflow/settings.py
+++ b/airflow-core/src/airflow/settings.py
@@ -589,6 +589,34 @@ def configure_adapters():
pass
+def _configure_secrets_masker():
+ """Configure the secrets masker with values from config."""
+ from airflow._shared.secrets_masker import (
+ DEFAULT_SENSITIVE_FIELDS,
+ _secrets_masker as secrets_masker_core,
+ )
+ from airflow.configuration import conf
+
+ min_length_to_mask = conf.getint("logging", "min_length_masked_secret",
fallback=5)
+ secret_mask_adapter = conf.getimport("logging", "secret_mask_adapter",
fallback=None)
+ sensitive_fields = DEFAULT_SENSITIVE_FIELDS.copy()
+ sensitive_variable_fields = conf.get("core", "sensitive_var_conn_names")
+ if sensitive_variable_fields:
+ sensitive_fields |= frozenset({field.strip() for field in
sensitive_variable_fields.split(",")})
+
+ core_masker = secrets_masker_core()
+ core_masker.min_length_to_mask = min_length_to_mask
+ core_masker.sensitive_variables_fields = list(sensitive_fields)
+ core_masker.secret_mask_adapter = secret_mask_adapter
+
+ from airflow.sdk._shared.secrets_masker import _secrets_masker as
sdk_secrets_masker
+
+ sdk_masker = sdk_secrets_masker()
+ sdk_masker.min_length_to_mask = min_length_to_mask
+ sdk_masker.sensitive_variables_fields = list(sensitive_fields)
+ sdk_masker.secret_mask_adapter = secret_mask_adapter
+
+
def configure_action_logging() -> None:
"""Any additional configuration (register callback) for
airflow.utils.action_loggers module."""
@@ -663,6 +691,9 @@ def initialize():
configure_orm()
configure_action_logging()
+ # Configure secrets masker before masking secrets
+ _configure_secrets_masker()
+
# mask the sensitive_config_values
conf.mask_secrets()
diff --git a/airflow-core/src/airflow/utils/cli.py
b/airflow-core/src/airflow/utils/cli.py
index bf1a83838c8..981bbe2c5cc 100644
--- a/airflow-core/src/airflow/utils/cli.py
+++ b/airflow-core/src/airflow/utils/cli.py
@@ -34,7 +34,6 @@ from pathlib import Path
from typing import TYPE_CHECKING, TypeVar, cast
from airflow import settings
-from airflow._shared.secrets_masker import should_hide_value_for_key
from airflow._shared.timezones import timezone
from airflow.dag_processing.bundles.manager import DagBundlesManager
from airflow.exceptions import AirflowException
@@ -139,6 +138,8 @@ def _build_metrics(func_name, namespace):
:param namespace: Namespace instance from argparse
:return: dict with metrics
"""
+ from airflow._shared.secrets_masker import _secrets_masker
+
sub_commands_to_check_for_sensitive_fields = {"users", "connections"}
sub_commands_to_check_for_sensitive_key = {"variables"}
sensitive_fields = {"-p", "--password", "--conn-password"}
@@ -147,7 +148,7 @@ def _build_metrics(func_name, namespace):
# For cases when value under sub_commands_to_check_for_sensitive_key have
sensitive info
if sub_command in sub_commands_to_check_for_sensitive_key:
key = full_command[-2] if len(full_command) > 3 else None
- if key and should_hide_value_for_key(key):
+ if key and _secrets_masker().should_hide_value_for_key(key):
# Mask the sensitive value since key contain sensitive keyword
full_command[-1] = "*" * 8
elif sub_command in sub_commands_to_check_for_sensitive_fields:
@@ -168,7 +169,7 @@ def _build_metrics(func_name, namespace):
json_index = full_command.index("--conn-json") + 1
conn_json = json.loads(full_command[json_index])
for k in conn_json:
- if k and should_hide_value_for_key(k):
+ if k and _secrets_masker().should_hide_value_for_key(k):
conn_json[k] = "*" * 8
full_command[json_index] = json.dumps(conn_json)
diff --git
a/providers/openlineage/src/airflow/providers/openlineage/utils/utils.py
b/providers/openlineage/src/airflow/providers/openlineage/utils/utils.py
index 6d184173aad..b9b66b5f429 100644
--- a/providers/openlineage/src/airflow/providers/openlineage/utils/utils.py
+++ b/providers/openlineage/src/airflow/providers/openlineage/utils/utils.py
@@ -76,7 +76,6 @@ if TYPE_CHECKING:
Redactable,
Redacted,
SecretsMasker,
- should_hide_value_for_key,
)
from airflow.sdk.execution_time.task_runner import RuntimeTaskInstance
from airflow.utils.state import DagRunState, TaskInstanceState
@@ -842,8 +841,19 @@ class OpenLineageRedactor(SecretsMasker):
instance = cls()
instance.patterns = other.patterns
instance.replacer = other.replacer
+ for attr in ["sensitive_variables_fields", "min_length_to_mask",
"secret_mask_adapter"]:
+ if hasattr(other, attr):
+ setattr(instance, attr, getattr(other, attr))
return instance
+ def _should_hide_value_for_key(self, name):
+ """Compatibility helper for should_hide_value_for_key across Airflow
versions."""
+ try:
+ return self.should_hide_value_for_key(name)
+ except AttributeError:
+ # fallback to module level function
+ return should_hide_value_for_key(name)
+
def _redact(self, item: Redactable, name: str | None, depth: int,
max_depth: int, **kwargs) -> Redacted: # type: ignore[override]
if AIRFLOW_V_3_0_PLUS:
# Keep compatibility for Airflow 2.x, remove when Airflow 3.0 is
the minimum version
@@ -864,7 +874,7 @@ class OpenLineageRedactor(SecretsMasker):
# Those are deprecated values in _DEPRECATION_REPLACEMENTS
# in airflow.utils.context.Context
return "<<non-redactable: Proxy>>"
- if name and should_hide_value_for_key(name):
+ if name and self._should_hide_value_for_key(name):
return self._redact_all(item, depth, max_depth)
if attrs.has(type(item)):
# TODO: FIXME when mypy gets compatible with new attrs
diff --git a/providers/openlineage/tests/unit/openlineage/plugins/test_utils.py
b/providers/openlineage/tests/unit/openlineage/plugins/test_utils.py
index 4d4bc287866..40c27f0b878 100644
--- a/providers/openlineage/tests/unit/openlineage/plugins/test_utils.py
+++ b/providers/openlineage/tests/unit/openlineage/plugins/test_utils.py
@@ -58,11 +58,17 @@ else:
from airflow.utils import timezone # type: ignore[attr-defined,no-redef]
if AIRFLOW_V_3_1_PLUS:
- from airflow.sdk._shared.secrets_masker import _secrets_masker
+ from airflow.sdk._shared.secrets_masker import DEFAULT_SENSITIVE_FIELDS,
SecretsMasker
elif AIRFLOW_V_3_0_PLUS:
- from airflow.sdk.execution_time.secrets_masker import _secrets_masker #
type: ignore[no-redef]
+ from airflow.sdk.execution_time.secrets_masker import ( # type:
ignore[no-redef]
+ DEFAULT_SENSITIVE_FIELDS,
+ SecretsMasker,
+ )
else:
- from airflow.utils.log.secrets_masker import _secrets_masker # type:
ignore[attr-defined,no-redef]
+ from airflow.utils.log.secrets_masker import ( # type:
ignore[attr-defined,no-redef]
+ DEFAULT_SENSITIVE_FIELDS,
+ SecretsMasker,
+ )
if AIRFLOW_V_3_0_PLUS:
from airflow.sdk import DAG
@@ -252,7 +258,10 @@ def test_is_name_redactable():
@pytest.mark.enable_redact
def test_redact_with_exclusions(monkeypatch):
- redactor = OpenLineageRedactor.from_masker(_secrets_masker())
+ sm = SecretsMasker()
+ if AIRFLOW_V_3_1_PLUS:
+ sm.sensitive_variables_fields = list(DEFAULT_SENSITIVE_FIELDS)
+ redactor = OpenLineageRedactor.from_masker(sm)
class NotMixin:
def __init__(self):
diff --git
a/shared/secrets_masker/src/airflow_shared/secrets_masker/__init__.py
b/shared/secrets_masker/src/airflow_shared/secrets_masker/__init__.py
index f0cac256dc4..2eb43c030db 100644
--- a/shared/secrets_masker/src/airflow_shared/secrets_masker/__init__.py
+++ b/shared/secrets_masker/src/airflow_shared/secrets_masker/__init__.py
@@ -24,8 +24,6 @@ from .secrets_masker import (
SecretsMasker,
_is_v1_env_var,
_secrets_masker,
- get_min_secret_length,
- get_sensitive_variables_fields,
mask_secret,
merge,
redact,
@@ -35,12 +33,10 @@ from .secrets_masker import (
__all__ = [
"SecretsMasker",
- "get_sensitive_variables_fields",
"mask_secret",
"redact",
"reset_secrets_masker",
"_is_v1_env_var",
- "get_min_secret_length",
"RedactedIO",
"merge",
"should_hide_value_for_key",
diff --git
a/shared/secrets_masker/src/airflow_shared/secrets_masker/secrets_masker.py
b/shared/secrets_masker/src/airflow_shared/secrets_masker/secrets_masker.py
index ad4fe77c8c9..5709f85ec5c 100644
--- a/shared/secrets_masker/src/airflow_shared/secrets_masker/secrets_masker.py
+++ b/shared/secrets_masker/src/airflow_shared/secrets_masker/secrets_masker.py
@@ -25,7 +25,7 @@ import inspect
import logging
import re
import sys
-from collections.abc import Callable, Generator, Iterable, Iterator
+from collections.abc import Generator, Iterable, Iterator
from enum import Enum
from functools import cache, cached_property
from re import Pattern
@@ -70,38 +70,13 @@ SECRETS_TO_SKIP_MASKING = {"airflow"}
"""Common terms that should be excluded from masking in both production and
tests"""
-@cache
-def get_min_secret_length() -> int:
- """Get minimum length for a secret to be considered for masking from
airflow.cfg."""
- from airflow.configuration import conf
-
- return conf.getint("logging", "min_length_masked_secret", fallback=5)
-
-
-@cache
-def get_sensitive_variables_fields():
- """Get comma-separated sensitive Variable Fields from airflow.cfg."""
- from airflow.configuration import conf
-
- sensitive_fields = DEFAULT_SENSITIVE_FIELDS.copy()
- sensitive_variable_fields = conf.get("core", "sensitive_var_conn_names")
- if sensitive_variable_fields:
- sensitive_fields |= frozenset({field.strip() for field in
sensitive_variable_fields.split(",")})
- return sensitive_fields
-
-
def should_hide_value_for_key(name):
"""
Return if the value for this given name should be hidden.
Name might be a Variable name, or key in conn.extra_dejson, for example.
"""
- from airflow import settings
-
- if isinstance(name, str) and settings.HIDE_SENSITIVE_VAR_CONN_FIELDS:
- name = name.strip().lower()
- return any(s in name for s in get_sensitive_variables_fields())
- return False
+ return _secrets_masker().should_hide_value_for_key(name)
def mask_secret(secret: JsonValue, name: str | None = None) -> None:
@@ -208,9 +183,13 @@ class SecretsMasker(logging.Filter):
_has_warned_short_secret = False
mask_secrets_in_logs = False
+ min_length_to_mask = 5
+ secret_mask_adapter = None
+
def __init__(self):
super().__init__()
self.patterns = set()
+ self.sensitive_variables_fields = []
@classmethod
def __init_subclass__(cls, **kwargs):
@@ -338,7 +317,7 @@ class SecretsMasker(logging.Filter):
if depth > max_depth:
return item
try:
- if name and should_hide_value_for_key(name):
+ if name and self.should_hide_value_for_key(name):
return self._redact_all(item, depth, max_depth,
replacement=replacement)
if isinstance(item, dict):
to_return = {
@@ -354,7 +333,7 @@ class SecretsMasker(logging.Filter):
)
if _is_v1_env_var(item):
tmp = item.to_dict()
- if should_hide_value_for_key(tmp.get("name", "")) and "value"
in tmp:
+ if self.should_hide_value_for_key(tmp.get("name", "")) and
"value" in tmp:
tmp["value"] = replacement
else:
return self._redact(
@@ -418,7 +397,7 @@ class SecretsMasker(logging.Filter):
try:
# Determine if we should treat this as sensitive
- is_sensitive = force_sensitive or (name is not None and
should_hide_value_for_key(name))
+ is_sensitive = force_sensitive or (name is not None and
self.should_hide_value_for_key(name))
if isinstance(new_item, dict) and isinstance(old_item, dict):
merged = {}
@@ -523,30 +502,32 @@ class SecretsMasker(logging.Filter):
replacement=replacement,
)
- @cached_property
- def _mask_adapter(self) -> None | Callable:
- """
- Pulls the secret mask adapter from config.
-
- This lives in a function here to be cached and only hit the config
once.
- """
- from airflow.configuration import conf
-
- return conf.getimport("logging", "secret_mask_adapter", fallback=None)
-
def _adaptations(self, secret: str) -> Generator[str, None, None]:
"""Yield the secret along with any adaptations to the secret that
should be masked."""
yield secret
- if self._mask_adapter:
+ if self.secret_mask_adapter:
# This can return an iterable of secrets to mask OR a single
secret as a string
- secret_or_secrets = self._mask_adapter(secret)
+ secret_or_secrets = self.secret_mask_adapter(secret)
if not isinstance(secret_or_secrets, str):
# if its not a string, it must be an iterable
yield from secret_or_secrets
else:
yield secret_or_secrets
+ def should_hide_value_for_key(self, name):
+ """
+ Return if the value for this given name should be hidden.
+
+ Name might be a Variable name, or key in conn.extra_dejson, for
example.
+ """
+ from airflow import settings
+
+ if isinstance(name, str) and settings.HIDE_SENSITIVE_VAR_CONN_FIELDS:
+ name = name.strip().lower()
+ return any(s in name for s in self.sensitive_variables_fields)
+ return False
+
def add_mask(self, secret: JsonValue, name: str | None = None):
"""Add a new secret to be masked to this filter instance."""
if isinstance(secret, dict):
@@ -559,7 +540,7 @@ class SecretsMasker(logging.Filter):
if secret.lower() in SECRETS_TO_SKIP_MASKING:
return
- min_length = get_min_secret_length()
+ min_length = self.min_length_to_mask
if len(secret) < min_length:
if not SecretsMasker._has_warned_short_secret:
log.warning(
@@ -580,7 +561,7 @@ class SecretsMasker(logging.Filter):
continue
pattern = re.escape(s)
- if pattern not in self.patterns and (not name or
should_hide_value_for_key(name)):
+ if pattern not in self.patterns and (not name or
self.should_hide_value_for_key(name)):
self.patterns.add(pattern)
new_mask = True
if new_mask:
diff --git a/shared/secrets_masker/tests/secrets_masker/test_secrets_masker.py
b/shared/secrets_masker/tests/secrets_masker/test_secrets_masker.py
index 09b5f6c5720..bada3b39281 100644
--- a/shared/secrets_masker/tests/secrets_masker/test_secrets_masker.py
+++ b/shared/secrets_masker/tests/secrets_masker/test_secrets_masker.py
@@ -30,14 +30,13 @@ from unittest.mock import patch
import pytest
from airflow_shared.secrets_masker.secrets_masker import (
+ DEFAULT_SENSITIVE_FIELDS,
RedactedIO,
SecretsMasker,
- get_sensitive_variables_fields,
mask_secret,
merge,
redact,
reset_secrets_masker,
- should_hide_value_for_key,
)
from tests_common.test_utils.config import env_vars
@@ -46,6 +45,17 @@ pytestmark = pytest.mark.enable_redact
p = "password"
+def configure_secrets_masker_for_test(
+ masker: SecretsMasker, min_length: int = 5, sensitive_fields: list[str] =
None
+):
+ """Helper function to configure a SecretsMasker instance for testing."""
+ masker.min_length_to_mask = min_length
+ if sensitive_fields is None:
+ masker.sensitive_variables_fields = list(DEFAULT_SENSITIVE_FIELDS)
+ else:
+ masker.sensitive_variables_fields = sensitive_fields
+
+
def lineno():
"""Returns the current line number in our program."""
return inspect.currentframe().f_back.f_lineno
@@ -84,6 +94,7 @@ def logger(caplog):
caplog.handler.setFormatter(formatter)
logger.handlers = [caplog.handler]
filt = SecretsMasker()
+ configure_secrets_masker_for_test(filt)
SecretsMasker.enable_log_masking()
logger.addFilter(filt)
@@ -244,8 +255,9 @@ class TestSecretsMasker:
)
def test_mask_secret(self, name, value, expected_mask):
filt = SecretsMasker()
- filt.add_mask(value, name)
+ configure_secrets_masker_for_test(filt)
+ filt.add_mask(value, name)
assert filt.patterns == expected_mask
@pytest.mark.parametrize(
@@ -281,6 +293,7 @@ class TestSecretsMasker:
)
def test_redact(self, patterns, name, value, expected):
filt = SecretsMasker()
+ configure_secrets_masker_for_test(filt)
for val in patterns:
filt.add_mask(val)
@@ -303,11 +316,13 @@ class TestSecretsMasker:
)
def test_redact_replacement(self, name, value, expected):
filt = SecretsMasker()
+ configure_secrets_masker_for_test(filt)
assert filt.redact(value, name, replacement="*️⃣*️⃣*️⃣") == expected
def test_redact_filehandles(self, caplog):
filt = SecretsMasker()
+ configure_secrets_masker_for_test(filt)
with open("/dev/null", "w") as handle:
assert filt.redact(handle, None) == handle
@@ -328,6 +343,7 @@ class TestSecretsMasker:
)
def test_redact_max_depth(self, val, expected, max_depth):
secrets_masker = SecretsMasker()
+ configure_secrets_masker_for_test(secrets_masker)
secrets_masker.add_mask("abcdef")
with patch(
"airflow_shared.secrets_masker.secrets_masker._secrets_masker",
return_value=secrets_masker
@@ -368,6 +384,7 @@ class TestSecretsMasker:
self,
):
secrets_masker = SecretsMasker()
+ configure_secrets_masker_for_test(secrets_masker)
secrets_masker.add_mask("mask_this")
secrets_masker.add_mask("and_this")
secrets_masker.add_mask("maybe_this_too")
@@ -426,7 +443,9 @@ class TestShouldHideValueForKey:
],
)
def test_hiding_defaults(self, key, expected_result):
- assert expected_result == should_hide_value_for_key(key)
+ masker = SecretsMasker()
+ configure_secrets_masker_for_test(masker)
+ assert expected_result == masker.should_hide_value_for_key(key)
@pytest.mark.parametrize(
("sensitive_variable_fields", "key", "expected_result"),
@@ -444,11 +463,16 @@ class TestShouldHideValueForKey:
def test_hiding_config(self, sensitive_variable_fields, key,
expected_result):
env_value = str(sensitive_variable_fields) if
sensitive_variable_fields is not None else ""
with env_vars({"AIRFLOW__CORE__SENSITIVE_VAR_CONN_NAMES": env_value}):
- get_sensitive_variables_fields.cache_clear()
- try:
- assert expected_result == should_hide_value_for_key(key)
- finally:
- get_sensitive_variables_fields.cache_clear()
+ masker = SecretsMasker()
+ sensitive_fields = list(DEFAULT_SENSITIVE_FIELDS)
+ if sensitive_variable_fields:
+ additional_fields = {
+ field.strip() for field in
sensitive_variable_fields.split(",") if field.strip()
+ }
+ sensitive_fields = list(DEFAULT_SENSITIVE_FIELDS |
additional_fields)
+
+ configure_secrets_masker_for_test(masker,
sensitive_fields=sensitive_fields)
+ assert expected_result == masker.should_hide_value_for_key(key)
class ShortExcFormatter(logging.Formatter):
@@ -463,6 +487,7 @@ class TestRedactedIO:
@pytest.fixture(scope="class", autouse=True)
def reset_secrets_masker(self):
self.secrets_masker = SecretsMasker()
+ configure_secrets_masker_for_test(self.secrets_masker)
with patch(
"airflow_shared.secrets_masker.secrets_masker._secrets_masker",
return_value=self.secrets_masker,
@@ -503,6 +528,7 @@ class TestMaskSecretAdapter:
@pytest.fixture(autouse=True)
def reset_secrets_masker_and_skip_escape(self):
self.secrets_masker = SecretsMasker()
+ configure_secrets_masker_for_test(self.secrets_masker)
with patch(
"airflow_shared.secrets_masker.secrets_masker._secrets_masker",
return_value=self.secrets_masker,
@@ -511,13 +537,21 @@ class TestMaskSecretAdapter:
yield
def test_calling_mask_secret_adds_adaptations_for_returned_str(self):
+ import urllib.parse
+
with env_vars({"AIRFLOW__LOGGING__SECRET_MASK_ADAPTER":
"urllib.parse.quote"}):
+ # Manually configure the adapter since we don't read from config
anymore
+ self.secrets_masker.secret_mask_adapter = urllib.parse.quote
mask_secret("secret<>&", None)
assert self.secrets_masker.patterns == {"secret%3C%3E%26", "secret<>&"}
def test_calling_mask_secret_adds_adaptations_for_returned_iterable(self):
+ import urllib.parse
+
with env_vars({"AIRFLOW__LOGGING__SECRET_MASK_ADAPTER":
"urllib.parse.urlparse"}):
+ # Manually configure the adapter since we don't read from config
anymore
+ self.secrets_masker.secret_mask_adapter = urllib.parse.urlparse
mask_secret("https://airflow.apache.org/docs/apache-airflow/stable";, "password")
assert self.secrets_masker.patterns == {
@@ -529,6 +563,8 @@ class TestMaskSecretAdapter:
def test_calling_mask_secret_not_set(self):
with env_vars({"AIRFLOW__LOGGING__SECRET_MASK_ADAPTER": ""}):
+ # Ensure no adapter is set
+ self.secrets_masker.secret_mask_adapter = None
mask_secret("a secret")
assert self.secrets_masker.patterns == {"a secret"}
@@ -551,24 +587,24 @@ class TestMaskSecretAdapter:
SecretsMasker._has_warned_short_secret = True
filt = SecretsMasker()
+ configure_secrets_masker_for_test(filt, min_length=5)
- with
patch("airflow_shared.secrets_masker.secrets_masker.get_min_secret_length",
return_value=5):
- caplog.clear()
+ caplog.clear()
- filt.add_mask(secret)
+ filt.add_mask(secret)
- if is_first_short:
- assert "Skipping masking for a secret as it's too short" in
caplog.text
- assert len(caplog.records) == 1
- else:
- assert "Skipping masking for a secret as it's too short" not
in caplog.text
+ if is_first_short:
+ assert "Skipping masking for a secret as it's too short" in
caplog.text
+ assert len(caplog.records) == 1
+ else:
+ assert "Skipping masking for a secret as it's too short" not in
caplog.text
- if should_be_masked:
- assert secret in filt.patterns
- else:
- assert secret not in filt.patterns
+ if should_be_masked:
+ assert secret in filt.patterns
+ else:
+ assert secret not in filt.patterns
- caplog.clear()
+ caplog.clear()
if should_be_masked:
assert filt.replacer is not None
@@ -577,6 +613,7 @@ class TestMaskSecretAdapter:
class TestStructuredVsUnstructuredMasking:
def test_structured_sensitive_fields_always_masked(self):
secrets_masker = SecretsMasker()
+ configure_secrets_masker_for_test(secrets_masker, min_length=5)
short_password = "pwd"
short_token = "tk"
@@ -591,16 +628,16 @@ class TestStructuredVsUnstructuredMasking:
with patch(
"airflow_shared.secrets_masker.secrets_masker._secrets_masker",
return_value=secrets_masker
):
- with
patch("airflow_shared.secrets_masker.secrets_masker.get_min_secret_length",
return_value=5):
- redacted_data = redact(test_data)
+ redacted_data = redact(test_data)
- assert redacted_data["password"] == "***"
- assert redacted_data["api_key"] == "***"
- assert redacted_data["connection"]["secret"] == "***"
+ assert redacted_data["password"] == "***"
+ assert redacted_data["api_key"] == "***"
+ assert redacted_data["connection"]["secret"] == "***"
def test_unstructured_text_min_length_enforced(self):
secrets_masker = SecretsMasker()
min_length = 5
+ configure_secrets_masker_for_test(secrets_masker,
min_length=min_length)
short_secret = "abc"
long_secret = "abcdef"
@@ -608,21 +645,18 @@ class TestStructuredVsUnstructuredMasking:
with patch(
"airflow_shared.secrets_masker.secrets_masker._secrets_masker",
return_value=secrets_masker
):
- with patch(
-
"airflow_shared.secrets_masker.secrets_masker.get_min_secret_length",
return_value=min_length
- ):
- secrets_masker.add_mask(short_secret)
- secrets_masker.add_mask(long_secret)
+ secrets_masker.add_mask(short_secret)
+ secrets_masker.add_mask(long_secret)
- assert short_secret not in secrets_masker.patterns
- assert long_secret in secrets_masker.patterns
+ assert short_secret not in secrets_masker.patterns
+ assert long_secret in secrets_masker.patterns
- test_data = f"Containing {short_secret} and {long_secret}"
- redacted = secrets_masker.redact(test_data)
+ test_data = f"Containing {short_secret} and {long_secret}"
+ redacted = secrets_masker.redact(test_data)
- assert short_secret in redacted
- assert long_secret not in redacted
- assert "***" in redacted
+ assert short_secret in redacted
+ assert long_secret not in redacted
+ assert "***" in redacted
class TestContainerTypesRedaction:
@@ -639,6 +673,7 @@ class TestContainerTypesRedaction:
normal_env_var = MockV1EnvVar("app_name", "my_app")
secrets_masker = SecretsMasker()
+ configure_secrets_masker_for_test(secrets_masker)
with patch(
"airflow_shared.secrets_masker.secrets_masker._secrets_masker",
return_value=secrets_masker
@@ -667,6 +702,7 @@ class TestContainerTypesRedaction:
}
secrets_masker = SecretsMasker()
+ configure_secrets_masker_for_test(secrets_masker)
secrets_masker.add_mask("secret_token")
secrets_masker.add_mask("password=secret")
@@ -693,6 +729,7 @@ class TestEdgeCases:
circular_dict["self_ref"] = circular_dict
secrets_masker = SecretsMasker()
+ configure_secrets_masker_for_test(secrets_masker)
with patch(
"airflow_shared.secrets_masker.secrets_masker._secrets_masker",
return_value=secrets_masker
@@ -708,6 +745,7 @@ class TestEdgeCases:
regex_secrets = ["password+with*chars", "token.with[special]chars",
"api_key^that$needs(escaping)"]
secrets_masker = SecretsMasker()
+ configure_secrets_masker_for_test(secrets_masker)
for secret in regex_secrets:
secrets_masker.add_mask(secret)
@@ -727,6 +765,7 @@ class TestEdgeCases:
class TestDirectMethodCalls:
def test_redact_all_directly(self):
secrets_masker = SecretsMasker()
+ configure_secrets_masker_for_test(secrets_masker)
test_data = {
"string": "should_be_masked",
@@ -752,6 +791,7 @@ class TestDirectMethodCalls:
class TestMixedDataScenarios:
def test_mixed_structured_unstructured_data(self):
secrets_masker = SecretsMasker()
+ configure_secrets_masker_for_test(secrets_masker)
unstructured_secret = "this_is_a_secret_pattern"
secrets_masker.add_mask(unstructured_secret)
@@ -779,6 +819,10 @@ class TestMixedDataScenarios:
class TestSecretsMaskerMerge:
"""Test the merge functionality for restoring original values from
redacted data."""
+ def setup_method(self):
+ self.masker = SecretsMasker()
+ configure_secrets_masker_for_test(self.masker)
+
@pytest.mark.parametrize(
("new_value", "old_value", "name", "expected"),
[
@@ -791,7 +835,7 @@ class TestSecretsMaskerMerge:
],
)
def test_merge_simple_strings(self, new_value, old_value, name, expected):
- result = merge(new_value, old_value, name)
+ result = self.masker.merge(new_value, old_value, name)
assert result == expected
@pytest.mark.parametrize(
@@ -843,7 +887,7 @@ class TestSecretsMaskerMerge:
],
)
def test_merge_dictionaries(self, old_data, new_data, expected):
- result = merge(new_data, old_data)
+ result = self.masker.merge(new_data, old_data)
assert result == expected
@pytest.mark.parametrize(
@@ -927,7 +971,7 @@ class TestSecretsMaskerMerge:
],
)
def test_merge_collections(self, old_data, new_data, name, expected):
- result = merge(new_data, old_data, name)
+ result = self.masker.merge(new_data, old_data, name)
assert result == expected
def test_merge_mismatched_types(self):
@@ -937,7 +981,7 @@ class TestSecretsMaskerMerge:
# When types don't match, prefer the new item
expected = "some_string"
- result = merge(new_data, old_data)
+ result = self.masker.merge(new_data, old_data)
assert result == expected
def test_merge_with_missing_keys(self):
@@ -955,7 +999,7 @@ class TestSecretsMaskerMerge:
"common_key": "new_common",
}
- result = merge(new_data, old_data)
+ result = self.masker.merge(new_data, old_data)
assert result == expected
def test_merge_complex_redacted_structures(self):
@@ -972,7 +1016,7 @@ class TestSecretsMaskerMerge:
"normal_field": "new_normal_value",
}
- result = merge(new_data, old_data)
+ result = self.masker.merge(new_data, old_data)
expected = {
"some_config": {
"nested_password": "original_nested_password",
@@ -1013,7 +1057,7 @@ class TestSecretsMaskerMerge:
}
}
- result = merge(new_data, old_data)
+ result = self.masker.merge(new_data, old_data)
assert result == expected
def test_merge_max_depth(self):
@@ -1023,14 +1067,14 @@ class TestSecretsMaskerMerge:
result = merge(new_data, old_data, max_depth=1)
assert result == new_data
- result = merge(new_data, old_data, max_depth=10)
+ result = self.masker.merge(new_data, old_data, max_depth=10)
assert result["level1"]["level2"]["level3"]["password"] ==
"original_password"
def test_merge_enum_values(self):
old_enum = MyEnum.testname
new_enum = MyEnum.testname2
- result = merge(new_enum, old_enum)
+ result = self.masker.merge(new_enum, old_enum)
assert result == new_enum
assert isinstance(result, MyEnum)
@@ -1043,7 +1087,7 @@ class TestSecretsMaskerMerge:
}
# Step 1: Redact the original data
- redacted_dict = redact(original_config)
+ redacted_dict = self.masker.redact(original_config)
# Verify sensitive fields are redacted
assert redacted_dict["database"]["password"] == "***"
@@ -1058,7 +1102,7 @@ class TestSecretsMaskerMerge:
# User left password as "***" (unchanged)
# Step 3: Merge to restore unchanged sensitive values
- final_dict = merge(updated_dict, original_config)
+ final_dict = self.masker.merge(updated_dict, original_config)
# Verify the results
assert final_dict["database"]["password"] == "super_secret_password"
# Restored
