This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ff3f61af405 Respect deprecated options in default config parser
(#61289)
ff3f61af405 is described below
commit ff3f61af4055a8b10c9e3227118517d17411a762
Author: Jason(Zhe-You) Liu <[email protected]>
AuthorDate: Tue Feb 3 21:32:58 2026 +0800
Respect deprecated options in default config parser (#61289)
* Refactor configuration parser to respect deprecated options in default
parser
* Add tests to ensure deprecated options are skipped in configuration parser
* Add test for handling default values of deprecated options in
AirflowConfigParser
* Add test case without fallback param
---
airflow-core/src/airflow/configuration.py | 13 +--
airflow-core/tests/unit/core/test_configuration.py | 53 +++++++++++
.../src/airflow_shared/configuration/parser.py | 29 ++++++
.../tests/configuration/test_parser.py | 100 ++++++++++++++++++++-
task-sdk/src/airflow/sdk/configuration.py | 20 ++---
5 files changed, 188 insertions(+), 27 deletions(-)
diff --git a/airflow-core/src/airflow/configuration.py
b/airflow-core/src/airflow/configuration.py
index 7bc34bec47a..82483b61794 100644
--- a/airflow-core/src/airflow/configuration.py
+++ b/airflow-core/src/airflow/configuration.py
@@ -43,6 +43,7 @@ from airflow._shared.configuration.parser import (
VALUE_NOT_FOUND_SENTINEL,
AirflowConfigParser as _SharedAirflowConfigParser,
ValueNotFound,
+ configure_parser_from_configuration_description,
)
from airflow._shared.module_loading import import_string
from airflow.exceptions import AirflowConfigException, RemovedInAirflow4Warning
@@ -684,17 +685,7 @@ def
create_default_config_parser(configuration_description: dict[str, dict[str,
"""
parser = ConfigParser()
all_vars = get_all_expansion_variables()
- for section, section_desc in configuration_description.items():
- parser.add_section(section)
- options = section_desc["options"]
- for key in options:
- default_value = options[key]["default"]
- is_template = options[key].get("is_template", False)
- if default_value is not None:
- if is_template or not isinstance(default_value, str):
- parser.set(section, key, default_value)
- else:
- parser.set(section, key, default_value.format(**all_vars))
+ configure_parser_from_configuration_description(parser,
configuration_description, all_vars)
return parser
diff --git a/airflow-core/tests/unit/core/test_configuration.py
b/airflow-core/tests/unit/core/test_configuration.py
index a3d02d070fa..9c7ec9a3d16 100644
--- a/airflow-core/tests/unit/core/test_configuration.py
+++ b/airflow-core/tests/unit/core/test_configuration.py
@@ -1743,6 +1743,59 @@ sql_alchemy_conn=sqlite://test
assert airflow_cfg.get("core", "hostname_callable") == "test-fn"
assert sum(1 for option in all_core_options_including_defaults if
option == "hostname_callable") == 1
+ def test_get_options_including_deprecated_defaults_should_skip(self,
tmp_path):
+ # 1. Import the real function before patching
+ from airflow.configuration import _default_config_file_path as
real_default_config_file_path
+
+ config_yaml_with_deprecated = textwrap.dedent(
+ """
+ test_deprecated_section:
+ description: Test section with deprecated options
+ options:
+ key_without_deprecation:
+ type: string
+ default: value1
+ key_with_version_deprecated:
+ type: string
+ default: value2
+ version_deprecated: 3.0.0
+ key_with_deprecation_reason:
+ type: string
+ default: value3
+ deprecation_reason: This key is deprecated.
+ key_with_both_deprecation:
+ type: string
+ default: value4
+ version_deprecated: 3.0.0
+ deprecation_reason: This key is deprecated.
+ """
+ )
+
+ # mock _default_config_file_path with tmp file containing above yaml
+ config_file = tmp_path / "config.yml"
+ config_file.write_text(config_yaml_with_deprecated)
+
+ # 2. Define the side effect using the captured 'real' function
+ def custom_default_config_file_path(file_name):
+ if file_name == "config.yml":
+ return os.fspath(config_file)
+ return real_default_config_file_path(file_name)
+
+ # 3. Apply the patch via context manager using the side_effect
+ with mock.patch(
+ "airflow.configuration._default_config_file_path",
side_effect=custom_default_config_file_path
+ ):
+ # act
+ airflow_cfg = AirflowConfigParser()
+
+ assert airflow_cfg.has_option("test_deprecated_section",
"key_without_deprecation")
+ for deprecated_key in [
+ "key_with_version_deprecated",
+ "key_with_deprecation_reason",
+ "key_with_both_deprecation",
+ ]:
+ assert not airflow_cfg.has_option("test_deprecated_section",
deprecated_key)
+
@skip_if_force_lowest_dependencies_marker
def test_sensitive_values():
diff --git a/shared/configuration/src/airflow_shared/configuration/parser.py
b/shared/configuration/src/airflow_shared/configuration/parser.py
index baf695daaa8..f7e3755c076 100644
--- a/shared/configuration/src/airflow_shared/configuration/parser.py
+++ b/shared/configuration/src/airflow_shared/configuration/parser.py
@@ -109,6 +109,35 @@ def _is_template(configuration_description: dict[str,
dict[str, Any]], section:
return configuration_description.get(section, {}).get(key,
{}).get("is_template", False)
+def configure_parser_from_configuration_description(
+ parser: ConfigParser,
+ configuration_description: dict[str, dict[str, Any]],
+ all_vars: dict[str, Any],
+) -> None:
+ """
+ Configure a ConfigParser based on configuration description.
+
+ :param parser: ConfigParser to configure
+ :param configuration_description: configuration description from config.yml
+ """
+ for section, section_desc in configuration_description.items():
+ parser.add_section(section)
+ options = section_desc["options"]
+ for key in options:
+ default_value = options[key]["default"]
+ is_template = options[key].get("is_template", False)
+ if (default_value is not None) and not (
+ options[key].get("version_deprecated") or
options[key].get("deprecation_reason")
+ ):
+ if is_template or not isinstance(default_value, str):
+ parser.set(section, key, str(default_value))
+ else:
+ try:
+ parser.set(section, key,
default_value.format(**all_vars))
+ except (KeyError, ValueError):
+ parser.set(section, key, default_value)
+
+
class AirflowConfigParser(ConfigParser):
"""
Base configuration parser with pure parsing logic.
diff --git a/shared/configuration/tests/configuration/test_parser.py
b/shared/configuration/tests/configuration/test_parser.py
index 484e6373f0e..16c8667dfad 100644
--- a/shared/configuration/tests/configuration/test_parser.py
+++ b/shared/configuration/tests/configuration/test_parser.py
@@ -31,7 +31,10 @@ from unittest.mock import patch
import pytest
from airflow_shared.configuration.exceptions import AirflowConfigException
-from airflow_shared.configuration.parser import AirflowConfigParser as
_SharedAirflowConfigParser
+from airflow_shared.configuration.parser import (
+ AirflowConfigParser as _SharedAirflowConfigParser,
+ configure_parser_from_configuration_description,
+)
class AirflowConfigParser(_SharedAirflowConfigParser):
@@ -790,3 +793,98 @@ existing_list = one,two,three
test_conf.set("Test", "NewKey", "new_value")
assert test_conf.get("test", "newkey") == "new_value"
assert test_conf.get("TEST", "NEWKEY") == "new_value"
+
+ def
test_configure_parser_from_configuration_description_with_deprecated_options(self):
+ """
+ Test that configure_parser_from_configuration_description respects
deprecated options.
+ """
+ configuration_description = {
+ "test_section": {
+ "options": {
+ "non_deprecated_key": {"default": "non_deprecated_value"},
+ "deprecated_key_version": {
+ "default": "deprecated_value_version",
+ "version_deprecated": "3.0.0",
+ },
+ "deprecated_key_reason": {
+ "default": "deprecated_value_reason",
+ "deprecation_reason": "Some reason",
+ },
+ "none_default_deprecated": {"default": None,
"deprecation_reason": "No default"},
+ }
+ }
+ }
+ parser = ConfigParser()
+ all_vars = {}
+
+ configure_parser_from_configuration_description(parser,
configuration_description, all_vars)
+
+ # Assert that the non-deprecated key is present
+ assert parser.has_option("test_section", "non_deprecated_key")
+ assert parser.get("test_section", "non_deprecated_key") ==
"non_deprecated_value"
+
+ # Assert that the deprecated keys are NOT present
+ assert not parser.has_option("test_section", "deprecated_key_version")
+ assert not parser.has_option("test_section", "deprecated_key_reason")
+
+ # Assert that options with default None are still not set
+ assert not parser.has_option("test_section", "none_default_deprecated")
+
+ def test_get_default_value_deprecated(self):
+ """Test 'conf.get' for deprecated options and should not return
default value."""
+
+ class TestConfigParser(AirflowConfigParser):
+ def __init__(self):
+ configuration_description = {
+ "test_section": {
+ "options": {
+ "deprecated_key": {
+ "default": "some_value",
+ "deprecation_reason": "deprecated",
+ },
+ "deprecated_key2": {
+ "default": "some_value",
+ "version_deprecated": "2.0.0",
+ },
+ "deprecated_key3": {
+ "default": None,
+ "version_deprecated": "2.0.0",
+ },
+ "active_key": {
+ "default": "active_value",
+ },
+ }
+ }
+ }
+ _default_values = ConfigParser()
+ # verify configure_parser_from_configuration_description logic
of skipping
+ configure_parser_from_configuration_description(
+ _default_values, configuration_description, {}
+ )
+ _SharedAirflowConfigParser.__init__(self,
configuration_description, _default_values)
+
+ test_conf = TestConfigParser()
+ deprecated_conf_list = [
+ ("test_section", "deprecated_key"),
+ ("test_section", "deprecated_key2"),
+ ("test_section", "deprecated_key3"),
+ ]
+
+ # case 1: using `get` with fallback
+ # should return fallback if not found
+ expected_sentinel = object()
+ for section, key in deprecated_conf_list:
+ assert test_conf.get(section, key, fallback=expected_sentinel) is
expected_sentinel
+
+ # case 2: using `get` without fallback
+ # should raise AirflowConfigException
+ for section, key in deprecated_conf_list:
+ with pytest.raises(
+ AirflowConfigException,
+ match=re.escape(f"section/key [{section}/{key}] not found in
config"),
+ ):
+ test_conf.get(section, key)
+
+ # case 3: active (non-deprecated) key
+ # Active key should be present
+ assert test_conf.get("test_section", "active_key") == "active_value"
diff --git a/task-sdk/src/airflow/sdk/configuration.py
b/task-sdk/src/airflow/sdk/configuration.py
index 61c1fb5686d..9d0da594b4f 100644
--- a/task-sdk/src/airflow/sdk/configuration.py
+++ b/task-sdk/src/airflow/sdk/configuration.py
@@ -27,7 +27,10 @@ from io import StringIO
from typing import Any
from airflow.sdk import yaml
-from airflow.sdk._shared.configuration.parser import AirflowConfigParser as
_SharedAirflowConfigParser
+from airflow.sdk._shared.configuration.parser import (
+ AirflowConfigParser as _SharedAirflowConfigParser,
+ configure_parser_from_configuration_description,
+)
from airflow.sdk.execution_time.secrets import
_SERVER_DEFAULT_SECRETS_SEARCH_PATH
log = logging.getLogger(__name__)
@@ -85,20 +88,7 @@ def create_default_config_parser(configuration_description:
dict[str, dict[str,
"""
parser = ConfigParser()
all_vars = get_sdk_expansion_variables()
- for section, section_desc in configuration_description.items():
- parser.add_section(section)
- options = section_desc["options"]
- for key in options:
- default_value = options[key]["default"]
- is_template = options[key].get("is_template", False)
- if default_value is not None:
- if is_template or not isinstance(default_value, str):
- parser.set(section, key, str(default_value))
- else:
- try:
- parser.set(section, key,
default_value.format(**all_vars))
- except (KeyError, ValueError):
- parser.set(section, key, default_value)
+ configure_parser_from_configuration_description(parser,
configuration_description, all_vars)
return parser
