jason810496 commented on code in PR #64209:
URL: https://github.com/apache/airflow/pull/64209#discussion_r2993015955
##########
shared/configuration/src/airflow_shared/configuration/parser.py:
##########
@@ -279,24 +279,77 @@ def _lookup_sequence(self) -> list[Callable]:
self._get_option_from_provider_metadata_config_fallbacks,
]
- def _get_config_sources_for_as_dict(self) -> list[tuple[str,
ConfigParser]]:
+ @functools.cached_property
+ def configuration_description(self) -> dict[str, dict[str, Any]]:
+ """
+ Return configuration description from multiple sources.
+
+ Respects the ``_use_providers_configuration`` flag to decide whether
to include
+ provider configuration.
+
+ The priority order is as follows (later sources override earlier ones):
+
+ 1. The base configuration description provided in ``__init__``,
usually loaded
+ from ``config.yml`` in core.
+ 2. ``_provider_cfg_config_fallback_default_values``, loaded from
+ ``provider_config_fallback_defaults.cfg``.
+ 3. ``_provider_metadata_config_fallback_default_values``, loaded from
provider
+ packages' ``get_provider_info`` method (via ProvidersManager /
+ RuntimeProvidersManager's ``.provider_configs`` property).
+
+ We use ``cached_property`` to cache the merged result; clear this
cache (via
+ ``invalidate_cache``) when toggling ``_use_providers_configuration``.
+ """
+ if not self._use_providers_configuration:
+ return self._configuration_description
+
+ merged_description: dict[str, dict[str, Any]] =
deepcopy(self._configuration_description)
+
+ # Merge full provider config descriptions (with metadata like
sensitive, description, etc.)
+ # from provider packages' get_provider_info method, reusing the cached
raw dict.
+ for section, section_content in
self._provider_metadata_configuration_description.items():
+ if section not in merged_description:
+ merged_description[section] = deepcopy(section_content)
+ else:
+ existing_options =
merged_description[section].setdefault("options", {})
+ for option, option_content in section_content.get("options",
{}).items():
+ if option not in existing_options:
+ existing_options[option] = deepcopy(option_content)
+
+ # Merge default values from cfg-based fallbacks (key=value only, no
metadata).
+ # Uses setdefault so provider metadata values above take priority.
+ cfg = self._provider_cfg_config_fallback_default_values
+ for section in cfg.sections():
+ section_options = merged_description.setdefault(section,
{"options": {}}).setdefault(
+ "options", {}
+ )
+ for option in cfg.options(section):
+ opt_dict = section_options.setdefault(option, {})
+ opt_dict.setdefault("default", cfg.get(section, option))
+ # For cfg-only options with no provider metadata, infer
sensitivity from name.
+ if "sensitive" not in opt_dict and
option.endswith(("password", "secret")):
+ opt_dict["sensitive"] = True
Review Comment:
Not sure do we need `_secrets_masker().should_hide_value_for_key(option)` or
not?
IMO, perhaps we don't even need to handle the sensitive case here as we have
secret masker for CLI, API, etc (anyplace that might expose the sensitive conf
to user). The secret masker will handle the masking for us at higher level
instead of at current `conf` level.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
