Taragolis commented on code in PR #23560:
URL: https://github.com/apache/airflow/pull/23560#discussion_r900877600
##########
airflow/configuration.py:
##########
@@ -1487,47 +1500,118 @@ def set(*args, **kwargs) -> None:
conf.set(*args, **kwargs)
-def ensure_secrets_loaded() -> List[BaseSecretsBackend]:
- """
- Ensure that all secrets backends are loaded.
- If the secrets_backend_list contains only 2 default backends, reload it.
- """
- # Check if the secrets_backend_list contains only 2 default backends
- if len(secrets_backend_list) == 2:
- return initialize_secrets_backends()
- return secrets_backend_list
+class DefaultSecretsBackend(UserList):
+ """List Container which use for store default secrets backends."""
+
+
+@dataclass(frozen=True)
+class SecretsBackendConfig:
+ """Secrets Backend Config dataclass helper."""
+
+ backend: str
+ backend_kwargs: Dict[str, Any] = field(default_factory=dict)
+
+ @classmethod
+ def from_dict(cls, d: Dict[str, Any]) -> 'SecretsBackendConfig':
+ """
+ Read Secret Backend Config from dictionary
+
+ Ignores all unexpected keywords
+ """
+ return cls(d['backend'], d.get('backend_kwargs', {}))
+ @classmethod
+ def from_config(cls) -> Optional['SecretsBackendConfig']:
+ """Try to get ``SecretsBackendConfig`` from airflow config [secrets]
section"""
+ secrets_backend = conf.get(section='secrets', key='backend',
fallback=None)
-def get_custom_secret_backend() -> Optional[BaseSecretsBackend]:
- """Get Secret Backend if defined in airflow.cfg"""
- secrets_backend_cls = conf.getimport(section='secrets', key='backend')
+ if not secrets_backend:
+ return None
- if secrets_backend_cls:
try:
- backends: Any = conf.get(section='secrets', key='backend_kwargs',
fallback='{}')
- alternative_secrets_config_dict = json.loads(backends)
+ secrets_backend_kwargs: Any = conf.get(section='secrets',
key='backend_kwargs', fallback='{}')
+ secrets_backend_kwargs = json.loads(secrets_backend_kwargs)
except JSONDecodeError:
- alternative_secrets_config_dict = {}
+ secrets_backend_kwargs = {}
+
+ return cls(secrets_backend, secrets_backend_kwargs)
+
+ def initialize(self) -> BaseSecretsBackend:
+ """Initialize Secrets Backend."""
+ return import_string(self.backend)(**self.backend_kwargs)
+
+
+class UniqueSecretsBackendsConfigs(UserList):
Review Comment:
This helper only remove backends with the same configuration.
I think there is no reason check again backend which return `None` again
because in most case the result in most case would be same, only show warning
that this Backend with exactly the same configuration already in list of
configuration
---
Example:
```json
[
{
"backend":"airflow.secrets.environment_variables.EnvironmentVariablesBackend"
},
{
"backend":"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
"backend_kwargs":{
"connections_prefix":"/airflow/connections",
"variables_prefix": "/airflow/variables",
"profile_name":"default"
}
},
{
"backend":"airflow.secrets.metastore.MetastoreBackend"
},
{
"backend":"airflow.secrets.environment_variables.EnvironmentVariablesBackend"
},
{
"backend":"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
"backend_kwargs":{
"connections_prefix":"/another/connections",
"variables_prefix": "/another/variables",
"profile_name":"awesome",
"region_name": "ap-southeast-1"
}
},
{
"backend":"airflow.secrets.metastore.MetastoreBackend"
}
]
```
keep only this one
```json
[
{
"backend":"airflow.secrets.environment_variables.EnvironmentVariablesBackend"
},
{
"backend":"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
"backend_kwargs":{
"connections_prefix":"/airflow/connections",
"variables_prefix": "/airflow/variables",
"profile_name":"default"
}
},
{
"backend":"airflow.secrets.metastore.MetastoreBackend"
},
{
"backend":"airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend",
"backend_kwargs":{
"connections_prefix":"/another/connections",
"variables_prefix": "/another/variables",
"profile_name":"awesome",
"region_name": "ap-southeast-1"
}
}
]
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
