ashb commented on a change in pull request #9645: URL: https://github.com/apache/airflow/pull/9645#discussion_r449642855
########## File path: airflow/configuration.py ########## @@ -115,7 +117,9 @@ class AirflowConfigParser(ConfigParser): # These configuration elements can be fetched as the stdout of commands # following the "{section}__{name}__cmd" pattern, the idea behind this # is to not store password on boxes in text files. - as_command_stdout = { + # These configs can also be fetched from Secrets backend + # following the "{section}__{name}__secret" pattern + configs_with_secret = { Review comment: ```suggestion senstive_config_values = { ``` ########## File path: airflow/providers/amazon/aws/secrets/secrets_manager.py ########## @@ -62,13 +67,15 @@ def __init__( self, connections_prefix: str = 'airflow/connections', variables_prefix: str = 'airflow/variables', + configurations_prefix: str = 'airflow/configuration', Review comment: ```suggestion config_prefix: str = 'airflow/config', ``` ########## File path: airflow/secrets/base_secrets.py ########## @@ -73,3 +73,12 @@ def get_variable(self, key: str) -> Optional[str]: :return: Variable Value """ raise NotImplementedError() + + def get_configuration(self, key: str) -> Optional[str]: Review comment: ```suggestion def get_configuration(self, section: str, key: str) -> Optional[str]: ``` ########## File path: airflow/providers/amazon/aws/secrets/secrets_manager.py ########## @@ -62,13 +67,15 @@ def __init__( self, connections_prefix: str = 'airflow/connections', variables_prefix: str = 'airflow/variables', + configurations_prefix: str = 'airflow/configuration', Review comment: etc (haven't made this change everywhere) ########## File path: airflow/providers/amazon/aws/secrets/secrets_manager.py ########## @@ -42,8 +42,11 @@ class SecretsManagerBackend(BaseSecretsBackend, LoggingMixin): For example, if secrets prefix is ``airflow/connections/smtp_default``, this would be accessible if you provide ``{"connections_prefix": "airflow/connections"}`` and request conn_id ``smtp_default``. - And if variables prefix is ``airflow/variables/hello``, this would be accessible + If variables prefix is ``airflow/variables/hello``, this would be accessible if you provide ``{"variables_prefix": "airflow/variables"}`` and request variable key ``hello``. + And if configurations_prefix is ``airflow/configurations/sql_alchemy_conn``, this would be accessible Review comment: ```suggestion And if configurations_prefix is ``airflow/config/core/sql_alchemy_conn``, this would be accessible ``` ########## File path: airflow/configuration.py ########## @@ -267,17 +271,49 @@ def _get_env_var_option(self, section, key): env_var_cmd = env_var + '_CMD' if env_var_cmd in os.environ: # if this is a valid command key... - if (section, key) in self.as_command_stdout: + if (section, key) in self.configs_with_secret: return run_command(os.environ[env_var_cmd]) def _get_cmd_option(self, section, key): fallback_key = key + '_cmd' # if this is a valid command key... - if (section, key) in self.as_command_stdout: + if (section, key) in self.configs_with_secret: if super().has_option(section, fallback_key): command = super().get(section, fallback_key) return run_command(command) + @cached_property + def _secrets_backend_client(self): + if super().has_option('secrets', 'backend') is False: + return None + + secrets_backend_cls = super().get('secrets', 'backend') + if not secrets_backend_cls: + return None + print("secrets_backend_cls", secrets_backend_cls) + secrets_backend = import_string(secrets_backend_cls) + if secrets_backend: + try: + alternative_secrets_config_dict = json.loads( + super().get('secrets', 'backend_kwargs', fallback='{}') + ) + except json.JSONDecodeError: + alternative_secrets_config_dict = {} + return secrets_backend(**alternative_secrets_config_dict) Review comment: Hmmmm, not a huge fan of the duplication here and in `airflow/secrets/__init__.py`. Since this would only be called _after_ the class is constructed would we instead do ```suggestion @cached_property def _secrets_backend_client(self): if super().has_option('secrets', 'backend') is False: return None from airflow.secrets import secrets_backend_list return secrets_backend_list ``` (give or take). By delaying the import to inside the function no import loop is created. This probably also means adding a `get_config(section, key)` to airflow.secrets, and possibly changing the default implemenation from `NotImplementedError` to `return None`. Note: we should pass they key and section in directly, and let the secret backends convert that to a single parameter _if needed_ (for instance we could have /config/core/sql_alchemy_conn, rather than config/core__sql_alchemy_conn) ########## File path: airflow/providers/amazon/aws/secrets/secrets_manager.py ########## @@ -42,8 +42,11 @@ class SecretsManagerBackend(BaseSecretsBackend, LoggingMixin): For example, if secrets prefix is ``airflow/connections/smtp_default``, this would be accessible if you provide ``{"connections_prefix": "airflow/connections"}`` and request conn_id ``smtp_default``. - And if variables prefix is ``airflow/variables/hello``, this would be accessible + If variables prefix is ``airflow/variables/hello``, this would be accessible if you provide ``{"variables_prefix": "airflow/variables"}`` and request variable key ``hello``. + And if configurations_prefix is ``airflow/configurations/sql_alchemy_conn``, this would be accessible + if you provide ``{"configurations_prefix": "airflow/configurations"}`` and request variable Review comment: ```suggestion if you provide ``{"configurations_prefix": "airflow/config"}`` and request variable ``` ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org