Taragolis commented on code in PR #36449: URL: https://github.com/apache/airflow/pull/36449#discussion_r1456058203
########## airflow/providers/yandex/utils/user_agent.py: ########## @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import warnings + +from airflow.providers.yandex.utils.defaults import conn_type, hook_name + + +def provider_user_agent() -> str | None: + """Construct User-Agent from Airflow core & provider package versions.""" + from airflow import __version__ as airflow_version + from airflow.configuration import conf + from airflow.providers_manager import ProvidersManager + + try: + manager = ProvidersManager() + provider_name = manager.hooks[conn_type].package_name # type: ignore[union-attr] + provider = manager.providers[provider_name] + return " ".join( + ( + conf.get("yandex", "sdk_user_agent_prefix", fallback=""), + f"apache-airflow/{airflow_version}", + f"{provider_name}/{provider.version}", + ) + ).strip() + except KeyError: + warnings.warn(f"Hook '{hook_name}' info is not initialized in airflow.ProviderManager") Review Comment: ```suggestion warnings.warn( f"Hook '{hook_name}' info is not initialized in airflow.ProviderManager", UserWarning, stacklevel=2, ) ``` See: https://github.com/apache/airflow/pull/36831 ########## airflow/providers/yandex/hooks/yandex.py: ########## @@ -85,28 +91,6 @@ def get_connection_form_widgets(cls) -> dict[str, Any]: ), } - @classmethod - def provider_user_agent(cls) -> str | None: - """Construct User-Agent from Airflow core & provider package versions.""" - from airflow import __version__ as airflow_version - from airflow.configuration import conf - from airflow.providers_manager import ProvidersManager - - try: - manager = ProvidersManager() - provider_name = manager.hooks[cls.conn_type].package_name # type: ignore[union-attr] - provider = manager.providers[provider_name] - return " ".join( - ( - conf.get("yandex", "sdk_user_agent_prefix", fallback=""), - f"apache-airflow/{airflow_version}", - f"{provider_name}/{provider.version}", - ) - ).strip() - except KeyError: - warnings.warn(f"Hook '{cls.hook_name}' info is not initialized in airflow.ProviderManager") - return None - Review Comment: Public method. Should be deprecated first instead of remove ########## airflow/providers/yandex/secrets/secrets_manager.py: ########## @@ -0,0 +1,280 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Objects relating to sourcing secrets from Yandex Cloud Lockbox.""" +from __future__ import annotations + +import logging +from functools import cached_property +from typing import Any + +import yandex.cloud.lockbox.v1.payload_pb2 as payload_pb +import yandex.cloud.lockbox.v1.payload_service_pb2 as payload_service_pb +import yandex.cloud.lockbox.v1.payload_service_pb2_grpc as payload_service_pb_grpc +import yandex.cloud.lockbox.v1.secret_pb2 as secret_pb +import yandex.cloud.lockbox.v1.secret_service_pb2 as secret_service_pb +import yandex.cloud.lockbox.v1.secret_service_pb2_grpc as secret_service_pb_grpc +import yandexcloud + +from airflow.models import Connection +from airflow.providers.yandex.utils.credentials import get_credentials +from airflow.providers.yandex.utils.defaults import default_conn_name +from airflow.providers.yandex.utils.fields import get_field_from_extras +from airflow.providers.yandex.utils.user_agent import provider_user_agent +from airflow.secrets import BaseSecretsBackend + + +class LockboxSecretBackend(BaseSecretsBackend): + """ + Retrieves Connection or Variables or Configs from Yandex Lockbox. + + Configurable via ``airflow.cfg`` like so: + + .. code-block:: ini + + [secrets] + backend = airflow.providers.yandex.secrets.secrets_manager.SecretsManagerBackend + backend_kwargs = {"connections_prefix": "airflow/connections"} + + For example, when ``{"connections_prefix": "airflow/connections"}`` is set, if a secret is defined with + the path ``airflow/connections/smtp_default``, the connection with conn_id ``smtp_default`` would be + accessible. + + When ``{"variables_prefix": "airflow/variables"}`` is set, if a secret is defined with + the path ``airflow/variables/hello``, the variable with the name ``hello`` would be accessible. + + When ``{"config_prefix": "airflow/config"}`` is set, if a secret is defined with + the path ``airflow/config/sql_alchemy_conn``, the config with key ``sql_alchemy_conn`` would be + accessible. + + When the prefix is empty, keys will use the Lockbox Secrets without any prefix. + + .. code-block:: ini + + [secrets] + backend = airflow.providers.yandex.secrets.secrets_manager.SecretsManagerBackend + backend_kwargs = {"yc_connection_id": "<connection_ID>", "folder_id": "<folder_ID>"} + + You need to specify credentials or id of yandexcloud connection to connect to Yandex Lockbox with. + Credentials will be used with this priority: + + * OAuth Token + * Service Account JSON file + * Service Account JSON + * Yandex Cloud Connection + + If no credentials specified, default connection id will be used. + + Also, you need to specify the Yandex Cloud folder ID to search for Yandex Lockbox secrets in. + + :param yc_oauth_token: Specifies the user account OAuth token to connect to Yandex Lockbox with. + Looks like ``y3_xxxxx``. + :param yc_sa_key_json: Specifies the service account auth JSON. + Looks like ``{"id": "...", "service_account_id": "...", "private_key": "..."}``. + :param yc_sa_key_json_path: Specifies the service account auth JSON file path. + Looks like ``/home/airflow/authorized_key.json``. + File content looks like ``{"id": "...", "service_account_id": "...", "private_key": "..."}``. + :param yc_connection_id: Specifies the connection ID to connect to Yandex Lockbox with. + Default: "yandexcloud_default" + :param folder_id: Specifies the folder ID to search for Yandex Lockbox secrets in. + If set to None (null in JSON), requests will use the connection folder_id if specified. + :param connections_prefix: Specifies the prefix of the secret to read to get Connections. + If set to None (null in JSON), requests for connections will not be sent to Yandex Lockbox. + Default: "airflow/connections" + :param variables_prefix: Specifies the prefix of the secret to read to get Variables. + If set to None (null in JSON), requests for variables will not be sent to Yandex Lockbox. + Default: "airflow/variables" + :param config_prefix: Specifies the prefix of the secret to read to get Configurations. + If set to None (null in JSON), requests for variables will not be sent to Yandex Lockbox. + Default: "airflow/config" + :param sep: Specifies the separator used to concatenate secret_prefix and secret_id. + Default: "/" + :param endpoint: Specifies an API endpoint. + Leave blank to use default. + """ + + def __init__( + self, + yc_oauth_token: str | None = None, + yc_sa_key_json: str | None = None, + yc_sa_key_json_path: str | None = None, + yc_connection_id: str | None = None, + folder_id: str = "", + connections_prefix: str | None = "airflow/connections", + variables_prefix: str | None = "airflow/variables", + config_prefix: str | None = "airflow/config", + sep: str = "/", + endpoint: str | None = None, + ): + super().__init__() + + self.yc_oauth_token = yc_oauth_token + self.yc_sa_key_json = yc_sa_key_json + self.yc_sa_key_json_path = yc_sa_key_json_path + self.yc_connection_id = None + if not any([yc_oauth_token, yc_sa_key_json, yc_sa_key_json_path]): + self.yc_connection_id = yc_connection_id or default_conn_name + else: + assert ( + yc_connection_id is None + ), "yc_connection_id should not be used if other credentials are specified" + + self.folder_id = folder_id + self.connections_prefix = connections_prefix.rstrip(sep) if connections_prefix is not None else None + self.variables_prefix = variables_prefix.rstrip(sep) if variables_prefix is not None else None + self.config_prefix = config_prefix.rstrip(sep) if config_prefix is not None else None + self.sep = sep + self.endpoint = endpoint + + def get_conn_value(self, conn_id: str) -> str | None: + """ + Retrieve from Secrets Backend a string value representing the Connection object. + + :param conn_id: Connection ID + :return: Connection Value + """ + if self.connections_prefix is None: + return None + + if conn_id == self.yc_connection_id: + return None + + return self._get_secret_value(self.connections_prefix, conn_id) + + def get_variable(self, key: str) -> str | None: + """ + Return value for Airflow Variable. + + :param key: Variable Key + :return: Variable Value + """ + if self.variables_prefix is None: + return None + + return self._get_secret_value(self.variables_prefix, key) + + def get_config(self, key: str) -> str | None: + """ + Return value for Airflow Config Key. + + :param key: Config Key + :return: Config Value + """ + if self.config_prefix is None: + return None + + return self._get_secret_value(self.config_prefix, key) + + @cached_property + def _client(self): + """ + Create a Yandex Cloud SDK client. + + Lazy loading is used here + because we can't establish a Connection until all secrets backends have been initialized. + """ + if self.yc_connection_id: + self.yc_oauth_token = self._get_field("oauth") + self.yc_sa_key_json = self._get_field("service_account_json") + self.yc_sa_key_json_path = self._get_field("service_account_json_path") + self.folder_id = self.folder_id or self._get_field("folder_id") + + credentials = get_credentials( + oauth_token=self.yc_oauth_token, + service_account_json=self.yc_sa_key_json, + service_account_json_path=self.yc_sa_key_json_path, + ) + sdk_config = self._get_endpoint() + return yandexcloud.SDK(user_agent=provider_user_agent(), **credentials, **sdk_config).client + + def _get_endpoint(self) -> dict[str, str]: + sdk_config = {} + + if self.endpoint: + sdk_config["endpoint"] = self.endpoint + + return sdk_config + + @cached_property + def _connection(self) -> Connection | None: + if not self.yc_connection_id: + return None + + conn = Connection.get_connection_from_secrets(self.yc_connection_id) + logging.info("Using connection ID '%s' for task execution.", conn.conn_id) Review Comment: Same here with Root logger you either could create new logger in top of the module or inherit class from `airflow.utils.log.logging_mixin.LoggingMixin` ```python ... from airflow.utils.log.logging_mixin import LoggingMixin ... class LockboxSecretBackend(BaseSecretsBackend, LoggingMixin): ... @cached_property def _connection(self) -> Connection | None: ... self.log.info("Using connection ID '%s' for task execution.", conn.conn_id) ``` ########## airflow/providers/yandex/hooks/yandex.py: ########## @@ -117,7 +101,7 @@ def get_ui_field_behaviour(cls) -> dict[str, Any]: def __init__( self, - # Connection id is deprecated. Use yandex_conn_id instead + # connection_id is deprecated, use yandex_conn_id instead Review Comment: I guess better to better move this information into the docstring. https://github.com/apache/airflow/blob/b1ce06ec54803be2768bb0b68db4e21dab119674/airflow/providers/yandex/hooks/yandex.py#L36-L40 This would reflected into [Class API](https://airflow.apache.org/docs/apache-airflow-providers-yandex/stable/_api/airflow/providers/yandex/hooks/yandex/index.html) as well as into the end users / developers IDEs ########## airflow/providers/yandex/utils/credentials.py: ########## @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import json +import logging +from typing import Any + + +def get_credentials( + oauth_token: str | None = None, + service_account_json: str | None = None, + service_account_json_path: str | None = None, +) -> dict[str, Any]: + """ + Return credentials JSON for Yandex Cloud SDK based on credentials. + + Credentials will be used with this priority: + + * OAuth Token + * Service Account JSON file + * Service Account JSON + * Metadata Service + + :param oauth_token: OAuth Token + :param service_account_json: Service Account JSON key + :param service_account_json_path: Service Account JSON key file path + :return: Credentials JSON + """ + if oauth_token: + return {"token": oauth_token} + + service_account_key = get_service_account_key( + service_account_json=service_account_json, + service_account_json_path=service_account_json_path, + ) + if service_account_key: + return {"service_account_key": service_account_key} + + logging.info("using metadata service as credentials") Review Comment: There is no reason to use Root logger, you might just create new one and use it instead ```python import logging ... log = logging.getLogger(__name__) ... def get_credentials( oauth_token: str | None = None, service_account_json: str | None = None, service_account_json_path: str | None = None, ) -> dict[str, Any]: ... log.info("using metadata service as credentials") ``` ########## airflow/providers/yandex/utils/defaults.py: ########## @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +conn_name_attr = "yandex_conn_id" +default_conn_name = "yandexcloud_default" +conn_type = "yandexcloud" +hook_name = "w" Review Comment: `hook_name` it is refers to how Connection would be named in to the UI Connection type drop list ```suggestion hook_name = "Yandex Cloud" ```  ########## airflow/providers/yandex/secrets/secrets_manager.py: ########## @@ -0,0 +1,280 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Objects relating to sourcing secrets from Yandex Cloud Lockbox.""" +from __future__ import annotations + +import logging +from functools import cached_property +from typing import Any + +import yandex.cloud.lockbox.v1.payload_pb2 as payload_pb +import yandex.cloud.lockbox.v1.payload_service_pb2 as payload_service_pb +import yandex.cloud.lockbox.v1.payload_service_pb2_grpc as payload_service_pb_grpc +import yandex.cloud.lockbox.v1.secret_pb2 as secret_pb +import yandex.cloud.lockbox.v1.secret_service_pb2 as secret_service_pb +import yandex.cloud.lockbox.v1.secret_service_pb2_grpc as secret_service_pb_grpc +import yandexcloud + +from airflow.models import Connection +from airflow.providers.yandex.utils.credentials import get_credentials +from airflow.providers.yandex.utils.defaults import default_conn_name +from airflow.providers.yandex.utils.fields import get_field_from_extras +from airflow.providers.yandex.utils.user_agent import provider_user_agent +from airflow.secrets import BaseSecretsBackend + + +class LockboxSecretBackend(BaseSecretsBackend): + """ + Retrieves Connection or Variables or Configs from Yandex Lockbox. + + Configurable via ``airflow.cfg`` like so: + + .. code-block:: ini + + [secrets] + backend = airflow.providers.yandex.secrets.secrets_manager.SecretsManagerBackend Review Comment: I guess you forgot to change the class name here ```suggestion backend = airflow.providers.yandex.secrets.secrets_manager.LockboxSecretBackend ``` ########## airflow/providers/yandex/secrets/secrets_manager.py: ########## @@ -0,0 +1,280 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Objects relating to sourcing secrets from Yandex Cloud Lockbox.""" +from __future__ import annotations + +import logging +from functools import cached_property +from typing import Any + +import yandex.cloud.lockbox.v1.payload_pb2 as payload_pb +import yandex.cloud.lockbox.v1.payload_service_pb2 as payload_service_pb +import yandex.cloud.lockbox.v1.payload_service_pb2_grpc as payload_service_pb_grpc +import yandex.cloud.lockbox.v1.secret_pb2 as secret_pb +import yandex.cloud.lockbox.v1.secret_service_pb2 as secret_service_pb +import yandex.cloud.lockbox.v1.secret_service_pb2_grpc as secret_service_pb_grpc +import yandexcloud + +from airflow.models import Connection +from airflow.providers.yandex.utils.credentials import get_credentials +from airflow.providers.yandex.utils.defaults import default_conn_name +from airflow.providers.yandex.utils.fields import get_field_from_extras +from airflow.providers.yandex.utils.user_agent import provider_user_agent +from airflow.secrets import BaseSecretsBackend + + +class LockboxSecretBackend(BaseSecretsBackend): + """ + Retrieves Connection or Variables or Configs from Yandex Lockbox. + + Configurable via ``airflow.cfg`` like so: + + .. code-block:: ini + + [secrets] + backend = airflow.providers.yandex.secrets.secrets_manager.SecretsManagerBackend + backend_kwargs = {"connections_prefix": "airflow/connections"} + + For example, when ``{"connections_prefix": "airflow/connections"}`` is set, if a secret is defined with + the path ``airflow/connections/smtp_default``, the connection with conn_id ``smtp_default`` would be + accessible. + + When ``{"variables_prefix": "airflow/variables"}`` is set, if a secret is defined with + the path ``airflow/variables/hello``, the variable with the name ``hello`` would be accessible. + + When ``{"config_prefix": "airflow/config"}`` is set, if a secret is defined with + the path ``airflow/config/sql_alchemy_conn``, the config with key ``sql_alchemy_conn`` would be + accessible. + + When the prefix is empty, keys will use the Lockbox Secrets without any prefix. + + .. code-block:: ini + + [secrets] + backend = airflow.providers.yandex.secrets.secrets_manager.SecretsManagerBackend + backend_kwargs = {"yc_connection_id": "<connection_ID>", "folder_id": "<folder_ID>"} + + You need to specify credentials or id of yandexcloud connection to connect to Yandex Lockbox with. + Credentials will be used with this priority: + + * OAuth Token + * Service Account JSON file + * Service Account JSON + * Yandex Cloud Connection + + If no credentials specified, default connection id will be used. + + Also, you need to specify the Yandex Cloud folder ID to search for Yandex Lockbox secrets in. + + :param yc_oauth_token: Specifies the user account OAuth token to connect to Yandex Lockbox with. + Looks like ``y3_xxxxx``. + :param yc_sa_key_json: Specifies the service account auth JSON. + Looks like ``{"id": "...", "service_account_id": "...", "private_key": "..."}``. + :param yc_sa_key_json_path: Specifies the service account auth JSON file path. + Looks like ``/home/airflow/authorized_key.json``. + File content looks like ``{"id": "...", "service_account_id": "...", "private_key": "..."}``. + :param yc_connection_id: Specifies the connection ID to connect to Yandex Lockbox with. + Default: "yandexcloud_default" + :param folder_id: Specifies the folder ID to search for Yandex Lockbox secrets in. + If set to None (null in JSON), requests will use the connection folder_id if specified. + :param connections_prefix: Specifies the prefix of the secret to read to get Connections. + If set to None (null in JSON), requests for connections will not be sent to Yandex Lockbox. + Default: "airflow/connections" + :param variables_prefix: Specifies the prefix of the secret to read to get Variables. + If set to None (null in JSON), requests for variables will not be sent to Yandex Lockbox. + Default: "airflow/variables" + :param config_prefix: Specifies the prefix of the secret to read to get Configurations. + If set to None (null in JSON), requests for variables will not be sent to Yandex Lockbox. + Default: "airflow/config" + :param sep: Specifies the separator used to concatenate secret_prefix and secret_id. + Default: "/" + :param endpoint: Specifies an API endpoint. + Leave blank to use default. + """ + + def __init__( + self, + yc_oauth_token: str | None = None, + yc_sa_key_json: str | None = None, + yc_sa_key_json_path: str | None = None, + yc_connection_id: str | None = None, Review Comment: I would rather avoid to use any Airflow Connections into the Secrets Backend. 1. **Chicken and Egg problem**: Very easy to get into the situation that Backend not yet initialised 2. **Security/Maintainability**: Authenticated UI users who has access to the Connections (Metadata) might change creds and make Deployment Managers surprised. See: [Airflow Security Model](https://airflow.apache.org/docs/apache-airflow/stable/security/security_model.html) ########## airflow/providers/yandex/utils/fields.py: ########## @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from typing import Any + + +def get_field_from_extras(extras: dict[str, Any], field_name: str, default: Any = None) -> Any: + """ + Get field from extras, first checking short name, then for backcompat checking for prefixed name. + + :param extras: Dictionary with extras keys + :param field_name: Field name to get from extras + :param default: Default value if field not found + :return: Field value or default if not found + """ + backcompat_prefix = "extra__yandexcloud__" + if field_name.startswith("extra__"): + raise ValueError( + f"Got prefixed name {field_name}; please remove the '{backcompat_prefix}' prefix " + "when using this method." Review Comment: ```suggestion "when using this function." ``` ########## airflow/providers/yandex/secrets/secrets_manager.py: ########## @@ -0,0 +1,280 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Objects relating to sourcing secrets from Yandex Cloud Lockbox.""" +from __future__ import annotations + +import logging +from functools import cached_property +from typing import Any + +import yandex.cloud.lockbox.v1.payload_pb2 as payload_pb +import yandex.cloud.lockbox.v1.payload_service_pb2 as payload_service_pb +import yandex.cloud.lockbox.v1.payload_service_pb2_grpc as payload_service_pb_grpc +import yandex.cloud.lockbox.v1.secret_pb2 as secret_pb +import yandex.cloud.lockbox.v1.secret_service_pb2 as secret_service_pb +import yandex.cloud.lockbox.v1.secret_service_pb2_grpc as secret_service_pb_grpc +import yandexcloud + +from airflow.models import Connection +from airflow.providers.yandex.utils.credentials import get_credentials +from airflow.providers.yandex.utils.defaults import default_conn_name +from airflow.providers.yandex.utils.fields import get_field_from_extras +from airflow.providers.yandex.utils.user_agent import provider_user_agent +from airflow.secrets import BaseSecretsBackend + + +class LockboxSecretBackend(BaseSecretsBackend): + """ + Retrieves Connection or Variables or Configs from Yandex Lockbox. + + Configurable via ``airflow.cfg`` like so: + + .. code-block:: ini + + [secrets] + backend = airflow.providers.yandex.secrets.secrets_manager.SecretsManagerBackend + backend_kwargs = {"connections_prefix": "airflow/connections"} + + For example, when ``{"connections_prefix": "airflow/connections"}`` is set, if a secret is defined with + the path ``airflow/connections/smtp_default``, the connection with conn_id ``smtp_default`` would be + accessible. + + When ``{"variables_prefix": "airflow/variables"}`` is set, if a secret is defined with + the path ``airflow/variables/hello``, the variable with the name ``hello`` would be accessible. + + When ``{"config_prefix": "airflow/config"}`` is set, if a secret is defined with + the path ``airflow/config/sql_alchemy_conn``, the config with key ``sql_alchemy_conn`` would be + accessible. + + When the prefix is empty, keys will use the Lockbox Secrets without any prefix. + + .. code-block:: ini + + [secrets] + backend = airflow.providers.yandex.secrets.secrets_manager.SecretsManagerBackend + backend_kwargs = {"yc_connection_id": "<connection_ID>", "folder_id": "<folder_ID>"} + + You need to specify credentials or id of yandexcloud connection to connect to Yandex Lockbox with. + Credentials will be used with this priority: + + * OAuth Token + * Service Account JSON file + * Service Account JSON + * Yandex Cloud Connection + + If no credentials specified, default connection id will be used. + + Also, you need to specify the Yandex Cloud folder ID to search for Yandex Lockbox secrets in. + + :param yc_oauth_token: Specifies the user account OAuth token to connect to Yandex Lockbox with. + Looks like ``y3_xxxxx``. + :param yc_sa_key_json: Specifies the service account auth JSON. + Looks like ``{"id": "...", "service_account_id": "...", "private_key": "..."}``. + :param yc_sa_key_json_path: Specifies the service account auth JSON file path. + Looks like ``/home/airflow/authorized_key.json``. + File content looks like ``{"id": "...", "service_account_id": "...", "private_key": "..."}``. + :param yc_connection_id: Specifies the connection ID to connect to Yandex Lockbox with. + Default: "yandexcloud_default" + :param folder_id: Specifies the folder ID to search for Yandex Lockbox secrets in. + If set to None (null in JSON), requests will use the connection folder_id if specified. + :param connections_prefix: Specifies the prefix of the secret to read to get Connections. + If set to None (null in JSON), requests for connections will not be sent to Yandex Lockbox. + Default: "airflow/connections" + :param variables_prefix: Specifies the prefix of the secret to read to get Variables. + If set to None (null in JSON), requests for variables will not be sent to Yandex Lockbox. + Default: "airflow/variables" + :param config_prefix: Specifies the prefix of the secret to read to get Configurations. + If set to None (null in JSON), requests for variables will not be sent to Yandex Lockbox. + Default: "airflow/config" + :param sep: Specifies the separator used to concatenate secret_prefix and secret_id. + Default: "/" + :param endpoint: Specifies an API endpoint. + Leave blank to use default. + """ + + def __init__( + self, + yc_oauth_token: str | None = None, + yc_sa_key_json: str | None = None, Review Comment: What if make this parameter either python dictionary or string? In this case users might avoid escaping this parameter in `[secrets] backend_kwargs` https://github.com/apache/airflow/blob/b1ce06ec54803be2768bb0b68db4e21dab119674/docs/apache-airflow-providers-yandex/secrets-backends/yandex-cloud-lockbox-secret-backend.rst?plain=1#L126-L131 ########## airflow/providers/yandex/secrets/secrets_manager.py: ########## @@ -0,0 +1,280 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Objects relating to sourcing secrets from Yandex Cloud Lockbox.""" +from __future__ import annotations + +import logging +from functools import cached_property +from typing import Any + +import yandex.cloud.lockbox.v1.payload_pb2 as payload_pb +import yandex.cloud.lockbox.v1.payload_service_pb2 as payload_service_pb +import yandex.cloud.lockbox.v1.payload_service_pb2_grpc as payload_service_pb_grpc +import yandex.cloud.lockbox.v1.secret_pb2 as secret_pb +import yandex.cloud.lockbox.v1.secret_service_pb2 as secret_service_pb +import yandex.cloud.lockbox.v1.secret_service_pb2_grpc as secret_service_pb_grpc +import yandexcloud + +from airflow.models import Connection +from airflow.providers.yandex.utils.credentials import get_credentials +from airflow.providers.yandex.utils.defaults import default_conn_name +from airflow.providers.yandex.utils.fields import get_field_from_extras +from airflow.providers.yandex.utils.user_agent import provider_user_agent +from airflow.secrets import BaseSecretsBackend + + +class LockboxSecretBackend(BaseSecretsBackend): + """ + Retrieves Connection or Variables or Configs from Yandex Lockbox. + + Configurable via ``airflow.cfg`` like so: + + .. code-block:: ini + + [secrets] + backend = airflow.providers.yandex.secrets.secrets_manager.SecretsManagerBackend + backend_kwargs = {"connections_prefix": "airflow/connections"} + + For example, when ``{"connections_prefix": "airflow/connections"}`` is set, if a secret is defined with + the path ``airflow/connections/smtp_default``, the connection with conn_id ``smtp_default`` would be + accessible. + + When ``{"variables_prefix": "airflow/variables"}`` is set, if a secret is defined with + the path ``airflow/variables/hello``, the variable with the name ``hello`` would be accessible. + + When ``{"config_prefix": "airflow/config"}`` is set, if a secret is defined with + the path ``airflow/config/sql_alchemy_conn``, the config with key ``sql_alchemy_conn`` would be + accessible. + + When the prefix is empty, keys will use the Lockbox Secrets without any prefix. + + .. code-block:: ini + + [secrets] + backend = airflow.providers.yandex.secrets.secrets_manager.SecretsManagerBackend Review Comment: ```suggestion backend = airflow.providers.yandex.secrets.secrets_manager.LockboxSecretBackend ``` ########## airflow/providers/yandex/secrets/secrets_manager.py: ########## Review Comment: If you wanted you also might change the name of the module to more Yandex specific. I guess this file originally based on `airflow/providers/amazon/aws/secrets/secrets_manager.py` where secrets_manager is stands for AWS Secrets Manager service -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
