This is an automated email from the ASF dual-hosted git repository. husseinawala pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push: new a9a6976dd2 docs: yandex provider grammatical improvements (#38589) a9a6976dd2 is described below commit a9a6976dd234ff84af7db4253fe700b9ccc6aa4a Author: Vadim Vladimirov <47372390+impressionableracc...@users.noreply.github.com> AuthorDate: Fri Mar 29 00:32:23 2024 +0300 docs: yandex provider grammatical improvements (#38589) * tests: fix IDE problems * docs: airflow provider grammatical improvements --- airflow/providers/yandex/hooks/yandex.py | 8 +- .../yandex/operators/yandexcloud_dataproc.py | 2 +- airflow/providers/yandex/secrets/lockbox.py | 74 +++++---- .../connections/yandexcloud.rst | 52 +++--- docs/apache-airflow-providers-yandex/operators.rst | 16 +- .../yandex-cloud-lockbox-secret-backend.rst | 183 +++++++++++---------- tests/providers/yandex/hooks/test_yandex.py | 6 +- .../yandex/hooks/test_yandexcloud_dataproc.py | 4 +- .../yandex/operators/test_yandexcloud_dataproc.py | 10 +- tests/providers/yandex/secrets/test_lockbox.py | 50 +++--- tests/providers/yandex/utils/test_credentials.py | 2 +- tests/providers/yandex/utils/test_fields.py | 12 +- 12 files changed, 217 insertions(+), 202 deletions(-) diff --git a/airflow/providers/yandex/hooks/yandex.py b/airflow/providers/yandex/hooks/yandex.py index 251a47b7b8..5ad7ce2895 100644 --- a/airflow/providers/yandex/hooks/yandex.py +++ b/airflow/providers/yandex/hooks/yandex.py @@ -79,14 +79,14 @@ class YandexCloudBaseHook(BaseHook): "folder_id": StringField( lazy_gettext("Default folder ID"), widget=BS3TextFieldWidget(), - description="Optional. This folder will be used " - "to create all new clusters and nodes by default", + description="Optional. " + "If specified, this ID will be used by default when creating nodes and clusters.", ), "public_ssh_key": StringField( lazy_gettext("Public SSH key"), widget=BS3TextFieldWidget(), - description="Optional. This key will be placed to all created Compute nodes " - "to let you have a root shell there", + description="Optional. The key will be placed to all created Compute nodes, " + "allowing you to have a root shell there.", ), "endpoint": StringField( lazy_gettext("API endpoint"), diff --git a/airflow/providers/yandex/operators/yandexcloud_dataproc.py b/airflow/providers/yandex/operators/yandexcloud_dataproc.py index 0489a2fd80..49bf136c3f 100644 --- a/airflow/providers/yandex/operators/yandexcloud_dataproc.py +++ b/airflow/providers/yandex/operators/yandexcloud_dataproc.py @@ -97,7 +97,7 @@ class DataprocCreateClusterOperator(BaseOperator): :param initialization_actions: Set of init-actions to run when cluster starts. Docs: https://cloud.yandex.com/docs/data-proc/concepts/init-action :param labels: Cluster labels as key:value pairs. No more than 64 per resource. - Docs: https://cloud.yandex.ru/docs/resource-manager/concepts/labels + Docs: https://cloud.yandex.com/docs/resource-manager/concepts/labels """ def __init__( diff --git a/airflow/providers/yandex/secrets/lockbox.py b/airflow/providers/yandex/secrets/lockbox.py index 16a28a3917..68879b446e 100644 --- a/airflow/providers/yandex/secrets/lockbox.py +++ b/airflow/providers/yandex/secrets/lockbox.py @@ -40,7 +40,7 @@ from airflow.utils.log.logging_mixin import LoggingMixin class LockboxSecretBackend(BaseSecretsBackend, LoggingMixin): """ - Retrieves Connection or Variables or Configs from Yandex Lockbox. + Retrieves connections or variables or configs from Yandex Lockbox. Configurable via ``airflow.cfg`` like so: @@ -61,7 +61,7 @@ class LockboxSecretBackend(BaseSecretsBackend, LoggingMixin): the path ``airflow/config/sql_alchemy_conn``, the config with key ``sql_alchemy_conn`` would be accessible. - When the prefix is empty, keys will use the Lockbox Secrets without any prefix. + If the prefix is empty, the requests will not be sent to Yandex Lockbox. .. code-block:: ini @@ -69,43 +69,49 @@ class LockboxSecretBackend(BaseSecretsBackend, LoggingMixin): backend = airflow.providers.yandex.secrets.lockbox.LockboxSecretBackend backend_kwargs = {"yc_connection_id": "<connection_ID>", "folder_id": "<folder_ID>"} - You need to specify credentials or id of yandexcloud connection to connect to Yandex Lockbox with. - Credentials will be used with this priority: + You need to specify credentials or the ID of the ``yandexcloud`` connection to connect to Yandex Lockbox. + The credentials will be used with the following priority: - * OAuth Token - * Service Account JSON file - * Service Account JSON - * Yandex Cloud Connection + * OAuth token + * Service account key in JSON from file + * Service account key in JSON + * Yandex Cloud connection - If no credentials specified, default connection id will be used. + If you do not specify any credentials, + the system will use the default connection ID:``yandexcloud_default``. Also, you need to specify the Yandex Cloud folder ID to search for Yandex Lockbox secrets in. - - :param yc_oauth_token: Specifies the user account OAuth token to connect to Yandex Lockbox with. - Looks like ``y3_xxxxx``. - :param yc_sa_key_json: Specifies the service account auth JSON. - Looks like ``{"id": "...", "service_account_id": "...", "private_key": "..."}``. - :param yc_sa_key_json_path: Specifies the service account auth JSON file path. - Looks like ``/home/airflow/authorized_key.json``. - File content looks like ``{"id": "...", "service_account_id": "...", "private_key": "..."}``. - :param yc_connection_id: Specifies the connection ID to connect to Yandex Lockbox with. - Default: "yandexcloud_default" + If you do not specify folder ID, the requests will use the connection ``folder_id`` if specified. + + :param yc_oauth_token: Specifies the user account OAuth token to connect to Yandex Lockbox. + The parameter value should look like ``y3_xx123``. + :param yc_sa_key_json: Specifies the service account key in JSON. + The parameter value should look like + ``{"id": "...", "service_account_id": "...", "private_key": "..."}``. + :param yc_sa_key_json_path: Specifies the service account key in JSON file path. + The parameter value should look like ``/home/airflow/authorized_key.json``, + while the file content should have the following format: + ``{"id": "...", "service_account_id": "...", "private_key": "..."}``. + :param yc_connection_id: Specifies the connection ID to connect to Yandex Lockbox. + The default value is ``yandexcloud_default``. :param folder_id: Specifies the folder ID to search for Yandex Lockbox secrets in. - If set to None (null in JSON), requests will use the connection folder_id if specified. - :param connections_prefix: Specifies the prefix of the secret to read to get Connections. - If set to None (null in JSON), requests for connections will not be sent to Yandex Lockbox. - Default: "airflow/connections" - :param variables_prefix: Specifies the prefix of the secret to read to get Variables. - If set to None (null in JSON), requests for variables will not be sent to Yandex Lockbox. - Default: "airflow/variables" - :param config_prefix: Specifies the prefix of the secret to read to get Configurations. - If set to None (null in JSON), requests for variables will not be sent to Yandex Lockbox. - Default: "airflow/config" - :param sep: Specifies the separator used to concatenate secret_prefix and secret_id. - Default: "/" - :param endpoint: Specifies an API endpoint. - If set to None (null in JSON), requests will use the connection endpoint, if specified, - or the default endpoint. + If set to ``None`` (``null`` in JSON), + the requests will use the connection ``folder_id``, if specified. + :param connections_prefix: Specifies the prefix of the secret to read to get connections. + If set to ``None`` (``null`` in JSON), + the requests for connections will not be sent to Yandex Lockbox. + The default value is ``airflow/connections``. + :param variables_prefix: Specifies the prefix of the secret to read to get variables. + If set to ``None`` (``null`` in JSON), the requests for variables will not be sent to Yandex Lockbox. + The default value is ``airflow/variables``. + :param config_prefix: Specifies the prefix of the secret to read to get configurations. + If set to ``None`` (``null`` in JSON), the requests for variables will not be sent to Yandex Lockbox. + The default value is ``airflow/config``. + :param sep: Specifies the separator to concatenate ``secret_prefix`` and ``secret_id``. + The default value is ``/``. + :param endpoint: Specifies the API endpoint. + If set to ``None`` (``null`` in JSON), the requests will use the connection endpoint, if specified; + otherwise, they will use the default endpoint. """ def __init__( diff --git a/docs/apache-airflow-providers-yandex/connections/yandexcloud.rst b/docs/apache-airflow-providers-yandex/connections/yandexcloud.rst index c67676d828..b1d8b4074c 100644 --- a/docs/apache-airflow-providers-yandex/connections/yandexcloud.rst +++ b/docs/apache-airflow-providers-yandex/connections/yandexcloud.rst @@ -36,60 +36,60 @@ Service account auth JSON file path Example: ``/home/airflow/authorized_key.json`` OAuth Token - OAuth token as a string. + User account OAuth token as a string. - Example: ``y3_Vdheub7w9bIut67GHeL345gfb5GAnd3dZnf08FRbvjeUFvetYiohGvc`` + Example: ``y3_Vd3eub7w9bIut67GHeL345gfb5GAnd3dZnf08FR1vjeUFve7Yi8hGvc`` SSH public key (optional) - The key will be placed to all created Compute nodes, allowing to have a root shell there. + The key will be placed to all created Compute nodes, allowing you to have a root shell there. Folder ID (optional) - Folder is a entity to separate different projects within the cloud. + A folder is an entity to separate different projects within the cloud. - If specified, this ID will be used by default during creation of nodes and clusters. + If specified, this ID will be used by default when creating nodes and clusters. - See https://cloud.yandex.com/docs/resource-manager/operations/folder/get-id for details + See `this guide <https://cloud.yandex.com/docs/resource-manager/operations/folder/get-id>`__ for details. Endpoint (optional) - Set API endpoint + Use this setting to configure your API endpoint. - See https://github.com/yandex-cloud/python-sdk for default + Leave blank to use default `endpoints <https://cloud.yandex.com/docs/api-design-guide/concepts/endpoints>`__. Default Connection IDs ---------------------- -All hooks and operators related to Yandex.Cloud use ``yandexcloud_default`` connection by default. +All hooks and operators related to Yandex.Cloud use the ``yandexcloud_default`` connection by default. Authenticating to Yandex.Cloud ------------------------------ -Using Authorized keys for authorization as service account -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Using authorized keys to authorize as a service account +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Before you start, make sure you have `created <https://cloud.yandex.com/en/docs/iam/operations/sa/create>`__ -a Yandex Cloud `Service Account <https://cloud.yandex.com/en/docs/iam/concepts/users/service-accounts>`__ -with the permissions ``lockbox.viewer`` and ``lockbox.payloadViewer``. +Before you start, make sure you have `created <https://cloud.yandex.com/docs/iam/operations/sa/create>`__ +a Yandex Cloud `service account <https://cloud.yandex.com/docs/iam/concepts/users/service-accounts>`__. -First, you need to create `Authorized key <https://cloud.yandex.com/en/docs/iam/concepts/authorization/key>`__ -for your service account and save the generated JSON file with public and private key parts. +First, you need to create an `authorized key <https://cloud.yandex.com/docs/iam/concepts/authorization/key>`__ +for your service account and save the generated JSON file with both public and private key parts. -Then you need to specify the key in the ``Service account auth JSON`` field. +Then, you need to specify the key in the ``Service account auth JSON`` field. -Alternatively, you can specify the path to JSON file in the ``Service account auth JSON file path`` field. +Alternatively, you can specify the path to the JSON file in the ``Service account auth JSON file path`` field. -Using OAuth token for authorization as users account -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Using an OAuth token to authorize as a user account +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -First, you need to create `OAuth token <https://cloud.yandex.com/en/docs/iam/concepts/authorization/oauth-token>`__ for user account. -It will looks like ``y3_Vdheub7w9bIut67GHeL345gfb5GAnd3dZnf08FRbvjeUFvetYiohGvc``. +First, you need to create +an `OAuth token <https://cloud.yandex.com/docs/iam/concepts/authorization/oauth-token>`__ for your user account. +Your token will look like this: ``y3_Vd3eub7w9bIut67GHeL345gfb5GAnd3dZnf08FR1vjeUFve7Yi8hGvc``. -Then you need to specify token in the ``OAuth Token`` field. +Then you need to specify your token in the ``OAuth Token`` field. Using metadata service ~~~~~~~~~~~~~~~~~~~~~~ -If no credentials are specified, the connection will attempt to use -the `metadata service <https://cloud.yandex.com/en/docs/compute/concepts/vm-metadata>`__ for authentication. +If you do not specify any credentials, the connection will attempt to use +the `metadata service <https://cloud.yandex.com/docs/compute/concepts/vm-metadata>`__ for authentication. -To do this, you need to `link <https://cloud.yandex.ru/en/docs/compute/operations/vm-connect/auth-inside-vm>`__ +To do this, you need to `link <https://cloud.yandex.com/docs/compute/operations/vm-connect/auth-inside-vm>`__ your service account with your VM. diff --git a/docs/apache-airflow-providers-yandex/operators.rst b/docs/apache-airflow-providers-yandex/operators.rst index fd696e96d0..2bb08d859d 100644 --- a/docs/apache-airflow-providers-yandex/operators.rst +++ b/docs/apache-airflow-providers-yandex/operators.rst @@ -19,15 +19,19 @@ Yandex.Cloud Data Proc Operators ================================ -The `Yandex.Cloud Data Proc <https://cloud.yandex.com/services/data-proc>`__ is a service that helps to deploy Apache Hadoop®* and Apache Spark™ clusters in the Yandex.Cloud infrastructure. +`Yandex Data Proc <https://cloud.yandex.com/services/data-proc>`__ is a service +that helps you deploy Apache Hadoop®* and Apache Spark™ clusters in the Yandex Cloud infrastructure. -You can control the cluster size, node capacity, and set of Apache® services -(Spark, HDFS, YARN, Hive, HBase, Oozie, Sqoop, Flume, Tez, Zeppelin). +With Data Proc, you can manage the cluster size and node capacity, +as well as work with various Apache® services, +such as Spark, HDFS, YARN, Hive, HBase, Oozie, Sqoop, Flume, Tez, and Zeppelin. Apache Hadoop is used for storing and analyzing structured and unstructured big data. -Apache Spark is a tool for quick data-processing that can be integrated with Apache Hadoop as well as with other storage systems. +Apache Spark is a tool for quick data processing +that can be integrated with Apache Hadoop and other storage systems. Using the operators -^^^^^^^^^^^^^^^^^^^^^ -See the usage examples in `example DAGs <https://github.com/apache/airflow/tree/providers-yandex/|version|/tests/system/providers/yandex/example_yandexcloud_dataproc.py>`_ +^^^^^^^^^^^^^^^^^^^ +To learn how to use Data Proc operators, +see `example DAGs <https://github.com/apache/airflow/tree/providers-yandex/|version|/tests/system/providers/yandex/example_yandexcloud_dataproc.py>`_. diff --git a/docs/apache-airflow-providers-yandex/secrets-backends/yandex-cloud-lockbox-secret-backend.rst b/docs/apache-airflow-providers-yandex/secrets-backends/yandex-cloud-lockbox-secret-backend.rst index e0d3625df4..f30346b24d 100644 --- a/docs/apache-airflow-providers-yandex/secrets-backends/yandex-cloud-lockbox-secret-backend.rst +++ b/docs/apache-airflow-providers-yandex/secrets-backends/yandex-cloud-lockbox-secret-backend.rst @@ -19,11 +19,11 @@ Yandex.Cloud Lockbox Secret Backend =================================== -This topic describes how to configure Apache Airflow to use `Yandex Lockbox <https://cloud.yandex.com/en/docs/lockbox>`__ +This topic describes how to configure Apache Airflow to use `Yandex Lockbox <https://cloud.yandex.com/docs/lockbox>`__ as a secret backend and how to manage secrets. -Before you begin ----------------- +Getting started +--------------- Before you start, make sure you have installed the ``yandex`` provider in your Apache Airflow installation: @@ -34,9 +34,9 @@ Before you start, make sure you have installed the ``yandex`` provider in your A Enabling the Yandex Lockbox secret backend ------------------------------------------ -To enable Yandex Lockbox as secrets backend, +To enable Yandex Lockbox as a secret backend, specify :py:class:`~airflow.providers.yandex.secrets.lockbox.LockboxSecretBackend` -as the ``backend`` in ``[secrets]`` section of ``airflow.cfg`` file. +as your ``backend`` in the ``[secrets]`` section of the ``airflow.cfg`` file. Here is a sample configuration: @@ -51,7 +51,8 @@ You can also set this with an environment variable: export AIRFLOW__SECRETS__BACKEND=airflow.providers.yandex.secrets.lockbox.LockboxSecretBackend -You can verify the correct setting of the configuration options by using the ``airflow config get-value`` command: +You can verify whether the configuration options have been set up correctly +using the ``airflow config get-value`` command: .. code-block:: console @@ -61,24 +62,24 @@ You can verify the correct setting of the configuration options by using the ``a Backend parameters ------------------ -The next step is to configure backend parameters using the ``backend_kwargs`` options. -You can pass the following parameters: +The next step is to configure backend parameters using the ``backend_kwargs`` options +that allow you to provide the following parameters: -* ``yc_oauth_token``: Specifies the user account OAuth token to connect to Yandex Lockbox with. Looks like ``y3_xxxxx``. -* ``yc_sa_key_json``: Specifies the service account auth JSON. Looks like ``{"id": "...", "service_account_id": "...", "private_key": "..."}``. -* ``yc_sa_key_json_path``: Specifies the service account auth JSON file path. Looks like ``/home/airflow/authorized_key.json``. File content looks like ``{"id": "...", "service_account_id": "...", "private_key": "..."}``. -* ``yc_connection_id``: Specifies the connection ID to connect to Yandex Lockbox with. Default: "yandexcloud_default" -* ``folder_id``: Specifies the folder ID to search for Yandex Lockbox secrets in. If set to None (null in JSON), requests will use the connection folder_id if specified. -* ``connections_prefix``: Specifies the prefix of the secret to read to get Connections. If set to None (null in JSON), requests for connections will not be sent to Yandex Lockbox. Default: "airflow/connections" -* ``variables_prefix``: Specifies the prefix of the secret to read to get Variables. If set to None (null in JSON), requests for variables will not be sent to Yandex Lockbox. Default: "airflow/variables" -* ``config_prefix``: Specifies the prefix of the secret to read to get Configurations. If set to None (null in JSON), requests for variables will not be sent to Yandex Lockbox. Default: "airflow/config" -* ``sep``: Specifies the separator used to concatenate secret_prefix and secret_id. Default: "/" -* ``endpoint``: Specifies an API endpoint. If set to None (null in JSON), requests will use the connection endpoint, if specified, or the default endpoint. +* ``yc_oauth_token``: Specifies the user account OAuth token to connect to Yandex Lockbox. The parameter value should look like ``y3_xx123``. +* ``yc_sa_key_json``: Specifies the service account key in JSON. The parameter value should look like ``{"id": "...", "service_account_id": "...", "private_key": "..."}``. +* ``yc_sa_key_json_path``: Specifies the service account key in JSON file path. The parameter value should look like ``/home/airflow/authorized_key.json``, while the file content should have the following format: ``{"id": "...", "service_account_id": "...", "private_key": "..."}``. +* ``yc_connection_id``: Specifies the connection ID to connect to Yandex Lockbox. The default value is ``yandexcloud_default``. +* ``folder_id``: Specifies the folder ID to search for Yandex Lockbox secrets in. If set to ``None`` (``null`` in JSON), the requests will use the connection ``folder_id``, if specified. +* ``connections_prefix``: Specifies the prefix of the secret to read to get connections. If set to ``None`` (``null`` in JSON), the requests for connections will not be sent to Yandex Lockbox. The default value is ``airflow/connections``. +* ``variables_prefix``: Specifies the prefix of the secret to read to get variables. If set to ``None`` (``null`` in JSON), the requests for variables will not be sent to Yandex Lockbox. The default value is ``airflow/variables``. +* ``config_prefix``: Specifies the prefix of the secret to read to get configurations. If set to ``None`` (``null`` in JSON), the requests for variables will not be sent to Yandex Lockbox. The default value is ``airflow/config``. +* ``sep``: Specifies the separator to concatenate ``secret_prefix`` and ``secret_id``. The default value is ``/``. +* ``endpoint``: Specifies the API endpoint. If set to ``None`` (``null`` in JSON), the requests will use the connection endpoint, if specified; otherwise, they will use the default endpoint. -All options should be passed as a JSON dictionary. +Make sure to provide all options as a JSON dictionary. -For example, if you want to set parameter ``connections_prefix`` to ``"example-connections-prefix"`` -and parameter ``variables_prefix`` to ``"example-variables-prefix"``, +For example, if you want to set ``connections_prefix`` to ``"example-connections-prefix"`` +and ``variables_prefix`` to ``"example-variables-prefix"``, your configuration file should look like this: .. code-block:: ini @@ -87,93 +88,96 @@ your configuration file should look like this: backend = airflow.providers.yandex.secrets.lockbox.LockboxSecretBackend backend_kwargs = {"connections_prefix": "example-connections-prefix", "variables_prefix": "example-variables-prefix"} -Set-up credentials ------------------- +Setting up credentials +---------------------- + +You need to specify credentials or the ID of the ``yandexcloud`` connection to connect to Yandex Lockbox. -You need to specify credentials or id of yandexcloud connection to connect to Yandex Lockbox with. -Credentials will be used with this priority: +The credentials will be used with the following priority: -* OAuth Token -* Service Account JSON file -* Service Account JSON -* Yandex Cloud Connection +* OAuth token +* Service account key in JSON from file +* Service account key in JSON +* Yandex Cloud connection -If no credentials specified, default connection id ``yandexcloud_default`` will be used. +If you do not specify any credentials, the system will use the default connection ID: ``yandexcloud_default``. -Using OAuth token for authorization as users account -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Using an OAuth token to authorize as a user account +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -First, you need to create `OAuth token <https://cloud.yandex.com/en/docs/iam/concepts/authorization/oauth-token>`__ for user account. -It will looks like ``y3_Vdheub7w9bIut67GHeL345gfb5GAnd3dZnf08FRbvjeUFvetYiohGvc``. +First, you need to create +an `OAuth token <https://cloud.yandex.com/docs/iam/concepts/authorization/oauth-token>`__ for your user account. +Your token will look like this: ``y3_Vd3eub7w9bIut67GHeL345gfb5GAnd3dZnf08FR1vjeUFve7Yi8hGvc``. -Then you need to specify the ``folder_id`` and token in the ``backend_kwargs``: +Then, you need to specify the ``folder_id`` and your token in ``backend_kwargs``: .. code-block:: ini [secrets] - backend_kwargs = {"folder_id": "b1g66mft1vopnevbn57j", "yc_oauth_token": "y3_Vdheub7w9bIut67GHeL345gfb5GAnd3dZnf08FRbvjeUFvetYiohGvc"} + backend_kwargs = {"folder_id": "b1g66mft1vo1n4vbn57j", "yc_oauth_token": "y3_Vd3eub7w9bIut67GHeL345gfb5GAnd3dZnf08FR1vjeUFve7Yi8hGvc"} -Using Authorized keys for authorization as service account -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Using authorized keys to authorize as a service account +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Before you start, make sure you have `created <https://cloud.yandex.com/en/docs/iam/operations/sa/create>`__ -a Yandex Cloud `Service Account <https://cloud.yandex.com/en/docs/iam/concepts/users/service-accounts>`__ -with the permissions ``lockbox.viewer`` and ``lockbox.payloadViewer``. +Before you start, make sure you have `created <https://cloud.yandex.com/docs/iam/operations/sa/create>`__ +a Yandex Cloud `service account <https://cloud.yandex.com/docs/iam/concepts/users/service-accounts>`__ +with the ``lockbox.viewer`` and ``lockbox.payloadViewer`` permissions. -First, you need to create `Authorized key <https://cloud.yandex.com/en/docs/iam/concepts/authorization/key>`__ -for your service account and save the generated JSON file with public and private key parts. +First, you need to create an `authorized key <https://cloud.yandex.com/docs/iam/concepts/authorization/key>`__ +for your service account and save the generated JSON file with both public and private key parts. -Then you need to specify the ``folder_id`` and key in the ``backend_kwargs``: +Then, you need to specify the ``folder_id`` and key in ``backend_kwargs``: .. code-block:: ini [secrets] - backend_kwargs = {"folder_id": "b1g66mft1vopnevbn57j", "yc_sa_key_json": {"id": "...", "service_account_id": "...", "private_key": "..."}"} + backend_kwargs = {"folder_id": "b1g66mft1vo1n4vbn57j", "yc_sa_key_json": {"id": "...", "service_account_id": "...", "private_key": "..."}"} -Alternatively, you can specify the path to JSON file in the ``backend_kwargs``: +Alternatively, you can specify the path to the JSON file in ``backend_kwargs``: .. code-block:: ini [secrets] - backend_kwargs = {"folder_id": "b1g66mft1vopnevbn57j", "yc_sa_key_json_path": "/home/airflow/authorized_key.json"} + backend_kwargs = {"folder_id": "b1g66mft1vo1n4vbn57j", "yc_sa_key_json_path": "/home/airflow/authorized_key.json"} -Using Yandex Cloud Connection for authorization +Using Yandex Cloud connection for authorization ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -First, you need to create :ref:`Yandex Cloud Connection <yandex_cloud_connection>`. +First, you need to create :ref:`Yandex Cloud connection <yandex_cloud_connection>`. -Then you need to specify the ``connection_id`` in the ``backend_kwargs``: +Then, you need to specify the ``connection_id`` in ``backend_kwargs``: .. code-block:: ini [secrets] backend_kwargs = {"yc_connection_id": "my_yc_connection"} -If no credentials specified, Lockbox Secret Backend will try to use default connection id ``yandexcloud_default``. +If you do not specify any credentials, +Lockbox Secret Backend will try to use the default connection ID: ``yandexcloud_default``. -Lockbox Secret Backend will try to use default folder id from Connection, -also you can specify the ``folder_id`` in the ``backend_kwargs``: +Lockbox Secret Backend will try to use the default folder ID from your connection. +You can also specify the ``folder_id`` in the ``backend_kwargs``: .. code-block:: ini [secrets] - backend_kwargs = {"folder_id": "b1g66mft1vopnevbn57j", "yc_connection_id": "my_yc_connection"} + backend_kwargs = {"folder_id": "b1g66mft1vo1n4vbn57j", "yc_connection_id": "my_yc_connection"} -Storing and Retrieving Connections +Storing and retrieving connections ---------------------------------- -To store a Connection, you need to `create secret <https://cloud.yandex.com/en/docs/lockbox/operations/secret-create>`__ -with name in format ``{connections_prefix}{sep}{connection_name}`` -and payload contains text value with any key. +To store a connection, you need to `create a secret <https://cloud.yandex.com/docs/lockbox/operations/secret-create>`__ +with a name in the following format: ``{connections_prefix}{sep}{connection_name}``. -Storing a Connection as a URI -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The payload must contain a text value with any key. -The main way is to save connections as a :ref:`connection URI representation <generating_connection_uri>`. +Storing a connection as a URI +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Example: ``mysql://myname:mypassw...@myhost.com?this_param=some+val&that_param=other+val%2A`` +The main way to save connections is using a :ref:`connection URI representation <generating_connection_uri>`, such as +``mysql://myname:mypassw...@myhost.com?this_param=some+val&that_param=other+val%2A``. -Here is an example of secret creation with the ``yc`` cli: +Here is an example of creating a secret with the ``yc`` CLI: .. code-block:: console @@ -183,16 +187,16 @@ Here is an example of secret creation with the ``yc`` cli: done (1s) name: airflow/connections/mysqldb -Storing a Connection as a JSON -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Storing a connection as JSON +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Alternatively, you can save connections in JSON format: +Another way to store connections is using JSON format: .. code-block:: json { "conn_type": "mysql", - "host": "myhost.com", + "host": "host.com", "login": "myname", "password": "mypassword", "extra": { @@ -201,17 +205,17 @@ Alternatively, you can save connections in JSON format: } } -Here is an example of secret creation with the ``yc`` cli: +Here is an example of creating a secret with the ``yc`` CLI: .. code-block:: console $ yc lockbox secret create \ - --name airflow/connections/mysqldbjson \ - --payload '[{"key": "value", "text_value": "{\"conn_type\": \"mysql\", \"host\": \"myhost.com\", \"login\": \"myname\", \"password\": \"mypassword\", \"extra\": {\"this_param\": \"some val\", \"that_param\": \"other val*\"}}"}]' + --name airflow/connections/my_sql_db_json \ + --payload '[{"key": "value", "text_value": "{\"conn_type\": \"mysql\", \"host\": \"host.com\", \"login\": \"myname\", \"password\": \"mypassword\", \"extra\": {\"this_param\": \"some val\", \"that_param\": \"other val*\"}}"}]' done (1s) - name: airflow/connections/mysqldbjson + name: airflow/connections/my_sql_db_json -Retrieving Connection +Retrieving connection ~~~~~~~~~~~~~~~~~~~~~ To check the connection is correctly read from the Lockbox Secret Backend, you can use ``airflow connections get``: @@ -219,18 +223,18 @@ To check the connection is correctly read from the Lockbox Secret Backend, you c .. code-block:: console $ airflow connections get mysqldb -o json - [{"id": null, "conn_id": "mysqldb", "conn_type": "mysql", "description": null, "host": "myhost.com", "schema": "", "login": "myname", "password": "mypassword", "port": null, "is_encrypted": "False", "is_extra_encrypted": "False", "extra_dejson": {"this_param": "some val", "that_param": "other val*"}, "get_uri": "mysql://myname:mypassw...@myhost.com/?this_param=some+val&that_param=other+val%2A"}] + [{"id": null, "conn_id": "mysqldb", "conn_type": "mysql", "description": null, "host": "host.com", "schema": "", "login": "myname", "password": "mypassword", "port": null, "is_encrypted": "False", "is_extra_encrypted": "False", "extra_dejson": {"this_param": "some val", "that_param": "other val*"}, "get_uri": "mysql://myname:mypassw...@myhost.com/?this_param=some+val&that_param=other+val%2A"}] -Storing and Retrieving Variables +Storing and retrieving variables -------------------------------- -To store a Variable, you need to `create secret <https://cloud.yandex.com/en/docs/lockbox/operations/secret-create>`__ -with name in format ``{variables_prefix}{sep}{variable_name}`` -and payload contains text value with any key. +To store a variable, you need to `create a secret <https://cloud.yandex.com/docs/lockbox/operations/secret-create>`__ +with a name in the following format: ``{variables_prefix}{sep}{variable_name}``. +The payload must contain a text value with any key. -This is an example variable value: ``some_secret_data`` +Here is how a variable value may look like: ``some_secret_data``. -Here is an example of secret creation with the ``yc`` cli: +Here is an example of creating a secret with the ``yc`` CLI: .. code-block:: console @@ -247,18 +251,19 @@ To check the variable is correctly read from the Lockbox Secret Backend, you can $ airflow variables get my_variable some_secret_data -Storing and Retrieving Configs +Storing and retrieving configs ------------------------------ -You can store some sensitive configs in the Lockbox Secret Backend. +Lockbox Secret Backend is also suitable for storing sensitive configurations. -For example, we will provide a secret for ``sentry.sentry_dsn`` and use ``sentry_dsn_value`` as the config value name. +For example, we will provide you with a secret for ``sentry.sentry_dsn`` +and use ``sentry_dsn_value`` as the config value name. -To store a Config, you need to `create secret <https://cloud.yandex.com/en/docs/lockbox/operations/secret-create>`__ -with name in format ``{config_prefix}{sep}{config_value_name}`` -and payload contains text value with any key. +To store a config, you need to `create a secret <https://cloud.yandex.com/docs/lockbox/operations/secret-create>`__ +with a name in the following format: ``{config_prefix}{sep}{config_value_name}``. +The payload must contain a text value with any key. -Here is an example of secret creation with the ``yc`` cli: +Here is an example of creating a secret with the ``yc`` CLI: .. code-block:: console @@ -282,10 +287,10 @@ To check the config value is correctly read from the Lockbox Secret Backend, you $ airflow config get-value sentry sentry_dsn https://pub...@sentry.example.com/1 -Clean up --------- +Cleaning up your secret +----------------------- -You can easily delete your secret with the ``yc`` cli: +You can easily delete your secret with the ``yc`` CLI: .. code-block:: console diff --git a/tests/providers/yandex/hooks/test_yandex.py b/tests/providers/yandex/hooks/test_yandex.py index 1ba8c800c1..ea885c6b3f 100644 --- a/tests/providers/yandex/hooks/test_yandex.py +++ b/tests/providers/yandex/hooks/test_yandex.py @@ -37,7 +37,7 @@ class TestYandexHook: default_public_ssh_key = "test_key" extra_dejson = '{"extras": "extra"}' - mock_get_connection["extra_dejson"] = "sdsd" + mock_get_connection["extra_dejson"] = "sds" mock_get_connection.extra_dejson = '{"extras": "extra"}' mock_get_connection.return_value = mock.Mock( connection_id="yandexcloud_default", extra_dejson=extra_dejson @@ -137,7 +137,7 @@ class TestYandexHook: default_public_ssh_key = "test_key" extra_dejson = {field_name: field_value} - mock_get_connection["extra_dejson"] = "sdsd" + mock_get_connection["extra_dejson"] = "sds" mock_get_connection.extra_dejson = '{"extras": "extra"}' mock_get_connection.return_value = mock.Mock( connection_id="yandexcloud_default", extra_dejson=extra_dejson @@ -160,7 +160,7 @@ class TestYandexHook: default = "some_default" extra_dejson = '{"extras": "extra"}' - get_connection_mock["extra_dejson"] = "sdsd" + get_connection_mock["extra_dejson"] = "sds" get_connection_mock.extra_dejson = '{"extras": "extra"}' get_connection_mock.return_value = mock.Mock( connection_id="yandexcloud_default", extra_dejson=extra_dejson diff --git a/tests/providers/yandex/hooks/test_yandexcloud_dataproc.py b/tests/providers/yandex/hooks/test_yandexcloud_dataproc.py index 71b94dcd57..5629a3678e 100644 --- a/tests/providers/yandex/hooks/test_yandexcloud_dataproc.py +++ b/tests/providers/yandex/hooks/test_yandexcloud_dataproc.py @@ -48,8 +48,8 @@ OAUTH_TOKEN = "my_oauth_token" # Ssh public keys will be placed to Dataproc cluster nodes, allowing to get a root shell at the nodes SSH_PUBLIC_KEYS = [ - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAYQCxO38tKAJXIs9ivPxt7AYdfybgtAR1ow3Qkb9GPQ6wkFHQq" - "cFDe6faKCxH6iDRteo4D8L8BxwzN42uZSB0nfmjkIxFTcEU3mFSXEbWByg78aoddMrAAjatyrhH1pON6P0=" + "ssh-rsa AAA5B3NzaC1yc2EAA1ADA2ABA3AA4QCxO38tKA0XIs9ivPxt7AYdf3bgtAR1ow3Qkb9GPQ6wkFHQq" + "cFDe6faKCxH6iDRt2o4D8L8Bx6zN42uZSB0nf8jkIxFTcEU3mFSXEbWByg78ao3dMrAAj1tyr1H1pON6P0=" ] # If Yandex.Cloud credentials are set than full test will be run. Otherwise, only mocked tests. diff --git a/tests/providers/yandex/operators/test_yandexcloud_dataproc.py b/tests/providers/yandex/operators/test_yandexcloud_dataproc.py index 083e3d538a..0a055dd377 100644 --- a/tests/providers/yandex/operators/test_yandexcloud_dataproc.py +++ b/tests/providers/yandex/operators/test_yandexcloud_dataproc.py @@ -55,11 +55,11 @@ OAUTH_TOKEN = "my_oauth_token" # Ssh public keys will be placed to Dataproc cluster nodes, allowing to get a root shell at the nodes SSH_PUBLIC_KEYS = [ - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAYQCxO38tKAJXIs9ivPxt7AYdfybgtAR1ow3Qkb9GPQ6wkFHQq" - "cFDe6faKCxH6iDRteo4D8L8BxwzN42uZSB0nfmjkIxFTcEU3mFSXEbWByg78aoddMrAAjatyrhH1pON6P0=" + "ssh-rsa AAA5B3NzaC1yc2EAA1ADA2ABA3AA4QCxO38tKA0XIs9ivPxt7AYdf3bgtAR1ow3Qkb9GPQ6wkFHQq" + "cFDe6faKCxH6iDRt2o4D8L8Bx6zN42uZSB0nf8jkIxFTcEU3mFSXEbWByg78ao3dMrAAj1tyr1H1pON6P0=" ] -# https://cloud.yandex.com/en-ru/docs/logging/concepts/log-group +# https://cloud.yandex.com/docs/logging/concepts/log-group LOG_GROUP_ID = "my_log_group_id" @@ -120,8 +120,8 @@ class TestDataprocClusterCreateOperator: service_account_id=None, services=("HDFS", "YARN", "MAPREDUCE", "HIVE", "SPARK"), ssh_public_keys=[ - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAYQCxO38tKAJXIs9ivPxt7AYdfybgtAR1ow3Qkb9GPQ6wkFHQqcFD" - "e6faKCxH6iDRteo4D8L8BxwzN42uZSB0nfmjkIxFTcEU3mFSXEbWByg78aoddMrAAjatyrhH1pON6P0=" + "ssh-rsa AAA5B3NzaC1yc2EAA1ADA2ABA3AA4QCxO38tKA0XIs9ivPxt7AYdf3bgtAR1ow3Qkb9GPQ6wkFHQq" + "cFDe6faKCxH6iDRt2o4D8L8Bx6zN42uZSB0nf8jkIxFTcEU3mFSXEbWByg78ao3dMrAAj1tyr1H1pON6P0=" ], subnet_id="my_subnet_id", zone="ru-central1-c", diff --git a/tests/providers/yandex/secrets/test_lockbox.py b/tests/providers/yandex/secrets/test_lockbox.py index ae91ae637b..4cd19c6fb4 100644 --- a/tests/providers/yandex/secrets/test_lockbox.py +++ b/tests/providers/yandex/secrets/test_lockbox.py @@ -71,8 +71,8 @@ class TestLockboxSecretBackend: @patch("airflow.providers.yandex.secrets.lockbox.LockboxSecretBackend._get_secret_value") def test_yandex_lockbox_secret_backend_get_variable(self, mock_get_value): - k = "thisiskey" - v = "thisisvalue" + k = "this-is-key" + v = "this-is-value" mock_get_value.return_value = v @@ -82,8 +82,8 @@ class TestLockboxSecretBackend: @patch("airflow.providers.yandex.secrets.lockbox.LockboxSecretBackend._get_secret_value") def test_yandex_lockbox_secret_backend_get_config(self, mock_get_value): - k = "thisiskey" - v = "thisisvalue" + k = "this-is-key" + v = "this-is-value" mock_get_value.return_value = v @@ -111,7 +111,7 @@ class TestLockboxSecretBackend: mock_get_value.return_value = uri conn = LockboxSecretBackend( - yc_oauth_token="y3_Vdheub7w9bIut67GHeL345gfb5GAnd3dZnf08FRbvjeUFvetYiohGvc", + yc_oauth_token="y3_Vd3eub7w9bIut67GHeL345gfb5GAnd3dZnf08FR1vjeUFve7Yi8hGvc", ).get_connection(conn_id) assert conn.conn_id == conn_id @@ -143,8 +143,8 @@ class TestLockboxSecretBackend: @patch("airflow.providers.yandex.secrets.lockbox.LockboxSecretBackend._get_secret_value") def test_yandex_lockbox_secret_backend_get_variable_prefix_is_none(self, mock_get_value): - k = "thisiskey" - v = "thisisvalue" + k = "this-is-key" + v = "this-is-value" mock_get_value.return_value = v @@ -156,8 +156,8 @@ class TestLockboxSecretBackend: @patch("airflow.providers.yandex.secrets.lockbox.LockboxSecretBackend._get_secret_value") def test_yandex_lockbox_secret_backend_get_config_prefix_is_none(self, mock_get_value): - k = "thisiskey" - v = "thisisvalue" + k = "this-is-key" + v = "this-is-value" mock_get_value.return_value = v @@ -168,7 +168,7 @@ class TestLockboxSecretBackend: assert value is None def test_yandex_lockbox_secret_backend__client_created_without_exceptions(self): - yc_oauth_token = "y3_Vdheub7w9bIut67GHeL345gfb5GAnd3dZnf08FRbvjeUFvetYiohGvc" + yc_oauth_token = "y3_Vd3eub7w9bIut67GHeL345gfb5GAnd3dZnf08FR1vjeUFve7Yi8hGvc" sm = LockboxSecretBackend( yc_oauth_token=yc_oauth_token, @@ -178,7 +178,7 @@ class TestLockboxSecretBackend: @patch("airflow.providers.yandex.secrets.lockbox.LockboxSecretBackend._get_field") def test_yandex_lockbox_secret_backend__client_credentials_received_from_connection(self, mock_get_field): - yc_oauth_token = "y3_Vdheub7w9bIut67GHeL345gfb5GAnd3dZnf08FRbvjeUFvetYiohGvc" + yc_oauth_token = "y3_Vd3eub7w9bIut67GHeL345gfb5GAnd3dZnf08FR1vjeUFve7Yi8hGvc" yc_sa_key_json = "sa_key_json" yc_sa_key_json_path = "sa_key_json_path" folder_id = "folder_id123" @@ -207,8 +207,8 @@ class TestLockboxSecretBackend: assert sm.endpoint == endpoint assert sm.yc_connection_id == yc_connection_id - def test_yandex_lockbox_secret_backedn__get_endpoint(self): - endpoint = "api.cloud.yandex.net" + def test_yandex_lockbox_secret_backend__get_endpoint(self): + endpoint = "some-custom-api-endpoint.cloud.yandex.net" expected = { "endpoint": endpoint, } @@ -219,7 +219,7 @@ class TestLockboxSecretBackend: assert res == expected - def test_yandex_lockbox_secret_backedn__get_endpoint_not_specified(self): + def test_yandex_lockbox_secret_backend__get_endpoint_not_specified(self): expected = {} res = LockboxSecretBackend()._get_endpoint() @@ -227,9 +227,9 @@ class TestLockboxSecretBackend: assert res == expected def test_yandex_lockbox_secret_backend__build_secret_name(self): - prefix = "thiisprefix" - key = "thisiskey" - expected = "thiisprefix/thisiskey" + prefix = "this-is-prefix" + key = "this-is-key" + expected = "this-is-prefix/this-is-key" res = LockboxSecretBackend()._build_secret_name(prefix, key) @@ -237,8 +237,8 @@ class TestLockboxSecretBackend: def test_yandex_lockbox_secret_backend__build_secret_name_no_prefix(self): prefix = "" - key = "thisiskey" - expected = "thisiskey" + key = "this-is-key" + expected = "this-is-key" res = LockboxSecretBackend()._build_secret_name(prefix, key) @@ -246,9 +246,9 @@ class TestLockboxSecretBackend: def test_yandex_lockbox_secret_backend__build_secret_name_custom_sep(self): sep = "_" - prefix = "thiisprefix" - key = "thisiskey" - expected = "thiisprefix_thisiskey" + prefix = "this-is-prefix" + key = "this-is-key" + expected = "this-is-prefix_this-is-key" res = LockboxSecretBackend( sep=sep, @@ -364,7 +364,7 @@ class TestLockboxSecretBackend: mock_client.return_value = None res = LockboxSecretBackend( - folder_id="someid", + folder_id="some-id", )._get_secrets() assert res == secrets.secrets @@ -402,7 +402,7 @@ class TestLockboxSecretBackend: mock_client.return_value = None res = LockboxSecretBackend( - folder_id="someid", + folder_id="some-id", )._get_secrets() assert res == [*first_secrets.secrets, *second_secrets.secrets] @@ -465,6 +465,6 @@ class TestLockboxSecretBackend: def test_yandex_lockbox_secret_backend__get_field_connection_not_specified(self): sm = LockboxSecretBackend() sm.yc_connection_id = None - res = sm._get_field("somefield") + res = sm._get_field("some-field") assert res is None diff --git a/tests/providers/yandex/utils/test_credentials.py b/tests/providers/yandex/utils/test_credentials.py index 5bc1e17490..d7256d93c8 100644 --- a/tests/providers/yandex/utils/test_credentials.py +++ b/tests/providers/yandex/utils/test_credentials.py @@ -27,7 +27,7 @@ from airflow.providers.yandex.utils.credentials import ( def test_get_credentials_oauth_token(): - oauth_token = "y3_Vdheub7w9bIut67GHeL345gfb5GAnd3dZnf08FRbvjeUFvetYiohGvc" + oauth_token = "y3_Vd3eub7w9bIut67GHeL345gfb5GAnd3dZnf08FR1vjeUFve7Yi8hGvc" service_account_key = { "id": "...", "service_account_id": "...", diff --git a/tests/providers/yandex/utils/test_fields.py b/tests/providers/yandex/utils/test_fields.py index 3b42282324..61aaf40448 100644 --- a/tests/providers/yandex/utils/test_fields.py +++ b/tests/providers/yandex/utils/test_fields.py @@ -22,9 +22,9 @@ from airflow.providers.yandex.utils.fields import get_field_from_extras def test_get_field_from_extras(): - field_name = "somefield" + field_name = "some-field" default = None - expected = "somevalue" + expected = "some-value" extras = { field_name: expected, } @@ -39,7 +39,7 @@ def test_get_field_from_extras(): def test_get_field_from_extras_not_found(): - field_name = "somefield" + field_name = "some-field" default = "default" expected = default extras = {} @@ -54,9 +54,9 @@ def test_get_field_from_extras_not_found(): def test_get_field_from_extras_prefixed_in_extra(): - field_name = "somefield" + field_name = "some-field" default = None - expected = "somevalue" + expected = "some-value" extras = { f"extra__yandexcloud__{field_name}": expected, } @@ -71,7 +71,7 @@ def test_get_field_from_extras_prefixed_in_extra(): def test_get_field_from_extras_field_name_with_extra_raise_exception(): - field_name = "extra__yandexcloud__fieldname" + field_name = "extra__yandexcloud__field" default = None extras = {}