[GitHub] [airflow] kaxil commented on a change in pull request #9639: Add secrets backend for microsoft azure key vault
kaxil commented on a change in pull request #9639:
URL: https://github.com/apache/airflow/pull/9639#discussion_r452664124
##
File path: airflow/providers/microsoft/azure/secrets/azure_key_vault.py
##
@@ -0,0 +1,131 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from typing import Optional
+
+from azure.core.exceptions import ResourceNotFoundError
+from azure.identity import DefaultAzureCredential
+from azure.keyvault.secrets import SecretClient
+from cached_property import cached_property
+
+from airflow.secrets import BaseSecretsBackend
+from airflow.utils.log.logging_mixin import LoggingMixin
+
+
+class AzureKeyVaultBackend(BaseSecretsBackend, LoggingMixin):
+"""
+Retrieves Airflow Connections or Variables from Azure Key Vault secrets.
+
+The Azure Key Vault can be configred as a secrets backend in the
``airflow.cfg``:
+
+.. code-block:: ini
+
+[secrets]
+backend =
airflow.providers.microsoft.azure.secrets.azure_key_vault.AzureKeyVaultBackend
+backend_kwargs = {"connections_prefix": "airflow-connections",
"vault_url": ""}
+
+For example, if the secrets prefix is
``airflow-connections-smtp-default``, this would be accessible
+if you provide ``{"connections_prefix": "airflow-connections"}`` and
request conn_id ``smtp-default``.
+And if variables prefix is ``airflow-variables-hello``, this would be
accessible
+if you provide ``{"variables_prefix": "airflow-variables"}`` and request
variable key ``hello``.
+
+:param vault_url: The URL of an Azure Key Vault to use
+:type vault_url: str
+:param connections_prefix: Specifies the prefix of the secret to read to
get Connections
+:type connections_prefix: str
+:param variables_prefix: Specifies the prefix of the secret to read to get
Variables
+:type variables_prefix: str
+:param sep: separator used to concatenate secret_prefix and secret_id.
Default: "-"
+:type sep: str
+"""
+
+def __init__(
+self,
+connections_prefix: str = 'airflow-connections',
+variables_prefix: str = 'airflow-variables',
+vault_url: str = '',
+sep: str = '-',
+**kwargs
+):
+super().__init__()
+self.vault_url = vault_url
+self.connections_prefix = connections_prefix.rstrip(sep)
+self.variables_prefix = variables_prefix.rstrip(sep)
+self.sep = sep
+self.kwargs = kwargs
+
+@cached_property
+def client(self):
+"""
+Create a Azure Key Vault client.
+"""
+credential = DefaultAzureCredential()
+client = SecretClient(
+vault_url=self.vault_url,
+credential=credential,
+**self.kwargs
+)
+return client
+
+def get_conn_uri(self, conn_id: str) -> Optional[str]:
+"""
+Get an Airflow Connection URI from an Azure Key Vault secret
+
+:param conn_id: The Airflow connection id to retrieve
+:type conn_id: str
+"""
+name = self.build_path(self.connections_prefix, conn_id, self.sep)
+return self.get_secret_value(name)
+
+def get_variable(self, key: str) -> Optional[str]:
+"""
+Get an Airflow Variable from an Azure Key Vault secret.
+
+:param key: Variable Key
+:type key: str
+:return: Variable Value
+"""
+name = self.build_path(self.variables_prefix, key, self.sep)
+return self.get_secret_value(name)
+
+@staticmethod
+def build_path(path_prefix: str, secret_id: str, sep: str = '-') -> str:
+"""
+Given a path_prefix and secret_id, build a valid secret name for the
Azure Key Vault Backend.
+Also replaces underscore in the path with dashes to support easy
switching between
+environmentvariables, so ``connection_default`` becomes
``connection-default``.
+
+:param path_prefix: The path prefix of the secret to retrieve
+:type path_prefix: str
+:param secret_id: Name of the secret
+:type secret_id: str
+:param sep: Separator used to concatenate path_prefix and secret_id
+:type sep: str
+"""
+path =
[GitHub] [airflow] kaxil commented on a change in pull request #9639: Add secrets backend for microsoft azure key vault
kaxil commented on a change in pull request #9639:
URL: https://github.com/apache/airflow/pull/9639#discussion_r452664124
##
File path: airflow/providers/microsoft/azure/secrets/azure_key_vault.py
##
@@ -0,0 +1,131 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from typing import Optional
+
+from azure.core.exceptions import ResourceNotFoundError
+from azure.identity import DefaultAzureCredential
+from azure.keyvault.secrets import SecretClient
+from cached_property import cached_property
+
+from airflow.secrets import BaseSecretsBackend
+from airflow.utils.log.logging_mixin import LoggingMixin
+
+
+class AzureKeyVaultBackend(BaseSecretsBackend, LoggingMixin):
+"""
+Retrieves Airflow Connections or Variables from Azure Key Vault secrets.
+
+The Azure Key Vault can be configred as a secrets backend in the
``airflow.cfg``:
+
+.. code-block:: ini
+
+[secrets]
+backend =
airflow.providers.microsoft.azure.secrets.azure_key_vault.AzureKeyVaultBackend
+backend_kwargs = {"connections_prefix": "airflow-connections",
"vault_url": ""}
+
+For example, if the secrets prefix is
``airflow-connections-smtp-default``, this would be accessible
+if you provide ``{"connections_prefix": "airflow-connections"}`` and
request conn_id ``smtp-default``.
+And if variables prefix is ``airflow-variables-hello``, this would be
accessible
+if you provide ``{"variables_prefix": "airflow-variables"}`` and request
variable key ``hello``.
+
+:param vault_url: The URL of an Azure Key Vault to use
+:type vault_url: str
+:param connections_prefix: Specifies the prefix of the secret to read to
get Connections
+:type connections_prefix: str
+:param variables_prefix: Specifies the prefix of the secret to read to get
Variables
+:type variables_prefix: str
+:param sep: separator used to concatenate secret_prefix and secret_id.
Default: "-"
+:type sep: str
+"""
+
+def __init__(
+self,
+connections_prefix: str = 'airflow-connections',
+variables_prefix: str = 'airflow-variables',
+vault_url: str = '',
+sep: str = '-',
+**kwargs
+):
+super().__init__()
+self.vault_url = vault_url
+self.connections_prefix = connections_prefix.rstrip(sep)
+self.variables_prefix = variables_prefix.rstrip(sep)
+self.sep = sep
+self.kwargs = kwargs
+
+@cached_property
+def client(self):
+"""
+Create a Azure Key Vault client.
+"""
+credential = DefaultAzureCredential()
+client = SecretClient(
+vault_url=self.vault_url,
+credential=credential,
+**self.kwargs
+)
+return client
+
+def get_conn_uri(self, conn_id: str) -> Optional[str]:
+"""
+Get an Airflow Connection URI from an Azure Key Vault secret
+
+:param conn_id: The Airflow connection id to retrieve
+:type conn_id: str
+"""
+name = self.build_path(self.connections_prefix, conn_id, self.sep)
+return self.get_secret_value(name)
+
+def get_variable(self, key: str) -> Optional[str]:
+"""
+Get an Airflow Variable from an Azure Key Vault secret.
+
+:param key: Variable Key
+:type key: str
+:return: Variable Value
+"""
+name = self.build_path(self.variables_prefix, key, self.sep)
+return self.get_secret_value(name)
+
+@staticmethod
+def build_path(path_prefix: str, secret_id: str, sep: str = '-') -> str:
+"""
+Given a path_prefix and secret_id, build a valid secret name for the
Azure Key Vault Backend.
+Also replaces underscore in the path with dashes to support easy
switching between
+environmentvariables, so ``connection_default`` becomes
``connection-default``.
+
+:param path_prefix: The path prefix of the secret to retrieve
+:type path_prefix: str
+:param secret_id: Name of the secret
+:type secret_id: str
+:param sep: Separator used to concatenate path_prefix and secret_id
+:type sep: str
+"""
+path =
[GitHub] [airflow] kaxil commented on a change in pull request #9639: Add secrets backend for microsoft azure key vault
kaxil commented on a change in pull request #9639: URL: https://github.com/apache/airflow/pull/9639#discussion_r450807727 ## File path: airflow/providers/microsoft/azure/secrets/azure_key_vault.py ## @@ -0,0 +1,107 @@ +""" Review comment: ```suggestion # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. """ ``` This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [airflow] kaxil commented on a change in pull request #9639: Add secrets backend for microsoft azure key vault
kaxil commented on a change in pull request #9639: URL: https://github.com/apache/airflow/pull/9639#discussion_r450807848 ## File path: airflow/providers/microsoft/azure/secrets/azure_key_vault.py ## @@ -0,0 +1,107 @@ +""" Review comment: Also in airflow/providers/microsoft/azure/secrets/__init__.py This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [airflow] kaxil commented on a change in pull request #9639: Add secrets backend for microsoft azure key vault
kaxil commented on a change in pull request #9639: URL: https://github.com/apache/airflow/pull/9639#discussion_r450807455 ## File path: airflow/providers/microsoft/azure/secrets/azure_key_vault.py ## @@ -0,0 +1,107 @@ +""" Review comment: You need to add Apache License to all the new files at the top ``` # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. ``` This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [airflow] kaxil commented on a change in pull request #9639: Add secrets backend for microsoft azure key vault
kaxil commented on a change in pull request #9639: URL: https://github.com/apache/airflow/pull/9639#discussion_r450807455 ## File path: airflow/providers/microsoft/azure/secrets/azure_key_vault.py ## @@ -0,0 +1,107 @@ +""" Review comment: You need to add Apache License to all the new files ``` # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. ``` This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [airflow] kaxil commented on a change in pull request #9639: Add secrets backend for microsoft azure key vault
kaxil commented on a change in pull request #9639:
URL: https://github.com/apache/airflow/pull/9639#discussion_r449522855
##
File path:
tests/providers/microsoft/azure/secrets/test_azure_key_vault_backend.py
##
@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest import TestCase, mock
+
+from moto import mock_secretsmanager
+
+from airflow.providers.microsoft.azure.secrets.azure_key_vault import
AzureKeyVaultBackend
+
+
+class TestAzureKeyVaultBackend(TestCase):
+@mock.patch("airflow.providers.microsoft.azure.secrets.azure_key_vault"
+"AzureKeyVaultBackend.get_conn_uri")
+def test_get_connections(self, mock_get_uri):
+mock_get_uri.return_value = "scheme://user:pass@host:100"
+conn_list = AzureKeyVaultBackend().get_connections("fake_conn")
+conn = conn_list[0]
+assert conn.host == 'host'
+
+@mock_secretsmanager
+def test_get_conn_uri(self):
+param = {
+'SecretId': 'airflow-connections-test_postgres',
+'SecretString': 'postgresql://airflow:airflow@host:5432/airflow'
+}
+
+backend = AzureKeyVaultBackend()
+backend.client.put_secret_value(**param)
+
+returned_uri = backend.get_conn_uri(conn_id="test_postgres")
+self.assertEqual('postgresql://airflow:airflow@host:5432/airflow',
returned_uri)
+
+@mock_secretsmanager
Review comment:
and here and everywhere in the file?
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [airflow] kaxil commented on a change in pull request #9639: Add secrets backend for microsoft azure key vault
kaxil commented on a change in pull request #9639: URL: https://github.com/apache/airflow/pull/9639#discussion_r449522757 ## File path: tests/providers/microsoft/azure/secrets/test_azure_key_vault_backend.py ## @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from unittest import TestCase, mock + +from moto import mock_secretsmanager Review comment: Why do we use `moto` here? This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [airflow] kaxil commented on a change in pull request #9639: Add secrets backend for microsoft azure key vault
kaxil commented on a change in pull request #9639:
URL: https://github.com/apache/airflow/pull/9639#discussion_r449521224
##
File path: airflow/providers/microsoft/azure/secrets/azure_key_vault.py
##
@@ -0,0 +1,107 @@
+"""
+This module contains a secrets backend for Azure Key Vault.
+"""
+from typing import Optional
+
+from azure.identity import DefaultAzureCredential
+from azure.keyvault.secrets import SecretClient
+from azure.core.exceptions import ResourceNotFoundError
+from cached_property import cached_property
+
+from airflow.secrets import BaseSecretsBackend
+from airflow.utils.log.logging_mixin import LoggingMixin
+
+
+class AzureKeyVaultBackend(BaseSecretsBackend, LoggingMixin):
+"""
+Retrieves Airflow Connections or Variables from Azure Key Vault secrets.
+
+The Azure Key Vault can be configred as a secrets backend in the
``airflow.cfg``:
+
+.. code-block:: ini
+
+[secrets]
+backend =
airflow.providers.microsoft.azure.secrets.azure_key_vault.AzureKeyVaultBackend
+backend_kwargs = {"vault_url": ""}
+
+For example, if the secrets prefix is
``airflow-connections-smtp-default``, this would be accessible
+if you provide ``{"connections_prefix": "airflow-connections"}`` and
request conn_id ``smtp-default``.
+And if variables prefix is ``airflow-variables-hello``, this would be
accessible
+if you provide ``{"variables_prefix": "airflow-variables"}`` and request
variable key ``hello``.
+
+:param vault_url: The URL of an Azure Key Vault to use
+:type vault_url: str
+:param connections_prefix: Specifies the prefix of the secret to read to
get Connections
+:type connections_prefix: str
+:param variables_prefix: Specifies the prefix of the secret to read to get
Variables
+:type variables_prefix: str
+:param sep: separator used to concatenate secret_prefix and secret_id.
Default: "-"
+:type sep: str
+"""
+
+def __init__(self, vault_url: str = None, connections_prefix: str =
'airflow-connections',
+ variables_prefix: str = 'airflow-variables', sep: str = '-',
**kwargs):
+super().__init__(**kwargs)
+self.connections_prefix = connections_prefix.rstrip(sep)
+self.variables_prefix = variables_prefix.rstrip(sep)
+self.vault_url = vault_url
+self.sep = sep
+self.kwargs = kwargs
+
+@cached_property
+def client(self):
+"""
+Create a Azure Key Vault client.
+"""
+credential = DefaultAzureCredential()
Review comment:
What is the limitation of using `DefaultAzureCredential`
https://docs.microsoft.com/en-us/azure/developer/python/azure-sdk-authenticate?tabs=cmd
##
File path: airflow/providers/microsoft/azure/secrets/azure_key_vault.py
##
@@ -0,0 +1,107 @@
+"""
+This module contains a secrets backend for Azure Key Vault.
+"""
+from typing import Optional
+
+from azure.identity import DefaultAzureCredential
+from azure.keyvault.secrets import SecretClient
+from azure.core.exceptions import ResourceNotFoundError
+from cached_property import cached_property
+
+from airflow.secrets import BaseSecretsBackend
+from airflow.utils.log.logging_mixin import LoggingMixin
+
+
+class AzureKeyVaultBackend(BaseSecretsBackend, LoggingMixin):
+"""
+Retrieves Airflow Connections or Variables from Azure Key Vault secrets.
+
+The Azure Key Vault can be configred as a secrets backend in the
``airflow.cfg``:
+
+.. code-block:: ini
+
+[secrets]
+backend =
airflow.providers.microsoft.azure.secrets.azure_key_vault.AzureKeyVaultBackend
+backend_kwargs = {"vault_url": ""}
+
+For example, if the secrets prefix is
``airflow-connections-smtp-default``, this would be accessible
+if you provide ``{"connections_prefix": "airflow-connections"}`` and
request conn_id ``smtp-default``.
+And if variables prefix is ``airflow-variables-hello``, this would be
accessible
+if you provide ``{"variables_prefix": "airflow-variables"}`` and request
variable key ``hello``.
+
+:param vault_url: The URL of an Azure Key Vault to use
+:type vault_url: str
+:param connections_prefix: Specifies the prefix of the secret to read to
get Connections
+:type connections_prefix: str
+:param variables_prefix: Specifies the prefix of the secret to read to get
Variables
+:type variables_prefix: str
+:param sep: separator used to concatenate secret_prefix and secret_id.
Default: "-"
+:type sep: str
+"""
+
+def __init__(self, vault_url: str = None, connections_prefix: str =
'airflow-connections',
+ variables_prefix: str = 'airflow-variables', sep: str = '-',
**kwargs):
+super().__init__(**kwargs)
Review comment:
```suggestion
super().__init__()
```
This is an automated message from the Apache Git Service.
To
