This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new be53013349e Support azure-datalake-store 1.x in AzureDataLakeHook
(#73819)
be53013349e is described below
commit be53013349eed7d5d7c6f3f188b5d5b6000d182a
Author: Haseeb Malik <[email protected]>
AuthorDate: Fri Oct 2 09:47:40 2026 -0400
Support azure-datalake-store 1.x in AzureDataLakeHook (#73819)
* Support azure-datalake-store 1.x in AzureDataLakeHook
* Require azure-datalake-store>=1.0.1 and drop 0.0.x support
---
providers/microsoft/azure/README.rst | 4 +-
providers/microsoft/azure/docs/index.rst | 4 +-
providers/microsoft/azure/pyproject.toml | 6 +--
.../providers/microsoft/azure/hooks/data_lake.py | 13 +++--
.../unit/microsoft/azure/hooks/test_data_lake.py | 61 +++++++++-------------
pyproject.toml | 1 -
uv.lock | 11 ++--
7 files changed, 43 insertions(+), 57 deletions(-)
diff --git a/providers/microsoft/azure/README.rst
b/providers/microsoft/azure/README.rst
index 0f0af5d8350..58167390169 100644
--- a/providers/microsoft/azure/README.rst
+++ b/providers/microsoft/azure/README.rst
@@ -55,13 +55,13 @@ PIP package Version required
========================================== ===================
``apache-airflow`` ``>=2.11.0``
``apache-airflow-providers-common-compat`` ``>=1.13.0``
-``adlfs`` ``>=2023.10.0``
+``adlfs`` ``>=2026.5.0``
``aiohttp`` ``>=3.14.0``
``azure-batch`` ``>=15.0.0``
``azure-ai-projects`` ``>=2.2.0``
``azure-cosmos`` ``>=4.15.0``
``azure-mgmt-cosmosdb`` ``>=9.9.0``
-``azure-datalake-store`` ``>=0.0.45,<1``
+``azure-datalake-store`` ``>=1.0.1``
``azure-identity`` ``>=1.25.3``
``azure-keyvault-secrets`` ``>=4.10.0``
``azure-mgmt-datalake-store`` ``>=0.5.0``
diff --git a/providers/microsoft/azure/docs/index.rst
b/providers/microsoft/azure/docs/index.rst
index 425673f098c..1c5cee9f736 100644
--- a/providers/microsoft/azure/docs/index.rst
+++ b/providers/microsoft/azure/docs/index.rst
@@ -111,13 +111,13 @@ PIP package Version
required
========================================== ===================
``apache-airflow`` ``>=2.11.0``
``apache-airflow-providers-common-compat`` ``>=1.13.0``
-``adlfs`` ``>=2023.10.0``
+``adlfs`` ``>=2026.5.0``
``aiohttp`` ``>=3.14.0``
``azure-batch`` ``>=15.0.0``
``azure-ai-projects`` ``>=2.2.0``
``azure-cosmos`` ``>=4.15.0``
``azure-mgmt-cosmosdb`` ``>=9.9.0``
-``azure-datalake-store`` ``>=0.0.45,<1``
+``azure-datalake-store`` ``>=1.0.1``
``azure-identity`` ``>=1.25.3``
``azure-keyvault-secrets`` ``>=4.10.0``
``azure-mgmt-datalake-store`` ``>=0.5.0``
diff --git a/providers/microsoft/azure/pyproject.toml
b/providers/microsoft/azure/pyproject.toml
index 29ebb327014..d0c489d1e82 100644
--- a/providers/microsoft/azure/pyproject.toml
+++ b/providers/microsoft/azure/pyproject.toml
@@ -61,16 +61,14 @@ requires-python = ">=3.10"
dependencies = [
"apache-airflow>=2.11.0",
"apache-airflow-providers-common-compat>=1.13.0",
- "adlfs>=2023.10.0",
+ "adlfs>=2026.5.0",
"aiohttp>=3.14.0",
"azure-batch>=15.0.0",
# 2.2.0 adds force deletion support for Hosted agents and agent versions.
"azure-ai-projects>=2.2.0",
"azure-cosmos>=4.15.0",
"azure-mgmt-cosmosdb>=9.9.0",
- # 1.x removed lib.auth used by AzureDataLakeHook; remove the cap once the
hook supports 1.x,
- # tracked at https://github.com/apache/airflow/issues/73795
- "azure-datalake-store>=0.0.45,<1",
+ "azure-datalake-store>=1.0.1",
"azure-identity>=1.25.3",
"azure-keyvault-secrets>=4.10.0",
"azure-mgmt-datalake-store>=0.5.0",
diff --git
a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_lake.py
b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_lake.py
index 8860c53fa2e..4cb4f1e46cd 100644
---
a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_lake.py
+++
b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_lake.py
@@ -21,7 +21,7 @@ from functools import cached_property
from typing import Any, cast
from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError
-from azure.datalake.store import core, lib, multithread
+from azure.datalake.store import core, multithread
from azure.identity import ClientSecretCredential, DefaultAzureCredential
from azure.storage.filedatalake import (
DataLakeDirectoryClient,
@@ -34,13 +34,12 @@ from azure.storage.filedatalake import (
from airflow.providers.common.compat.sdk import AirflowException, BaseHook
from airflow.providers.microsoft.azure.utils import (
- AzureIdentityCredentialAdapter,
add_managed_identity_connection_widgets,
get_field,
get_sync_default_azure_credential,
)
-Credentials = ClientSecretCredential | AzureIdentityCredentialAdapter |
DefaultAzureCredential
+Credentials = ClientSecretCredential | DefaultAzureCredential
class AzureDataLakeHook(BaseHook):
@@ -122,11 +121,15 @@ class AzureDataLakeHook(BaseHook):
credential: Credentials
tenant = self._get_field(extras, "tenant")
if tenant:
- credential = lib.auth(tenant_id=tenant,
client_secret=conn.password, client_id=conn.login)
+ credential = ClientSecretCredential(
+ tenant_id=tenant,
+ client_id=cast("str", conn.login),
+ client_secret=cast("str", conn.password),
+ )
else:
managed_identity_client_id = self._get_field(extras,
"managed_identity_client_id")
workload_identity_tenant_id = self._get_field(extras,
"workload_identity_tenant_id")
- credential = AzureIdentityCredentialAdapter(
+ credential = get_sync_default_azure_credential(
managed_identity_client_id=managed_identity_client_id,
workload_identity_tenant_id=workload_identity_tenant_id,
)
diff --git
a/providers/microsoft/azure/tests/unit/microsoft/azure/hooks/test_data_lake.py
b/providers/microsoft/azure/tests/unit/microsoft/azure/hooks/test_data_lake.py
index 262910da211..e1c0c37500b 100644
---
a/providers/microsoft/azure/tests/unit/microsoft/azure/hooks/test_data_lake.py
+++
b/providers/microsoft/azure/tests/unit/microsoft/azure/hooks/test_data_lake.py
@@ -73,44 +73,36 @@ class TestAzureDataLakeHook:
)
)
- @mock.patch(f"{MODULE}.lib", autospec=True)
- def test_conn(self, mock_lib):
- from azure.datalake.store import core
-
+ @mock.patch(f"{MODULE}.core.AzureDLFileSystem", autospec=True)
+ @mock.patch(f"{MODULE}.ClientSecretCredential", autospec=True)
+ def test_conn_with_tenant_uses_client_secret_credential(self,
mock_credential, mock_filesystem):
from airflow.providers.microsoft.azure.hooks.data_lake import
AzureDataLakeHook
hook = AzureDataLakeHook(azure_data_lake_conn_id="adl_test_key")
- assert hook._conn is None
- assert hook.conn_id == "adl_test_key"
- assert isinstance(hook.get_conn(), core.AzureDLFileSystem)
- assert mock_lib.auth.called
- @pytest.mark.usefixtures("connection_without_tenant")
- @mock.patch(f"{MODULE}.lib")
- @mock.patch(f"{MODULE}.AzureIdentityCredentialAdapter")
- def
test_fallback_to_azure_identity_credential_adppter_when_tenant_is_not_provided(
- self,
- mock_azure_identity_credential_adapter,
- mock_datalake_store_lib,
- ):
- from azure.datalake.store import core
+ assert hook.get_conn() is mock_filesystem.return_value
+ mock_credential.assert_called_once_with(
+ tenant_id="tenant", client_id="client_id", client_secret="client
secret"
+ )
+ mock_filesystem.assert_called_once_with(mock_credential.return_value,
store_name="accountname")
+ @pytest.mark.usefixtures("connection_without_tenant")
+ @mock.patch(f"{MODULE}.core.AzureDLFileSystem", autospec=True)
+ @mock.patch(f"{MODULE}.get_sync_default_azure_credential", autospec=True)
+ def test_conn_without_tenant_uses_default_azure_credential(self,
mock_credential, mock_filesystem):
from airflow.providers.microsoft.azure.hooks.data_lake import
AzureDataLakeHook
hook =
AzureDataLakeHook(azure_data_lake_conn_id="adl_test_key_without_tenant")
- assert hook._conn is None
- assert hook.conn_id == "adl_test_key_without_tenant"
- assert isinstance(hook.get_conn(), core.AzureDLFileSystem)
- mock_azure_identity_credential_adapter.assert_called()
- args = mock_azure_identity_credential_adapter.call_args
- assert args.kwargs["managed_identity_client_id"] is None
- assert args.kwargs["workload_identity_tenant_id"] is None
- mock_datalake_store_lib.auth.assert_not_called()
+
+ assert hook.get_conn() is mock_filesystem.return_value
+ mock_credential.assert_called_once_with(
+ managed_identity_client_id=None, workload_identity_tenant_id=None
+ )
+ mock_filesystem.assert_called_once_with(mock_credential.return_value,
store_name="accountname")
@pytest.mark.usefixtures("connection")
@mock.patch(f"{MODULE}.core.AzureDLFileSystem", autospec=True)
- @mock.patch(f"{MODULE}.lib", autospec=True)
- def test_check_for_blob(self, mock_lib, mock_filesystem):
+ def test_check_for_blob(self, mock_filesystem):
from airflow.providers.microsoft.azure.hooks.data_lake import
AzureDataLakeHook
mocked_glob = mock_filesystem.return_value.glob
@@ -120,8 +112,7 @@ class TestAzureDataLakeHook:
@pytest.mark.usefixtures("connection")
@mock.patch(f"{MODULE}.multithread.ADLUploader", autospec=True)
- @mock.patch(f"{MODULE}.lib", autospec=True)
- def test_upload_file(self, mock_lib, mock_uploader):
+ def test_upload_file(self, mock_uploader):
from airflow.providers.microsoft.azure.hooks.data_lake import
AzureDataLakeHook
hook = AzureDataLakeHook(azure_data_lake_conn_id="adl_test_key")
@@ -145,8 +136,7 @@ class TestAzureDataLakeHook:
@pytest.mark.usefixtures("connection")
@mock.patch(f"{MODULE}.multithread.ADLDownloader", autospec=True)
- @mock.patch(f"{MODULE}.lib", autospec=True)
- def test_download_file(self, mock_lib, mock_downloader):
+ def test_download_file(self, mock_downloader):
from airflow.providers.microsoft.azure.hooks.data_lake import
AzureDataLakeHook
hook = AzureDataLakeHook(azure_data_lake_conn_id="adl_test_key")
@@ -170,8 +160,7 @@ class TestAzureDataLakeHook:
@pytest.mark.usefixtures("connection")
@mock.patch(f"{MODULE}.core.AzureDLFileSystem", autospec=True)
- @mock.patch(f"{MODULE}.lib", autospec=True)
- def test_list_glob(self, mock_lib, mock_fs):
+ def test_list_glob(self, mock_fs):
from airflow.providers.microsoft.azure.hooks.data_lake import
AzureDataLakeHook
hook = AzureDataLakeHook(azure_data_lake_conn_id="adl_test_key")
@@ -180,8 +169,7 @@ class TestAzureDataLakeHook:
@pytest.mark.usefixtures("connection")
@mock.patch(f"{MODULE}.core.AzureDLFileSystem", autospec=True)
- @mock.patch(f"{MODULE}.lib", autospec=True)
- def test_list_walk(self, mock_lib, mock_fs):
+ def test_list_walk(self, mock_fs):
from airflow.providers.microsoft.azure.hooks.data_lake import
AzureDataLakeHook
hook = AzureDataLakeHook(azure_data_lake_conn_id="adl_test_key")
@@ -190,8 +178,7 @@ class TestAzureDataLakeHook:
@pytest.mark.usefixtures("connection")
@mock.patch(f"{MODULE}.core.AzureDLFileSystem", autospec=True)
- @mock.patch(f"{MODULE}.lib", autospec=True)
- def test_remove(self, mock_lib, mock_fs):
+ def test_remove(self, mock_fs):
from airflow.providers.microsoft.azure.hooks.data_lake import
AzureDataLakeHook
hook = AzureDataLakeHook(azure_data_lake_conn_id="adl_test_key")
diff --git a/pyproject.toml b/pyproject.toml
index 8e816c27718..d72847f42a5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1674,7 +1674,6 @@ airbyte-api = "1.x breaks provider tests (#69081)"
mysql-connector-python = "Releases without wheels for >=3.12 (#60889, #66026)"
anthropic = "Provider SDK migration is version-sensitive (#72072, #72094)"
azure-ai-projects = "2.5+ needs openai>=3, broke main (#73621, #73627)"
-azure-datalake-store = "1.x removed lib.auth used by AzureDataLakeHook
(#73795)"
[tool.uv.pip]
# Synchroonize with scripts/ci/prek/upgrade_important_versions.py
diff --git a/uv.lock b/uv.lock
index cbe40edf729..0e3392d9ffc 100644
--- a/uv.lock
+++ b/uv.lock
@@ -6750,7 +6750,7 @@ docs = [
[package.metadata]
requires-dist = [
- { name = "adlfs", specifier = ">=2023.10.0" },
+ { name = "adlfs", specifier = ">=2026.5.0" },
{ name = "aiohttp", specifier = ">=3.14.0" },
{ name = "apache-airflow", editable = "." },
{ name = "apache-airflow-providers-amazon", marker = "extra == 'amazon'",
editable = "providers/amazon" },
@@ -6763,7 +6763,7 @@ requires-dist = [
{ name = "azure-ai-projects", specifier = ">=2.2.0" },
{ name = "azure-batch", specifier = ">=15.0.0" },
{ name = "azure-cosmos", specifier = ">=4.15.0" },
- { name = "azure-datalake-store", specifier = ">=0.0.45,<1" },
+ { name = "azure-datalake-store", specifier = ">=1.0.1" },
{ name = "azure-identity", specifier = ">=1.25.3" },
{ name = "azure-keyvault-secrets", specifier = ">=4.10.0" },
{ name = "azure-kusto-data", specifier = ">=4.1.0,!=5.0.0" },
@@ -10162,16 +10162,15 @@ wheels = [
[[package]]
name = "azure-datalake-store"
-version = "0.0.53"
+version = "1.0.1"
source = { registry = "https://pypi.org/simple"; }
dependencies = [
{ name = "cffi" },
- { name = "msal" },
{ name = "requests" },
]
-sdist = { url =
"https://files.pythonhosted.org/packages/22/ff/61369d06422b5ac48067215ff404841342651b14a89b46c8d8e1507c8f17/azure-datalake-store-0.0.53.tar.gz";,
hash =
"sha256:05b6de62ee3f2a0a6e6941e6933b792b800c3e7f6ffce2fc324bc19875757393", size
= 71430, upload-time = "2023-05-10T21:17:05.665Z" }
+sdist = { url =
"https://files.pythonhosted.org/packages/74/58/41042543710a3a0be3bd1b7851c790a3087cdbf4c8eb14efcd7a0a910ea7/azure_datalake_store-1.0.1.tar.gz";,
hash =
"sha256:5364d4445aab154a1c7cb10215629c3ce46ce5c7aaaf16071890c03fae53a035", size
= 69462, upload-time = "2025-06-11T15:43:07.211Z" }
wheels = [
- { url =
"https://files.pythonhosted.org/packages/88/2a/75f56b14f115189155cf12e46b366ad1fe3357af5a1a7c09f7446662d617/azure_datalake_store-0.0.53-py2.py3-none-any.whl";,
hash =
"sha256:a30c902a6e360aa47d7f69f086b426729784e71c536f330b691647a51dc42b2b", size
= 55308, upload-time = "2023-05-10T21:17:02.629Z" },
+ { url =
"https://files.pythonhosted.org/packages/75/bd/9cc9f114dbf90717dac49f1f7365156a9a005ef7016df2d4eb28d6442b90/azure_datalake_store-1.0.1-py2.py3-none-any.whl";,
hash =
"sha256:3772a2a247aaf9f5fa4b0f2cc0a0225072960cc245cfc8130588babb2b9fe705", size
= 53141, upload-time = "2025-06-03T23:23:46.377Z" },
]
[[package]]
