This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-3886-59b7c1c6ef93c839bceff3e8a1739341b6586e47 in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
commit 12d4a9a3ced73da290626bde6019ac2835d5c636 Author: Kaanchan Krishna <[email protected]> AuthorDate: Sat Oct 24 23:00:19 2026 +0000 FsspecFileIO: Resolve the ADLS account per location instead of caching it in properties (#3886) * FsspecFileIO: Resolve the ADLS account per location instead of caching it in properties _adls wrote the account name it inferred back into the properties dict it was given, and the caller passes the FileIO's own properties. The first location therefore pinned the account for every later one, so a second storage account was served a filesystem built for the first. Resolve the account and the SAS token into locals and leave the caller's dict alone. Precedence is unchanged: an explicit adls.account-name still wins, then a per account SAS token key, then the hostname. * Remove unnecessary comment * Cover SAS token write-back in ADLS no-mutation test Add a per-account SAS key so the test also guards against adls.sas-token being written back into the FileIO properties. Co-authored-by: Copilot App <[email protected]> --------- Co-authored-by: Kevin Liu <[email protected]> Co-authored-by: Copilot App <[email protected]> --- pyiceberg/io/fsspec.py | 21 ++++++++++++--------- tests/io/test_fsspec.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index 09bbe6f1d..98275f25a 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -265,17 +265,20 @@ def _adls(properties: Properties, hostname: str | None = None) -> AbstractFileSy from azure.core.credentials import AccessToken from azure.core.credentials_async import AsyncTokenCredential - for key, sas_token in { + account_name = properties.get(ADLS_ACCOUNT_NAME) + sas_token = properties.get(ADLS_SAS_TOKEN) + + for key, per_account_sas_token in { key.replace(f"{ADLS_SAS_TOKEN}.", ""): value for key, value in properties.items() if key.startswith(f"{ADLS_SAS_TOKEN}.") }.items(): - if ADLS_ACCOUNT_NAME not in properties: - properties[ADLS_ACCOUNT_NAME] = key.split(".")[0] - if ADLS_SAS_TOKEN not in properties: - properties[ADLS_SAS_TOKEN] = sas_token + if account_name is None: + account_name = key.split(".")[0] + if sas_token is None: + sas_token = per_account_sas_token # Fallback: extract account_name from URI hostname (e.g. "account.dfs.core.windows.net" -> "account") - if hostname and ADLS_ACCOUNT_NAME not in properties: - properties[ADLS_ACCOUNT_NAME] = hostname.split(".")[0] + if hostname and account_name is None: + account_name = hostname.split(".")[0] class StaticTokenCredential(AsyncTokenCredential): _DEFAULT_EXPIRY_SECONDS = 3600 @@ -298,9 +301,9 @@ def _adls(properties: Properties, hostname: str | None = None) -> AbstractFileSy return AzureBlobFileSystem( connection_string=properties.get(ADLS_CONNECTION_STRING), credential=credential, - account_name=properties.get(ADLS_ACCOUNT_NAME), + account_name=account_name, account_key=properties.get(ADLS_ACCOUNT_KEY), - sas_token=properties.get(ADLS_SAS_TOKEN), + sas_token=sas_token, tenant_id=properties.get(ADLS_TENANT_ID), client_id=properties.get(ADLS_CLIENT_ID), client_secret=properties.get(ADLS_CLIENT_SECRET), diff --git a/tests/io/test_fsspec.py b/tests/io/test_fsspec.py index 45835a08e..7b9d87155 100644 --- a/tests/io/test_fsspec.py +++ b/tests/io/test_fsspec.py @@ -697,6 +697,37 @@ def test_adls_account_name_extracted_from_uri_hostname() -> None: ) +def test_adls_account_name_resolved_per_location() -> None: + """Two locations in different accounts must each get a filesystem for their own account.""" + session_properties: Properties = {"adls.tenant-id": "test-tenant-id"} + + with mock.patch("adlfs.AzureBlobFileSystem") as mock_adlfs: + adls_fileio = FsspecFileIO(properties=session_properties) + + adls_fileio.new_input(location="abfss://[email protected]/wh/t/a.parquet") + adls_fileio.new_input(location="abfss://[email protected]/wh/t/b.parquet") + + account_names = [call.kwargs["account_name"] for call in mock_adlfs.call_args_list] + assert account_names == ["accountone", "accounttwo"] + + +def test_adls_does_not_mutate_properties() -> None: + """Resolving an account or SAS token from a location must not write it back into the shared properties.""" + original_properties: Properties = { + "adls.tenant-id": "test-tenant-id", + "adls.sas-token.accountone.dfs.core.windows.net": "test-sas-token", + } + session_properties: Properties = dict(original_properties) + + with mock.patch("adlfs.AzureBlobFileSystem"): + adls_fileio = FsspecFileIO(properties=session_properties) + adls_fileio.new_input(location="abfss://[email protected]/wh/t/a.parquet") + + assert "adls.account-name" not in adls_fileio.properties + assert "adls.sas-token" not in adls_fileio.properties + assert session_properties == original_properties + + def test_adls_account_name_not_overridden_when_in_properties() -> None: """Test that explicit adls.account-name in properties is not overridden by URI hostname.""" session_properties: Properties = {
