This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
The following commit(s) were added to refs/heads/main by this push:
new d1490372 bug(storage-azdls): Fix inferred WASB endpoint (#1417)
d1490372 is described below
commit d149037282ad1436f05f125ec158a24e5d4f2b91
Author: Jannik Steinmann <[email protected]>
AuthorDate: Mon Jun 9 11:20:33 2025 +0200
bug(storage-azdls): Fix inferred WASB endpoint (#1417)
## Which issue does this PR close?
- Relates to #1360.
I tested again fileIO operations with the `wasbs://` scheme, and the
current implementation mistakenly used the `blob` endpoint which is
lacking the ADLS APIs.
## What changes are included in this PR?
The code now always uses the `dfs` endpoint, regardless of whether a
user passes `wasb[s]://` or `abfs[s]://`. Because the
`AzureStoragePath::storage_service` field isn't read, I've removed it.
## Are these changes tested?
Yes, I added a new unit test for the `AzureStoragePath::endpoint`
method.
---------
Signed-off-by: DerGut <[email protected]>
Signed-off-by: Jannik Steinmann <[email protected]>
---
crates/iceberg/src/io/storage_azdls.rs | 54 +++++++++++++++++++++++++++++-----
1 file changed, 46 insertions(+), 8 deletions(-)
diff --git a/crates/iceberg/src/io/storage_azdls.rs
b/crates/iceberg/src/io/storage_azdls.rs
index 2cc6f1e2..c3daa4f5 100644
--- a/crates/iceberg/src/io/storage_azdls.rs
+++ b/crates/iceberg/src/io/storage_azdls.rs
@@ -230,9 +230,6 @@ struct AzureStoragePath {
account_name: String,
- /// Either `blob` or `dfs` for Blob Storage and ADLSv2 respectively.
- storage_service: String,
-
/// The endpoint suffix, e.g., `core.windows.net` for the public cloud
/// endpoint.
endpoint_suffix: String,
@@ -249,10 +246,9 @@ impl AzureStoragePath {
/// This is possible because the path is fully qualified.
fn as_endpoint(&self) -> String {
format!(
- "{}://{}.{}.{}",
+ "{}://{}.dfs.{}",
self.scheme.as_http_scheme(),
self.account_name,
- self.storage_service,
self.endpoint_suffix
)
}
@@ -278,7 +274,6 @@ impl FromStr for AzureStoragePath {
scheme,
filesystem: filesystem.to_string(),
account_name: account_name.to_string(),
- storage_service: storage_service.to_string(),
endpoint_suffix: endpoint_suffix.to_string(),
path: url.path().to_string(),
})
@@ -505,7 +500,7 @@ mod tests {
}
#[test]
- fn test_parse_azure_storage_path() {
+ fn test_azure_storage_path_parse() {
let test_cases = vec![
(
"succeeds",
@@ -514,7 +509,6 @@ mod tests {
scheme: AzureStorageScheme::Abfss,
filesystem: "somefs".to_string(),
account_name: "myaccount".to_string(),
- storage_service: "dfs".to_string(),
endpoint_suffix: "core.windows.net".to_string(),
path: "/path/to/file.parquet".to_string(),
}),
@@ -549,4 +543,48 @@ mod tests {
}
}
}
+
+ #[test]
+ fn test_azure_storage_path_endpoint() {
+ let test_cases = vec![
+ (
+ "abfss uses https",
+ AzureStoragePath {
+ scheme: AzureStorageScheme::Abfss,
+ filesystem: "myfs".to_string(),
+ account_name: "myaccount".to_string(),
+ endpoint_suffix: "core.windows.net".to_string(),
+ path: "/path/to/file.parquet".to_string(),
+ },
+ "https://myaccount.dfs.core.windows.net";,
+ ),
+ (
+ "abfs uses http",
+ AzureStoragePath {
+ scheme: AzureStorageScheme::Abfs,
+ filesystem: "myfs".to_string(),
+ account_name: "myaccount".to_string(),
+ endpoint_suffix: "core.windows.net".to_string(),
+ path: "/path/to/file.parquet".to_string(),
+ },
+ "http://myaccount.dfs.core.windows.net";,
+ ),
+ (
+ "wasbs uses https and dfs",
+ AzureStoragePath {
+ scheme: AzureStorageScheme::Abfss,
+ filesystem: "myfs".to_string(),
+ account_name: "myaccount".to_string(),
+ endpoint_suffix: "core.windows.net".to_string(),
+ path: "/path/to/file.parquet".to_string(),
+ },
+ "https://myaccount.dfs.core.windows.net";,
+ ),
+ ];
+
+ for (name, path, expected) in test_cases {
+ let endpoint = path.as_endpoint();
+ assert_eq!(endpoint, expected, "Test case: {}", name);
+ }
+ }
}
