This is an automated email from the ASF dual-hosted git repository.
liurenjie1024 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 04d90bfd3 feat(io): Add specific storage configs (#2072)
04d90bfd3 is described below
commit 04d90bfd3af1db3890696ae8b88102d08562a585
Author: Shawn Chang <[email protected]>
AuthorDate: Tue Jan 27 17:42:47 2026 -0800
feat(io): Add specific storage configs (#2072)
## Which issue does this PR close?
- Closes #2055
## What changes are included in this PR?
- Added specific storage configs: `S3Config`, `GcsConfig`,
`AzdlsConfig`, `OssConfig`
- Moved config keys from files like `storage_s3.rs` to `config/s3.rs`
## Are these changes tested?
Added uts to test `TryFrom<StorageConfig>`
---------
Co-authored-by: Renjie Liu <[email protected]>
---
crates/iceberg/src/io/config/azdls.rs | 180 ++++++++++++++++++++
crates/iceberg/src/io/config/gcs.rs | 189 ++++++++++++++++++++
crates/iceberg/src/io/config/mod.rs | 18 +-
crates/iceberg/src/io/config/oss.rs | 124 ++++++++++++++
crates/iceberg/src/io/config/s3.rs | 303 +++++++++++++++++++++++++++++++++
crates/iceberg/src/io/mod.rs | 10 +-
crates/iceberg/src/io/storage_azdls.rs | 33 +---
crates/iceberg/src/io/storage_gcs.rs | 28 +--
crates/iceberg/src/io/storage_oss.rs | 12 +-
crates/iceberg/src/io/storage_s3.rs | 48 +-----
10 files changed, 824 insertions(+), 121 deletions(-)
diff --git a/crates/iceberg/src/io/config/azdls.rs
b/crates/iceberg/src/io/config/azdls.rs
new file mode 100644
index 000000000..059012942
--- /dev/null
+++ b/crates/iceberg/src/io/config/azdls.rs
@@ -0,0 +1,180 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Azure Data Lake Storage configuration.
+//!
+//! This module provides configuration constants and types for Azure Data Lake
Storage.
+
+use serde::{Deserialize, Serialize};
+use typed_builder::TypedBuilder;
+
+use super::StorageConfig;
+use crate::Result;
+
+/// A connection string.
+///
+/// Note, this string is parsed first, and any other passed adls.* properties
+/// will override values from the connection string.
+pub const ADLS_CONNECTION_STRING: &str = "adls.connection-string";
+/// The account that you want to connect to.
+pub const ADLS_ACCOUNT_NAME: &str = "adls.account-name";
+/// The key to authentication against the account.
+pub const ADLS_ACCOUNT_KEY: &str = "adls.account-key";
+/// The shared access signature.
+pub const ADLS_SAS_TOKEN: &str = "adls.sas-token";
+/// The tenant-id.
+pub const ADLS_TENANT_ID: &str = "adls.tenant-id";
+/// The client-id.
+pub const ADLS_CLIENT_ID: &str = "adls.client-id";
+/// The client-secret.
+pub const ADLS_CLIENT_SECRET: &str = "adls.client-secret";
+/// The authority host of the service principal.
+/// - required for client_credentials authentication
+/// - default value: `https://login.microsoftonline.com`
+pub const ADLS_AUTHORITY_HOST: &str = "adls.authority-host";
+
+/// Azure Data Lake Storage configuration.
+///
+/// This struct contains all the configuration options for connecting to Azure
Data Lake Storage.
+/// Use the builder pattern via `AzdlsConfig::builder()` to construct
instances.
+/// ```
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize,
TypedBuilder)]
+pub struct AzdlsConfig {
+ /// Connection string.
+ #[builder(default, setter(strip_option, into))]
+ pub connection_string: Option<String>,
+ /// Account name.
+ #[builder(default, setter(strip_option, into))]
+ pub account_name: Option<String>,
+ /// Account key.
+ #[builder(default, setter(strip_option, into))]
+ pub account_key: Option<String>,
+ /// SAS token.
+ #[builder(default, setter(strip_option, into))]
+ pub sas_token: Option<String>,
+ /// Tenant ID.
+ #[builder(default, setter(strip_option, into))]
+ pub tenant_id: Option<String>,
+ /// Client ID.
+ #[builder(default, setter(strip_option, into))]
+ pub client_id: Option<String>,
+ /// Client secret.
+ #[builder(default, setter(strip_option, into))]
+ pub client_secret: Option<String>,
+ /// Authority host.
+ #[builder(default, setter(strip_option, into))]
+ pub authority_host: Option<String>,
+ /// Endpoint URL.
+ #[builder(default, setter(strip_option, into))]
+ pub endpoint: Option<String>,
+ /// Filesystem name.
+ #[builder(default, setter(into))]
+ pub filesystem: String,
+}
+
+impl TryFrom<&StorageConfig> for AzdlsConfig {
+ type Error = crate::Error;
+
+ fn try_from(config: &StorageConfig) -> Result<Self> {
+ let props = config.props();
+
+ let mut cfg = AzdlsConfig::default();
+
+ if let Some(connection_string) = props.get(ADLS_CONNECTION_STRING) {
+ cfg.connection_string = Some(connection_string.clone());
+ }
+ if let Some(account_name) = props.get(ADLS_ACCOUNT_NAME) {
+ cfg.account_name = Some(account_name.clone());
+ }
+ if let Some(account_key) = props.get(ADLS_ACCOUNT_KEY) {
+ cfg.account_key = Some(account_key.clone());
+ }
+ if let Some(sas_token) = props.get(ADLS_SAS_TOKEN) {
+ cfg.sas_token = Some(sas_token.clone());
+ }
+ if let Some(tenant_id) = props.get(ADLS_TENANT_ID) {
+ cfg.tenant_id = Some(tenant_id.clone());
+ }
+ if let Some(client_id) = props.get(ADLS_CLIENT_ID) {
+ cfg.client_id = Some(client_id.clone());
+ }
+ if let Some(client_secret) = props.get(ADLS_CLIENT_SECRET) {
+ cfg.client_secret = Some(client_secret.clone());
+ }
+ if let Some(authority_host) = props.get(ADLS_AUTHORITY_HOST) {
+ cfg.authority_host = Some(authority_host.clone());
+ }
+
+ Ok(cfg)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_azdls_config_builder() {
+ let config = AzdlsConfig::builder()
+ .account_name("myaccount")
+ .account_key("my-account-key")
+ .build();
+
+ assert_eq!(config.account_name.as_deref(), Some("myaccount"));
+ assert_eq!(config.account_key.as_deref(), Some("my-account-key"));
+ }
+
+ #[test]
+ fn test_azdls_config_from_storage_config() {
+ let storage_config = StorageConfig::new()
+ .with_prop(ADLS_ACCOUNT_NAME, "myaccount")
+ .with_prop(ADLS_ACCOUNT_KEY, "my-account-key");
+
+ let azdls_config = AzdlsConfig::try_from(&storage_config).unwrap();
+
+ assert_eq!(azdls_config.account_name.as_deref(), Some("myaccount"));
+ assert_eq!(azdls_config.account_key.as_deref(),
Some("my-account-key"));
+ }
+
+ #[test]
+ fn test_azdls_config_with_sas_token() {
+ let storage_config = StorageConfig::new()
+ .with_prop(ADLS_ACCOUNT_NAME, "myaccount")
+ .with_prop(ADLS_SAS_TOKEN, "my-sas-token");
+
+ let azdls_config = AzdlsConfig::try_from(&storage_config).unwrap();
+
+ assert_eq!(azdls_config.account_name.as_deref(), Some("myaccount"));
+ assert_eq!(azdls_config.sas_token.as_deref(), Some("my-sas-token"));
+ }
+
+ #[test]
+ fn test_azdls_config_with_client_credentials() {
+ let storage_config = StorageConfig::new()
+ .with_prop(ADLS_ACCOUNT_NAME, "myaccount")
+ .with_prop(ADLS_TENANT_ID, "my-tenant")
+ .with_prop(ADLS_CLIENT_ID, "my-client")
+ .with_prop(ADLS_CLIENT_SECRET, "my-secret");
+
+ let azdls_config = AzdlsConfig::try_from(&storage_config).unwrap();
+
+ assert_eq!(azdls_config.account_name.as_deref(), Some("myaccount"));
+ assert_eq!(azdls_config.tenant_id.as_deref(), Some("my-tenant"));
+ assert_eq!(azdls_config.client_id.as_deref(), Some("my-client"));
+ assert_eq!(azdls_config.client_secret.as_deref(), Some("my-secret"));
+ }
+}
diff --git a/crates/iceberg/src/io/config/gcs.rs
b/crates/iceberg/src/io/config/gcs.rs
new file mode 100644
index 000000000..5b11567f4
--- /dev/null
+++ b/crates/iceberg/src/io/config/gcs.rs
@@ -0,0 +1,189 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Google Cloud Storage configuration.
+//!
+//! This module provides configuration constants and types for Google Cloud
Storage.
+//! Reference:
https://github.com/apache/iceberg/blob/main/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java
+
+use serde::{Deserialize, Serialize};
+use typed_builder::TypedBuilder;
+
+use super::StorageConfig;
+use crate::Result;
+use crate::io::is_truthy;
+
+/// Google Cloud Project ID.
+pub const GCS_PROJECT_ID: &str = "gcs.project-id";
+/// Google Cloud Storage endpoint.
+pub const GCS_SERVICE_PATH: &str = "gcs.service.path";
+/// Google Cloud user project.
+pub const GCS_USER_PROJECT: &str = "gcs.user-project";
+/// Allow unauthenticated requests.
+pub const GCS_NO_AUTH: &str = "gcs.no-auth";
+/// Google Cloud Storage credentials JSON string, base64 encoded.
+///
+/// E.g.
base64::prelude::BASE64_STANDARD.encode(serde_json::to_string(credential).as_bytes())
+pub const GCS_CREDENTIALS_JSON: &str = "gcs.credentials-json";
+/// Google Cloud Storage token.
+pub const GCS_TOKEN: &str = "gcs.oauth2.token";
+/// Option to skip signing requests (e.g. for public buckets/folders).
+pub const GCS_ALLOW_ANONYMOUS: &str = "gcs.allow-anonymous";
+/// Option to skip loading the credential from GCE metadata server.
+pub const GCS_DISABLE_VM_METADATA: &str = "gcs.disable-vm-metadata";
+/// Option to skip loading configuration from config file and the env.
+pub const GCS_DISABLE_CONFIG_LOAD: &str = "gcs.disable-config-load";
+
+/// Google Cloud Storage configuration.
+///
+/// This struct contains all the configuration options for connecting to
Google Cloud Storage.
+/// Use the builder pattern via `GcsConfig::builder()` to construct instances.
+/// ```
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize,
TypedBuilder)]
+pub struct GcsConfig {
+ /// Google Cloud Project ID.
+ #[builder(default, setter(strip_option, into))]
+ pub project_id: Option<String>,
+ /// GCS service endpoint.
+ #[builder(default, setter(strip_option, into))]
+ pub endpoint: Option<String>,
+ /// User project for requester pays buckets.
+ #[builder(default, setter(strip_option, into))]
+ pub user_project: Option<String>,
+ /// Credentials JSON (base64 encoded).
+ #[builder(default, setter(strip_option, into))]
+ pub credential: Option<String>,
+ /// OAuth2 token.
+ #[builder(default, setter(strip_option, into))]
+ pub token: Option<String>,
+ /// Allow anonymous access.
+ #[builder(default)]
+ pub allow_anonymous: bool,
+ /// Disable VM metadata.
+ #[builder(default)]
+ pub disable_vm_metadata: bool,
+ /// Disable config load.
+ #[builder(default)]
+ pub disable_config_load: bool,
+}
+
+impl TryFrom<&StorageConfig> for GcsConfig {
+ type Error = crate::Error;
+
+ fn try_from(config: &StorageConfig) -> Result<Self> {
+ let props = config.props();
+
+ let mut cfg = GcsConfig::default();
+
+ if let Some(project_id) = props.get(GCS_PROJECT_ID) {
+ cfg.project_id = Some(project_id.clone());
+ }
+ if let Some(endpoint) = props.get(GCS_SERVICE_PATH) {
+ cfg.endpoint = Some(endpoint.clone());
+ }
+ if let Some(user_project) = props.get(GCS_USER_PROJECT) {
+ cfg.user_project = Some(user_project.clone());
+ }
+ if let Some(credential) = props.get(GCS_CREDENTIALS_JSON) {
+ cfg.credential = Some(credential.clone());
+ }
+ if let Some(token) = props.get(GCS_TOKEN) {
+ cfg.token = Some(token.clone());
+ }
+
+ // GCS_NO_AUTH enables all anonymous/no-auth options
+ if props.get(GCS_NO_AUTH).is_some() {
+ cfg.allow_anonymous = true;
+ cfg.disable_vm_metadata = true;
+ cfg.disable_config_load = true;
+ }
+
+ if let Some(allow_anonymous) = props.get(GCS_ALLOW_ANONYMOUS)
+ && is_truthy(allow_anonymous.to_lowercase().as_str())
+ {
+ cfg.allow_anonymous = true;
+ }
+ if let Some(disable_vm_metadata) = props.get(GCS_DISABLE_VM_METADATA)
+ && is_truthy(disable_vm_metadata.to_lowercase().as_str())
+ {
+ cfg.disable_vm_metadata = true;
+ }
+ if let Some(disable_config_load) = props.get(GCS_DISABLE_CONFIG_LOAD)
+ && is_truthy(disable_config_load.to_lowercase().as_str())
+ {
+ cfg.disable_config_load = true;
+ }
+
+ Ok(cfg)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_gcs_config_builder() {
+ let config = GcsConfig::builder()
+ .project_id("my-project")
+ .credential("base64-creds")
+ .endpoint("http://localhost:4443";)
+ .build();
+
+ assert_eq!(config.project_id.as_deref(), Some("my-project"));
+ assert_eq!(config.credential.as_deref(), Some("base64-creds"));
+ assert_eq!(config.endpoint.as_deref(), Some("http://localhost:4443";));
+ }
+
+ #[test]
+ fn test_gcs_config_from_storage_config() {
+ let storage_config = StorageConfig::new()
+ .with_prop(GCS_PROJECT_ID, "my-project")
+ .with_prop(GCS_CREDENTIALS_JSON, "base64-creds")
+ .with_prop(GCS_SERVICE_PATH, "http://localhost:4443";);
+
+ let gcs_config = GcsConfig::try_from(&storage_config).unwrap();
+
+ assert_eq!(gcs_config.project_id.as_deref(), Some("my-project"));
+ assert_eq!(gcs_config.credential.as_deref(), Some("base64-creds"));
+ assert_eq!(
+ gcs_config.endpoint.as_deref(),
+ Some("http://localhost:4443";)
+ );
+ }
+
+ #[test]
+ fn test_gcs_config_no_auth() {
+ let storage_config = StorageConfig::new().with_prop(GCS_NO_AUTH,
"true");
+
+ let gcs_config = GcsConfig::try_from(&storage_config).unwrap();
+
+ assert!(gcs_config.allow_anonymous);
+ assert!(gcs_config.disable_vm_metadata);
+ assert!(gcs_config.disable_config_load);
+ }
+
+ #[test]
+ fn test_gcs_config_allow_anonymous() {
+ let storage_config =
StorageConfig::new().with_prop(GCS_ALLOW_ANONYMOUS, "true");
+
+ let gcs_config = GcsConfig::try_from(&storage_config).unwrap();
+
+ assert!(gcs_config.allow_anonymous);
+ assert!(!gcs_config.disable_vm_metadata);
+ }
+}
diff --git a/crates/iceberg/src/io/config/mod.rs
b/crates/iceberg/src/io/config/mod.rs
index 648e8baf4..cbdb53730 100644
--- a/crates/iceberg/src/io/config/mod.rs
+++ b/crates/iceberg/src/io/config/mod.rs
@@ -30,19 +30,17 @@
//! - [`OssConfig`]: Alibaba Cloud OSS specific configuration
//! - [`AzdlsConfig`]: Azure Data Lake Storage specific configuration
-// TODO Add specific configs
-// mod azdls;
-// mod gcs;
-// mod oss;
-// mod s3;
+mod azdls;
+mod gcs;
+mod oss;
+mod s3;
use std::collections::HashMap;
-// TODO Add specific configs
-// pub use azdls::*;
-// pub use gcs::*;
-// pub use oss::*;
-// pub use s3::*;
+pub use azdls::*;
+pub use gcs::*;
+pub use oss::*;
+pub use s3::*;
use serde::{Deserialize, Serialize};
/// Configuration properties for storage backends.
diff --git a/crates/iceberg/src/io/config/oss.rs
b/crates/iceberg/src/io/config/oss.rs
new file mode 100644
index 000000000..986710ef5
--- /dev/null
+++ b/crates/iceberg/src/io/config/oss.rs
@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Alibaba Cloud OSS storage configuration.
+//!
+//! This module provides configuration constants and types for Alibaba Cloud
OSS storage.
+
+use serde::{Deserialize, Serialize};
+use typed_builder::TypedBuilder;
+
+use super::StorageConfig;
+use crate::Result;
+
+/// Aliyun OSS endpoint.
+pub const OSS_ENDPOINT: &str = "oss.endpoint";
+/// Aliyun OSS access key ID.
+pub const OSS_ACCESS_KEY_ID: &str = "oss.access-key-id";
+/// Aliyun OSS access key secret.
+pub const OSS_ACCESS_KEY_SECRET: &str = "oss.access-key-secret";
+
+/// Alibaba Cloud OSS storage configuration.
+///
+/// This struct contains all the configuration options for connecting to
Alibaba Cloud OSS.
+/// Use the builder pattern via `OssConfig::builder()` to construct instances.
+/// ```
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize,
TypedBuilder)]
+pub struct OssConfig {
+ /// OSS endpoint URL.
+ #[builder(default, setter(strip_option, into))]
+ pub endpoint: Option<String>,
+ /// OSS access key ID.
+ #[builder(default, setter(strip_option, into))]
+ pub access_key_id: Option<String>,
+ /// OSS access key secret.
+ #[builder(default, setter(strip_option, into))]
+ pub access_key_secret: Option<String>,
+}
+
+impl TryFrom<&StorageConfig> for OssConfig {
+ type Error = crate::Error;
+
+ fn try_from(config: &StorageConfig) -> Result<Self> {
+ let props = config.props();
+
+ let mut cfg = OssConfig::default();
+ if let Some(endpoint) = props.get(OSS_ENDPOINT) {
+ cfg.endpoint = Some(endpoint.clone());
+ }
+ if let Some(access_key_id) = props.get(OSS_ACCESS_KEY_ID) {
+ cfg.access_key_id = Some(access_key_id.clone());
+ }
+ if let Some(access_key_secret) = props.get(OSS_ACCESS_KEY_SECRET) {
+ cfg.access_key_secret = Some(access_key_secret.clone());
+ }
+
+ Ok(cfg)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_oss_config_builder() {
+ let config = OssConfig::builder()
+ .endpoint("https://oss-cn-hangzhou.aliyuncs.com";)
+ .access_key_id("my-access-key")
+ .access_key_secret("my-secret-key")
+ .build();
+
+ assert_eq!(
+ config.endpoint.as_deref(),
+ Some("https://oss-cn-hangzhou.aliyuncs.com";)
+ );
+ assert_eq!(config.access_key_id.as_deref(), Some("my-access-key"));
+ assert_eq!(config.access_key_secret.as_deref(), Some("my-secret-key"));
+ }
+
+ #[test]
+ fn test_oss_config_from_storage_config() {
+ let storage_config = StorageConfig::new()
+ .with_prop(OSS_ENDPOINT, "https://oss-cn-hangzhou.aliyuncs.com";)
+ .with_prop(OSS_ACCESS_KEY_ID, "my-access-key")
+ .with_prop(OSS_ACCESS_KEY_SECRET, "my-secret-key");
+
+ let oss_config = OssConfig::try_from(&storage_config).unwrap();
+
+ assert_eq!(
+ oss_config.endpoint.as_deref(),
+ Some("https://oss-cn-hangzhou.aliyuncs.com";)
+ );
+ assert_eq!(oss_config.access_key_id.as_deref(), Some("my-access-key"));
+ assert_eq!(
+ oss_config.access_key_secret.as_deref(),
+ Some("my-secret-key")
+ );
+ }
+
+ #[test]
+ fn test_oss_config_empty() {
+ let storage_config = StorageConfig::new();
+
+ let oss_config = OssConfig::try_from(&storage_config).unwrap();
+
+ assert_eq!(oss_config.endpoint, None);
+ assert_eq!(oss_config.access_key_id, None);
+ assert_eq!(oss_config.access_key_secret, None);
+ }
+}
diff --git a/crates/iceberg/src/io/config/s3.rs
b/crates/iceberg/src/io/config/s3.rs
new file mode 100644
index 000000000..fae3a1475
--- /dev/null
+++ b/crates/iceberg/src/io/config/s3.rs
@@ -0,0 +1,303 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Amazon S3 storage configuration.
+//!
+//! This module provides configuration constants and types for Amazon S3
storage.
+//! These are based on the [Iceberg S3 FileIO
configuration](https://py.iceberg.apache.org/configuration/#s3).
+
+use serde::{Deserialize, Serialize};
+use typed_builder::TypedBuilder;
+
+use super::StorageConfig;
+use crate::io::is_truthy;
+use crate::{Error, ErrorKind, Result};
+
+/// S3 endpoint URL.
+pub const S3_ENDPOINT: &str = "s3.endpoint";
+/// S3 access key ID.
+pub const S3_ACCESS_KEY_ID: &str = "s3.access-key-id";
+/// S3 secret access key.
+pub const S3_SECRET_ACCESS_KEY: &str = "s3.secret-access-key";
+/// S3 session token (required when using temporary credentials).
+pub const S3_SESSION_TOKEN: &str = "s3.session-token";
+/// S3 region.
+pub const S3_REGION: &str = "s3.region";
+/// Region to use for the S3 client (takes precedence over [`S3_REGION`]).
+pub const CLIENT_REGION: &str = "client.region";
+/// S3 Path Style Access.
+pub const S3_PATH_STYLE_ACCESS: &str = "s3.path-style-access";
+/// S3 Server Side Encryption Type.
+pub const S3_SSE_TYPE: &str = "s3.sse.type";
+/// S3 Server Side Encryption Key.
+/// If S3 encryption type is kms, input is a KMS Key ID.
+/// In case this property is not set, default key "aws/s3" is used.
+/// If encryption type is custom, input is a custom base-64 AES256 symmetric
key.
+pub const S3_SSE_KEY: &str = "s3.sse.key";
+/// S3 Server Side Encryption MD5.
+pub const S3_SSE_MD5: &str = "s3.sse.md5";
+/// If set, all AWS clients will assume a role of the given ARN, instead of
using the default
+/// credential chain.
+pub const S3_ASSUME_ROLE_ARN: &str = "client.assume-role.arn";
+/// Optional external ID used to assume an IAM role.
+pub const S3_ASSUME_ROLE_EXTERNAL_ID: &str = "client.assume-role.external-id";
+/// Optional session name used to assume an IAM role.
+pub const S3_ASSUME_ROLE_SESSION_NAME: &str =
"client.assume-role.session-name";
+/// Option to skip signing requests (e.g. for public buckets/folders).
+pub const S3_ALLOW_ANONYMOUS: &str = "s3.allow-anonymous";
+/// Option to skip loading the credential from EC2 metadata (typically used in
conjunction with
+/// `S3_ALLOW_ANONYMOUS`).
+pub const S3_DISABLE_EC2_METADATA: &str = "s3.disable-ec2-metadata";
+/// Option to skip loading configuration from config file and the env.
+pub const S3_DISABLE_CONFIG_LOAD: &str = "s3.disable-config-load";
+
+/// Amazon S3 storage configuration.
+///
+/// This struct contains all the configuration options for connecting to
Amazon S3.
+/// Use the builder pattern via `S3Config::builder()` to construct instances.
+/// ```
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize,
TypedBuilder)]
+pub struct S3Config {
+ /// S3 endpoint URL.
+ #[builder(default, setter(strip_option, into))]
+ pub endpoint: Option<String>,
+ /// S3 access key ID.
+ #[builder(default, setter(strip_option, into))]
+ pub access_key_id: Option<String>,
+ /// S3 secret access key.
+ #[builder(default, setter(strip_option, into))]
+ pub secret_access_key: Option<String>,
+ /// S3 session token.
+ #[builder(default, setter(strip_option, into))]
+ pub session_token: Option<String>,
+ /// S3 region.
+ #[builder(default, setter(strip_option, into))]
+ pub region: Option<String>,
+ /// Enable virtual host style (opposite of path style access).
+ #[builder(default)]
+ pub enable_virtual_host_style: bool,
+ /// Server side encryption type.
+ #[builder(default, setter(strip_option, into))]
+ pub server_side_encryption: Option<String>,
+ /// Server side encryption AWS KMS key ID.
+ #[builder(default, setter(strip_option, into))]
+ pub server_side_encryption_aws_kms_key_id: Option<String>,
+ /// Server side encryption customer algorithm.
+ #[builder(default, setter(strip_option, into))]
+ pub server_side_encryption_customer_algorithm: Option<String>,
+ /// Server side encryption customer key.
+ #[builder(default, setter(strip_option, into))]
+ pub server_side_encryption_customer_key: Option<String>,
+ /// Server side encryption customer key MD5.
+ #[builder(default, setter(strip_option, into))]
+ pub server_side_encryption_customer_key_md5: Option<String>,
+ /// Role ARN for assuming a role.
+ #[builder(default, setter(strip_option, into))]
+ pub role_arn: Option<String>,
+ /// External ID for assuming a role.
+ #[builder(default, setter(strip_option, into))]
+ pub external_id: Option<String>,
+ /// Session name for assuming a role.
+ #[builder(default, setter(strip_option, into))]
+ pub role_session_name: Option<String>,
+ /// Allow anonymous access.
+ #[builder(default)]
+ pub allow_anonymous: bool,
+ /// Disable EC2 metadata.
+ #[builder(default)]
+ pub disable_ec2_metadata: bool,
+ /// Disable config load.
+ #[builder(default)]
+ pub disable_config_load: bool,
+}
+
+impl TryFrom<&StorageConfig> for S3Config {
+ type Error = crate::Error;
+
+ fn try_from(config: &StorageConfig) -> Result<Self> {
+ let props = config.props();
+
+ let mut cfg = S3Config::default();
+
+ if let Some(endpoint) = props.get(S3_ENDPOINT) {
+ cfg.endpoint = Some(endpoint.clone());
+ }
+ if let Some(access_key_id) = props.get(S3_ACCESS_KEY_ID) {
+ cfg.access_key_id = Some(access_key_id.clone());
+ }
+ if let Some(secret_access_key) = props.get(S3_SECRET_ACCESS_KEY) {
+ cfg.secret_access_key = Some(secret_access_key.clone());
+ }
+ if let Some(session_token) = props.get(S3_SESSION_TOKEN) {
+ cfg.session_token = Some(session_token.clone());
+ }
+ if let Some(region) = props.get(S3_REGION) {
+ cfg.region = Some(region.clone());
+ }
+ // CLIENT_REGION takes precedence over S3_REGION
+ if let Some(region) = props.get(CLIENT_REGION) {
+ cfg.region = Some(region.clone());
+ }
+ if let Some(path_style_access) = props.get(S3_PATH_STYLE_ACCESS) {
+ cfg.enable_virtual_host_style =
!is_truthy(path_style_access.to_lowercase().as_str());
+ }
+ if let Some(arn) = props.get(S3_ASSUME_ROLE_ARN) {
+ cfg.role_arn = Some(arn.clone());
+ }
+ if let Some(external_id) = props.get(S3_ASSUME_ROLE_EXTERNAL_ID) {
+ cfg.external_id = Some(external_id.clone());
+ }
+ if let Some(session_name) = props.get(S3_ASSUME_ROLE_SESSION_NAME) {
+ cfg.role_session_name = Some(session_name.clone());
+ }
+
+ // Handle SSE configuration
+ let s3_sse_key = props.get(S3_SSE_KEY).cloned();
+ if let Some(sse_type) = props.get(S3_SSE_TYPE) {
+ match sse_type.to_lowercase().as_str() {
+ // No Server Side Encryption
+ "none" => {}
+ // S3 SSE-S3 encryption (S3 managed keys).
https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html
+ "s3" => {
+ cfg.server_side_encryption = Some("AES256".to_string());
+ }
+ // S3 SSE KMS, either using default or custom KMS key.
https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
+ "kms" => {
+ cfg.server_side_encryption = Some("aws:kms".to_string());
+ cfg.server_side_encryption_aws_kms_key_id = s3_sse_key;
+ }
+ // S3 SSE-C, using customer managed keys.
https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
+ "custom" => {
+ cfg.server_side_encryption_customer_algorithm =
Some("AES256".to_string());
+ cfg.server_side_encryption_customer_key = s3_sse_key;
+ cfg.server_side_encryption_customer_key_md5 =
props.get(S3_SSE_MD5).cloned();
+ }
+ _ => {
+ return Err(Error::new(
+ ErrorKind::DataInvalid,
+ format!(
+ "Invalid {S3_SSE_TYPE}: {sse_type}. Expected one
of (custom, kms, s3, none)"
+ ),
+ ));
+ }
+ }
+ }
+
+ if let Some(allow_anonymous) = props.get(S3_ALLOW_ANONYMOUS)
+ && is_truthy(allow_anonymous.to_lowercase().as_str())
+ {
+ cfg.allow_anonymous = true;
+ }
+ if let Some(disable_ec2_metadata) = props.get(S3_DISABLE_EC2_METADATA)
+ && is_truthy(disable_ec2_metadata.to_lowercase().as_str())
+ {
+ cfg.disable_ec2_metadata = true;
+ }
+ if let Some(disable_config_load) = props.get(S3_DISABLE_CONFIG_LOAD)
+ && is_truthy(disable_config_load.to_lowercase().as_str())
+ {
+ cfg.disable_config_load = true;
+ }
+
+ Ok(cfg)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_s3_config_builder() {
+ let config = S3Config::builder()
+ .region("us-east-1")
+ .access_key_id("my-access-key")
+ .secret_access_key("my-secret-key")
+ .endpoint("http://localhost:9000";)
+ .build();
+
+ assert_eq!(config.region.as_deref(), Some("us-east-1"));
+ assert_eq!(config.access_key_id.as_deref(), Some("my-access-key"));
+ assert_eq!(config.secret_access_key.as_deref(), Some("my-secret-key"));
+ assert_eq!(config.endpoint.as_deref(), Some("http://localhost:9000";));
+ }
+
+ #[test]
+ fn test_s3_config_from_storage_config() {
+ let storage_config = StorageConfig::new()
+ .with_prop(S3_REGION, "us-east-1")
+ .with_prop(S3_ACCESS_KEY_ID, "my-access-key")
+ .with_prop(S3_SECRET_ACCESS_KEY, "my-secret-key")
+ .with_prop(S3_ENDPOINT, "http://localhost:9000";);
+
+ let s3_config = S3Config::try_from(&storage_config).unwrap();
+
+ assert_eq!(s3_config.region.as_deref(), Some("us-east-1"));
+ assert_eq!(s3_config.access_key_id.as_deref(), Some("my-access-key"));
+ assert_eq!(
+ s3_config.secret_access_key.as_deref(),
+ Some("my-secret-key")
+ );
+ assert_eq!(s3_config.endpoint.as_deref(),
Some("http://localhost:9000";));
+ }
+
+ #[test]
+ fn test_s3_config_client_region_precedence() {
+ let storage_config = StorageConfig::new()
+ .with_prop(S3_REGION, "us-east-1")
+ .with_prop(CLIENT_REGION, "eu-west-1");
+
+ let s3_config = S3Config::try_from(&storage_config).unwrap();
+
+ // CLIENT_REGION should take precedence
+ assert_eq!(s3_config.region.as_deref(), Some("eu-west-1"));
+ }
+
+ #[test]
+ fn test_s3_config_path_style_access() {
+ let storage_config =
StorageConfig::new().with_prop(S3_PATH_STYLE_ACCESS, "true");
+
+ let s3_config = S3Config::try_from(&storage_config).unwrap();
+
+ // path style access = true means virtual host style = false
+ assert!(!s3_config.enable_virtual_host_style);
+ }
+
+ #[test]
+ fn test_s3_config_sse_kms() {
+ let storage_config = StorageConfig::new()
+ .with_prop(S3_SSE_TYPE, "kms")
+ .with_prop(S3_SSE_KEY, "my-kms-key-id");
+
+ let s3_config = S3Config::try_from(&storage_config).unwrap();
+
+ assert_eq!(s3_config.server_side_encryption.as_deref(),
Some("aws:kms"));
+ assert_eq!(
+ s3_config.server_side_encryption_aws_kms_key_id.as_deref(),
+ Some("my-kms-key-id")
+ );
+ }
+
+ #[test]
+ fn test_s3_config_allow_anonymous() {
+ let storage_config =
StorageConfig::new().with_prop(S3_ALLOW_ANONYMOUS, "true");
+
+ let s3_config = S3Config::try_from(&storage_config).unwrap();
+
+ assert!(s3_config.allow_anonymous);
+ }
+}
diff --git a/crates/iceberg/src/io/mod.rs b/crates/iceberg/src/io/mod.rs
index 55f2f262c..8e40bae97 100644
--- a/crates/iceberg/src/io/mod.rs
+++ b/crates/iceberg/src/io/mod.rs
@@ -70,7 +70,9 @@ mod config;
mod file_io;
mod storage;
+pub use config::*;
pub use file_io::*;
+pub use storage::{Storage, StorageFactory};
pub(crate) mod object_cache;
#[cfg(feature = "storage-azdls")]
@@ -86,18 +88,16 @@ mod storage_oss;
#[cfg(feature = "storage-s3")]
mod storage_s3;
-pub use config::*;
-pub use storage::{Storage, StorageFactory};
#[cfg(feature = "storage-azdls")]
-pub use storage_azdls::*;
+use storage_azdls::*;
#[cfg(feature = "storage-fs")]
use storage_fs::*;
#[cfg(feature = "storage-gcs")]
-pub use storage_gcs::*;
+use storage_gcs::*;
#[cfg(feature = "storage-memory")]
use storage_memory::*;
#[cfg(feature = "storage-oss")]
-pub use storage_oss::*;
+use storage_oss::*;
#[cfg(feature = "storage-s3")]
pub use storage_s3::*;
diff --git a/crates/iceberg/src/io/storage_azdls.rs
b/crates/iceberg/src/io/storage_azdls.rs
index 5abb0cd6e..ce2b7427b 100644
--- a/crates/iceberg/src/io/storage_azdls.rs
+++ b/crates/iceberg/src/io/storage_azdls.rs
@@ -23,37 +23,12 @@ use opendal::Configurator;
use opendal::services::AzdlsConfig;
use url::Url;
+use crate::io::config::{
+ ADLS_ACCOUNT_KEY, ADLS_ACCOUNT_NAME, ADLS_AUTHORITY_HOST, ADLS_CLIENT_ID,
ADLS_CLIENT_SECRET,
+ ADLS_CONNECTION_STRING, ADLS_SAS_TOKEN, ADLS_TENANT_ID,
+};
use crate::{Error, ErrorKind, Result, ensure_data_valid};
-/// A connection string.
-///
-/// Note, this string is parsed first, and any other passed adls.* properties
-/// will override values from the connection string.
-const ADLS_CONNECTION_STRING: &str = "adls.connection-string";
-
-/// The account that you want to connect to.
-pub const ADLS_ACCOUNT_NAME: &str = "adls.account-name";
-
-/// The key to authentication against the account.
-pub const ADLS_ACCOUNT_KEY: &str = "adls.account-key";
-
-/// The shared access signature.
-pub const ADLS_SAS_TOKEN: &str = "adls.sas-token";
-
-/// The tenant-id.
-pub const ADLS_TENANT_ID: &str = "adls.tenant-id";
-
-/// The client-id.
-pub const ADLS_CLIENT_ID: &str = "adls.client-id";
-
-/// The client-secret.
-pub const ADLS_CLIENT_SECRET: &str = "adls.client-secret";
-
-/// The authority host of the service principal.
-/// - required for client_credentials authentication
-/// - default value: `https://login.microsoftonline.com`
-pub const ADLS_AUTHORITY_HOST: &str = "adls.authority-host";
-
/// Parses adls.* prefixed configuration properties.
pub(crate) fn azdls_config_parse(mut properties: HashMap<String, String>) ->
Result<AzdlsConfig> {
let mut config = AzdlsConfig::default();
diff --git a/crates/iceberg/src/io/storage_gcs.rs
b/crates/iceberg/src/io/storage_gcs.rs
index 7718df603..5c6145d32 100644
--- a/crates/iceberg/src/io/storage_gcs.rs
+++ b/crates/iceberg/src/io/storage_gcs.rs
@@ -22,33 +22,13 @@ use opendal::Operator;
use opendal::services::GcsConfig;
use url::Url;
+use crate::io::config::{
+ GCS_ALLOW_ANONYMOUS, GCS_CREDENTIALS_JSON, GCS_DISABLE_CONFIG_LOAD,
GCS_DISABLE_VM_METADATA,
+ GCS_NO_AUTH, GCS_SERVICE_PATH, GCS_TOKEN,
+};
use crate::io::is_truthy;
use crate::{Error, ErrorKind, Result};
-// Reference:
https://github.com/apache/iceberg/blob/main/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java
-
-/// Google Cloud Project ID
-pub const GCS_PROJECT_ID: &str = "gcs.project-id";
-/// Google Cloud Storage endpoint
-pub const GCS_SERVICE_PATH: &str = "gcs.service.path";
-/// Google Cloud user project
-pub const GCS_USER_PROJECT: &str = "gcs.user-project";
-/// Allow unauthenticated requests
-pub const GCS_NO_AUTH: &str = "gcs.no-auth";
-/// Google Cloud Storage credentials JSON string, base64 encoded.
-///
-/// E.g.
base64::prelude::BASE64_STANDARD.encode(serde_json::to_string(credential).as_bytes())
-pub const GCS_CREDENTIALS_JSON: &str = "gcs.credentials-json";
-/// Google Cloud Storage token
-pub const GCS_TOKEN: &str = "gcs.oauth2.token";
-
-/// Option to skip signing requests (e.g. for public buckets/folders).
-pub const GCS_ALLOW_ANONYMOUS: &str = "gcs.allow-anonymous";
-/// Option to skip loading the credential from GCE metadata server (typically
used in conjunction with `GCS_ALLOW_ANONYMOUS`).
-pub const GCS_DISABLE_VM_METADATA: &str = "gcs.disable-vm-metadata";
-/// Option to skip loading configuration from config file and the env.
-pub const GCS_DISABLE_CONFIG_LOAD: &str = "gcs.disable-config-load";
-
/// Parse iceberg properties to [`GcsConfig`].
pub(crate) fn gcs_config_parse(mut m: HashMap<String, String>) ->
Result<GcsConfig> {
let mut cfg = GcsConfig::default();
diff --git a/crates/iceberg/src/io/storage_oss.rs
b/crates/iceberg/src/io/storage_oss.rs
index e82dda23a..83fc1424a 100644
--- a/crates/iceberg/src/io/storage_oss.rs
+++ b/crates/iceberg/src/io/storage_oss.rs
@@ -21,19 +21,9 @@ use opendal::services::OssConfig;
use opendal::{Configurator, Operator};
use url::Url;
+use crate::io::config::{OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET,
OSS_ENDPOINT};
use crate::{Error, ErrorKind, Result};
-/// Required configuration arguments for creating an Aliyun OSS Operator with
OpenDAL:
-/// - `oss.endpoint`: The OSS service endpoint URL
-/// - `oss.access-key-id`: The access key ID for authentication
-/// - `oss.access-key-secret`: The access key secret for authentication
-/// Aliyun oss endpoint.
-pub const OSS_ENDPOINT: &str = "oss.endpoint";
-/// Aliyun oss access key id.
-pub const OSS_ACCESS_KEY_ID: &str = "oss.access-key-id";
-/// Aliyun oss access key secret.
-pub const OSS_ACCESS_KEY_SECRET: &str = "oss.access-key-secret";
-
/// Parse iceberg props to oss config.
pub(crate) fn oss_config_parse(mut m: HashMap<String, String>) ->
Result<OssConfig> {
let mut cfg: OssConfig = OssConfig::default();
diff --git a/crates/iceberg/src/io/storage_s3.rs
b/crates/iceberg/src/io/storage_s3.rs
index f069e0e2f..bf7399e01 100644
--- a/crates/iceberg/src/io/storage_s3.rs
+++ b/crates/iceberg/src/io/storage_s3.rs
@@ -25,51 +25,15 @@ pub use reqsign::{AwsCredential, AwsCredentialLoad};
use reqwest::Client;
use url::Url;
+use crate::io::config::{
+ CLIENT_REGION, S3_ACCESS_KEY_ID, S3_ALLOW_ANONYMOUS, S3_ASSUME_ROLE_ARN,
+ S3_ASSUME_ROLE_EXTERNAL_ID, S3_ASSUME_ROLE_SESSION_NAME,
S3_DISABLE_CONFIG_LOAD,
+ S3_DISABLE_EC2_METADATA, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION,
S3_SECRET_ACCESS_KEY,
+ S3_SESSION_TOKEN, S3_SSE_KEY, S3_SSE_MD5, S3_SSE_TYPE,
+};
use crate::io::is_truthy;
use crate::{Error, ErrorKind, Result};
-/// Following are arguments for [s3 file
io](https://py.iceberg.apache.org/configuration/#s3).
-/// S3 endpoint.
-pub const S3_ENDPOINT: &str = "s3.endpoint";
-/// S3 access key id.
-pub const S3_ACCESS_KEY_ID: &str = "s3.access-key-id";
-/// S3 secret access key.
-pub const S3_SECRET_ACCESS_KEY: &str = "s3.secret-access-key";
-/// S3 session token.
-/// This is required when using temporary credentials.
-pub const S3_SESSION_TOKEN: &str = "s3.session-token";
-/// S3 region.
-pub const S3_REGION: &str = "s3.region";
-/// Region to use for the S3 client.
-///
-/// This takes precedence over [`S3_REGION`].
-pub const CLIENT_REGION: &str = "client.region";
-/// S3 Path Style Access.
-pub const S3_PATH_STYLE_ACCESS: &str = "s3.path-style-access";
-/// S3 Server Side Encryption Type.
-pub const S3_SSE_TYPE: &str = "s3.sse.type";
-/// S3 Server Side Encryption Key.
-/// If S3 encryption type is kms, input is a KMS Key ID.
-/// In case this property is not set, default key "aws/s3" is used.
-/// If encryption type is custom, input is a custom base-64 AES256 symmetric
key.
-pub const S3_SSE_KEY: &str = "s3.sse.key";
-/// S3 Server Side Encryption MD5.
-pub const S3_SSE_MD5: &str = "s3.sse.md5";
-/// If set, all AWS clients will assume a role of the given ARN, instead of
using the default
-/// credential chain.
-pub const S3_ASSUME_ROLE_ARN: &str = "client.assume-role.arn";
-/// Optional external ID used to assume an IAM role.
-pub const S3_ASSUME_ROLE_EXTERNAL_ID: &str = "client.assume-role.external-id";
-/// Optional session name used to assume an IAM role.
-pub const S3_ASSUME_ROLE_SESSION_NAME: &str =
"client.assume-role.session-name";
-/// Option to skip signing requests (e.g. for public buckets/folders).
-pub const S3_ALLOW_ANONYMOUS: &str = "s3.allow-anonymous";
-/// Option to skip loading the credential from EC2 metadata (typically used in
conjunction with
-/// `S3_ALLOW_ANONYMOUS`).
-pub const S3_DISABLE_EC2_METADATA: &str = "s3.disable-ec2-metadata";
-/// Option to skip loading configuration from config file and the env.
-pub const S3_DISABLE_CONFIG_LOAD: &str = "s3.disable-config-load";
-
/// Parse iceberg props to s3 config.
pub(crate) fn s3_config_parse(mut m: HashMap<String, String>) ->
Result<S3Config> {
let mut cfg = S3Config::default();
