This is an automated email from the ASF dual-hosted git repository.
mneumann pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs-object-store.git
The following commit(s) were added to refs/heads/main by this push:
new bdcac43 feat: Add support for AWS_ENDPOINT_URL_S3 environment
variable (#590)
bdcac43 is described below
commit bdcac43fc637ac89cc80f18f4e16b3fb66fd0ec8
Author: Rajat Goel <[email protected]>
AuthorDate: Thu Mar 12 04:11:16 2026 -0700
feat: Add support for AWS_ENDPOINT_URL_S3 environment variable (#590)
* feat: Add support for AWS_ENDPOINT_URL_S3 env var
* cargo fmt
* comments
---
src/aws/builder.rs | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 50 insertions(+), 3 deletions(-)
diff --git a/src/aws/builder.rs b/src/aws/builder.rs
index b698cf0..8b767db 100644
--- a/src/aws/builder.rs
+++ b/src/aws/builder.rs
@@ -135,6 +135,8 @@ pub struct AmazonS3Builder {
bucket_name: Option<String>,
/// Endpoint for communicating with AWS S3
endpoint: Option<String>,
+ /// Service-specific S3 endpoint URL (takes precedence over endpoint)
+ s3_endpoint: Option<String>,
/// Token to use for requests
token: Option<String>,
/// Url
@@ -263,6 +265,14 @@ pub enum AmazonS3ConfigKey {
/// - `endpoint_url`
Endpoint,
+ /// Service-specific S3 endpoint URL
+ ///
+ /// When set, takes precedence over [`Endpoint`](Self::Endpoint) in the
build method.
+ ///
+ /// Supported keys:
+ /// - `aws_endpoint_url_s3`
+ S3Endpoint,
+
/// Token to use for requests (passed to underlying provider)
///
/// See [`AmazonS3Builder::with_token`] for details.
@@ -448,6 +458,7 @@ impl AsRef<str> for AmazonS3ConfigKey {
Self::Region => "aws_region",
Self::Bucket => "aws_bucket",
Self::Endpoint => "aws_endpoint",
+ Self::S3Endpoint => "aws_endpoint_url_s3",
Self::Token => "aws_session_token",
Self::ImdsV1Fallback => "aws_imdsv1_fallback",
Self::VirtualHostedStyleRequest =>
"aws_virtual_hosted_style_request",
@@ -485,6 +496,7 @@ impl FromStr for AmazonS3ConfigKey {
"aws_region" | "region" => Ok(Self::Region),
"aws_bucket" | "aws_bucket_name" | "bucket_name" | "bucket" =>
Ok(Self::Bucket),
"aws_endpoint_url" | "aws_endpoint" | "endpoint_url" | "endpoint"
=> Ok(Self::Endpoint),
+ "aws_endpoint_url_s3" => Ok(Self::S3Endpoint),
"aws_session_token" | "aws_token" | "session_token" | "token" =>
Ok(Self::Token),
"aws_virtual_hosted_style_request" |
"virtual_hosted_style_request" => {
Ok(Self::VirtualHostedStyleRequest)
@@ -552,6 +564,7 @@ impl AmazonS3Builder {
/// * `AWS_SECRET_ACCESS_KEY` -> secret_access_key
/// * `AWS_DEFAULT_REGION` -> region
/// * `AWS_ENDPOINT` -> endpoint
+ /// * `AWS_ENDPOINT_URL_S3` -> s3_endpoint (takes precedence over endpoint
in build)
/// * `AWS_SESSION_TOKEN` -> token
/// * `AWS_WEB_IDENTITY_TOKEN_FILE` -> path to file containing web
identity token for AssumeRoleWithWebIdentity
/// * `AWS_ROLE_ARN` -> ARN of the role to assume when using web identity
token
@@ -573,7 +586,6 @@ impl AmazonS3Builder {
/// ```
pub fn from_env() -> Self {
let mut builder: Self = Default::default();
-
for (os_key, os_value) in std::env::vars_os() {
if let (Some(key), Some(value)) = (os_key.to_str(),
os_value.to_str()) {
if key.starts_with("AWS_") {
@@ -583,7 +595,6 @@ impl AmazonS3Builder {
}
}
}
-
builder
}
@@ -620,6 +631,7 @@ impl AmazonS3Builder {
AmazonS3ConfigKey::Region => self.region = Some(value.into()),
AmazonS3ConfigKey::Bucket => self.bucket_name = Some(value.into()),
AmazonS3ConfigKey::Endpoint => self.endpoint = Some(value.into()),
+ AmazonS3ConfigKey::S3Endpoint => self.s3_endpoint =
Some(value.into()),
AmazonS3ConfigKey::Token => self.token = Some(value.into()),
AmazonS3ConfigKey::ImdsV1Fallback =>
self.imdsv1_fallback.parse(value),
AmazonS3ConfigKey::VirtualHostedStyleRequest => {
@@ -703,6 +715,7 @@ impl AmazonS3Builder {
AmazonS3ConfigKey::Region | AmazonS3ConfigKey::DefaultRegion =>
self.region.clone(),
AmazonS3ConfigKey::Bucket => self.bucket_name.clone(),
AmazonS3ConfigKey::Endpoint => self.endpoint.clone(),
+ AmazonS3ConfigKey::S3Endpoint => self.s3_endpoint.clone(),
AmazonS3ConfigKey::Token => self.token.clone(),
AmazonS3ConfigKey::ImdsV1Fallback =>
Some(self.imdsv1_fallback.to_string()),
AmazonS3ConfigKey::VirtualHostedStyleRequest => {
@@ -1192,10 +1205,13 @@ impl AmazonS3Builder {
false => (None, None),
};
+ // S3-specific endpoint takes precedence over generic endpoint
+ let endpoint = self.s3_endpoint.or(self.endpoint);
+
// If `endpoint` is provided it's assumed to be consistent with
`virtual_hosted_style_request` or `s3_express`.
// For example, if `virtual_hosted_style_request` is true then
`endpoint` should have bucket name included.
let virtual_hosted = self.virtual_hosted_style_request.get()?;
- let bucket_endpoint = match (&self.endpoint, zonal_endpoint,
virtual_hosted) {
+ let bucket_endpoint = match (&endpoint, zonal_endpoint,
virtual_hosted) {
(Some(endpoint), _, true) => endpoint.clone(),
(Some(endpoint), _, false) => format!("{}/{}",
endpoint.trim_end_matches("/"), bucket),
(None, Some(endpoint), _) => endpoint,
@@ -1487,6 +1503,37 @@ mod tests {
assert!(builder.unsigned_payload.get().unwrap());
}
+ #[test]
+ fn s3_test_endpoint_url_s3_config() {
+ // Verify aws_endpoint_url_s3 parses to S3Endpoint config key
+ let key: AmazonS3ConfigKey = "aws_endpoint_url_s3".parse().unwrap();
+ assert!(matches!(key, AmazonS3ConfigKey::S3Endpoint));
+
+ // Verify S3Endpoint takes precedence over Endpoint in build,
regardless of order
+ let s3 = AmazonS3Builder::new()
+ .with_config(AmazonS3ConfigKey::Endpoint,
"http://generic-endpoint";)
+ .with_config(AmazonS3ConfigKey::S3Endpoint,
"http://s3-specific-endpoint";)
+ .with_bucket_name("test-bucket")
+ .build()
+ .unwrap();
+ assert_eq!(
+ s3.client.config.bucket_endpoint,
+ "http://s3-specific-endpoint/test-bucket";
+ );
+
+ // Verify precedence works even when S3Endpoint is set first
+ let s3 = AmazonS3Builder::new()
+ .with_config(AmazonS3ConfigKey::S3Endpoint,
"http://s3-specific-endpoint";)
+ .with_config(AmazonS3ConfigKey::Endpoint,
"http://generic-endpoint";)
+ .with_bucket_name("test-bucket")
+ .build()
+ .unwrap();
+ assert_eq!(
+ s3.client.config.bucket_endpoint,
+ "http://s3-specific-endpoint/test-bucket";
+ );
+ }
+
#[test]
fn s3_test_config_get_value() {
let aws_access_key_id = "object_store:fake_access_key_id".to_string();
