This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new efe867a5a2 [object_store] Propagate env vars as object store client
options (#6334)
efe867a5a2 is described below
commit efe867a5a202f03846d8b6c737cb62ff16054940
Author: Costi Ciudatu <[email protected]>
AuthorDate: Wed Sep 4 12:00:02 2024 +0300
[object_store] Propagate env vars as object store client options (#6334)
* [object_store] Propagate env vars as object store client options
* [object_store] Include the missing variants in the FromStr implementation
of ClientConfigKey
* cargo fmt
---
object_store/src/aws/builder.rs | 15 ++++++++++++++-
object_store/src/azure/builder.rs | 15 ++++++++++++++-
object_store/src/client/mod.rs | 2 ++
object_store/src/gcp/builder.rs | 15 ++++++++++++++-
4 files changed, 44 insertions(+), 3 deletions(-)
diff --git a/object_store/src/aws/builder.rs b/object_store/src/aws/builder.rs
index 574345c389..75acb73e56 100644
--- a/object_store/src/aws/builder.rs
+++ b/object_store/src/aws/builder.rs
@@ -402,7 +402,7 @@ impl FromStr for AmazonS3ConfigKey {
"aws_sse_customer_key_base64" => Ok(Self::Encryption(
S3EncryptionConfigKey::CustomerEncryptionKey,
)),
- _ => match s.parse() {
+ _ => match s.strip_prefix("aws_").unwrap_or(s).parse() {
Ok(key) => Ok(Self::Client(key)),
Err(_) => Err(Error::UnknownConfigurationKey { key: s.into()
}.into()),
},
@@ -1455,4 +1455,17 @@ mod tests {
assert_eq!(parse_bucket_az(bucket), expected)
}
}
+
+ #[test]
+ fn aws_test_client_opts() {
+ let key = "AWS_PROXY_URL";
+ if let Ok(config_key) = key.to_ascii_lowercase().parse() {
+ assert_eq!(
+ AmazonS3ConfigKey::Client(ClientConfigKey::ProxyUrl),
+ config_key
+ );
+ } else {
+ panic!("{} not propagated as ClientConfigKey", key);
+ }
+ }
}
diff --git a/object_store/src/azure/builder.rs
b/object_store/src/azure/builder.rs
index c0c4e8983a..0208073e85 100644
--- a/object_store/src/azure/builder.rs
+++ b/object_store/src/azure/builder.rs
@@ -408,7 +408,7 @@ impl FromStr for AzureConfigKey {
"azure_disable_tagging" | "disable_tagging" =>
Ok(Self::DisableTagging),
// Backwards compatibility
"azure_allow_http" => Ok(Self::Client(ClientConfigKey::AllowHttp)),
- _ => match s.parse() {
+ _ => match s.strip_prefix("azure_").unwrap_or(s).parse() {
Ok(key) => Ok(Self::Client(key)),
Err(_) => Err(Error::UnknownConfigurationKey { key: s.into()
}.into()),
},
@@ -1103,4 +1103,17 @@ mod tests {
let pairs = split_sas(raw_sas).unwrap();
assert_eq!(expected, pairs);
}
+
+ #[test]
+ fn azure_test_client_opts() {
+ let key = "AZURE_PROXY_URL";
+ if let Ok(config_key) = key.to_ascii_lowercase().parse() {
+ assert_eq!(
+ AzureConfigKey::Client(ClientConfigKey::ProxyUrl),
+ config_key
+ );
+ } else {
+ panic!("{} not propagated as ClientConfigKey", key);
+ }
+ }
}
diff --git a/object_store/src/client/mod.rs b/object_store/src/client/mod.rs
index c45833b89d..9a3b705545 100644
--- a/object_store/src/client/mod.rs
+++ b/object_store/src/client/mod.rs
@@ -157,6 +157,8 @@ impl FromStr for ClientConfigKey {
"pool_idle_timeout" => Ok(Self::PoolIdleTimeout),
"pool_max_idle_per_host" => Ok(Self::PoolMaxIdlePerHost),
"proxy_url" => Ok(Self::ProxyUrl),
+ "proxy_ca_certificate" => Ok(Self::ProxyCaCertificate),
+ "proxy_excludes" => Ok(Self::ProxyExcludes),
"timeout" => Ok(Self::Timeout),
"user_agent" => Ok(Self::UserAgent),
_ => Err(super::Error::UnknownConfigurationKey {
diff --git a/object_store/src/gcp/builder.rs b/object_store/src/gcp/builder.rs
index 82dab14437..26cc8211d2 100644
--- a/object_store/src/gcp/builder.rs
+++ b/object_store/src/gcp/builder.rs
@@ -185,7 +185,7 @@ impl FromStr for GoogleConfigKey {
"google_service_account_key" | "service_account_key" =>
Ok(Self::ServiceAccountKey),
"google_bucket" | "google_bucket_name" | "bucket" | "bucket_name"
=> Ok(Self::Bucket),
"google_application_credentials" =>
Ok(Self::ApplicationCredentials),
- _ => match s.parse() {
+ _ => match s.strip_prefix("google_").unwrap_or(s).parse() {
Ok(key) => Ok(Self::Client(key)),
Err(_) => Err(Error::UnknownConfigurationKey { key: s.into()
}.into()),
},
@@ -671,4 +671,17 @@ mod tests {
google_bucket_name
);
}
+
+ #[test]
+ fn gcp_test_client_opts() {
+ let key = "GOOGLE_PROXY_URL";
+ if let Ok(config_key) = key.to_ascii_lowercase().parse() {
+ assert_eq!(
+ GoogleConfigKey::Client(ClientConfigKey::ProxyUrl),
+ config_key
+ );
+ } else {
+ panic!("{} not propagated as ClientConfigKey", key);
+ }
+ }
}
