sunchao commented on code in PR #6059:
URL: https://github.com/apache/datafusion-comet/pull/6059#discussion_r4178585504
##########
native/core/src/parquet/objectstore/azure.rs:
##########
@@ -189,38 +1007,126 @@ fn account_scoped_value(
base_key: &str,
account: Option<&str>,
) -> Option<String> {
- if let Some(acc) = account {
- for suffix in ENDPOINT_SUFFIXES {
- let scoped = format!("{base_key}.{acc}.{suffix}");
- if let Some(v) = configs.get(&scoped) {
- return Some(v.clone());
- }
- }
- let bare = format!("{base_key}.{acc}");
- if let Some(v) = configs.get(&bare) {
- return Some(v.clone());
- }
+ account_scoped_entry(configs, base_key, account).map(|(_, value)| value)
+}
+
+/// Like `account_scoped_value`, but also returns the configuration key that
matched.
+fn account_scoped_entry(
+ configs: &HashMap<String, String>,
+ base_key: &str,
+ account: Option<&str>,
+) -> Option<(String, String)> {
+ let mut candidates = account_candidates(base_key, account);
+ candidates.push(base_key.to_string());
+ first_entry(configs, candidates)
+}
+
+/// The account-scoped forms of `base_key` in lookup order:
+/// `<base>.<account>.<endpoint-suffix>`, then `<base>.<account>`. Empty
without an account.
+fn account_candidates(base_key: &str, account: Option<&str>) -> Vec<String> {
+ let Some(acc) = account else {
+ return Vec::new();
+ };
+ ENDPOINT_SUFFIXES
+ .iter()
+ .map(|suffix| format!("{base_key}.{acc}.{suffix}"))
+ .chain(std::iter::once(format!("{base_key}.{acc}")))
+ .collect()
+}
+
+/// Look up a token provider class the way Hadoop's
`AbfsConfiguration.getTokenProviderClass`
+/// does: the account-scoped key first, then the global key only when the
global auth type
+/// is also the account's (`global_auth_type_matches`).
+fn token_provider_entry(
+ configs: &HashMap<String, String>,
+ base_key: &str,
+ account: Option<&str>,
+) -> Option<(String, String)> {
+ if let Some(entry) = first_entry(configs, account_candidates(base_key,
account)) {
+ return Some(entry);
+ }
+ if !global_auth_type_matches(configs, account) {
+ return None;
}
- configs.get(base_key).cloned()
+ configs
+ .get(base_key)
+ .map(|value| (base_key.to_string(), value.clone()))
}
-/// Resolve the SAS token for `(container, account)`, accepting any of the
-/// `fs.azure.sas.<container>.<account>[.<endpoint-suffix>]` variants.
-fn sas_value(configs: &HashMap<String, String>, container: &str, account:
&str) -> Option<String> {
- for suffix in ENDPOINT_SUFFIXES {
- let key = format!("{HADOOP_SAS_PREFIX}{container}.{account}.{suffix}");
- if let Some(v) = configs.get(&key) {
- return Some(v.clone());
- }
+/// Whether the global `fs.azure.account.auth.type` equals the one the account
resolves to,
+/// as `getTokenProviderClass` compares them. An unset global value is
`SharedKey`, the
+/// default of `getAccountAgnosticEnum`; an account without its own value
resolves to the
+/// global one, so the two always match then.
+fn global_auth_type_matches(configs: &HashMap<String, String>, account:
Option<&str>) -> bool {
+ let global = configs
+ .get(HADOOP_AUTH_TYPE)
+ .map_or("SharedKey", |value| value.trim());
+ first_entry(configs, account_candidates(HADOOP_AUTH_TYPE, account))
+ .is_none_or(|(_, scoped)| scoped.trim().eq_ignore_ascii_case(global))
+}
+
+/// The first `(key, value)` of `candidates` that is present in `configs`.
+fn first_entry(
+ configs: &HashMap<String, String>,
+ candidates: Vec<String>,
+) -> Option<(String, String)> {
+ candidates
+ .into_iter()
+ .find_map(|key| configs.get(&key).map(|v| (key, v.clone())))
+}
+
+/// `sas_token`, but only while SAS is the selected mechanism.
+fn active_sas_token(
+ configs: &HashMap<String, String>,
+ account: Option<&str>,
+ container: Option<UrlContainer<'_>>,
+) -> Option<(String, String)> {
+ if !mechanism_is_read(configs, account, AuthMechanism::Sas) {
+ return None;
}
- let bare = format!("{HADOOP_SAS_PREFIX}{container}.{account}");
- configs.get(&bare).cloned()
+ sas_token(configs, account, container)
+}
+
+/// Resolve the SAS token and the key it came from. Hadoop's
+/// `AbfsConfiguration.getSASTokenProvider` reads `fs.azure.sas.fixed.token`
through
+/// `getPasswordString`, which in Hadoop 3.4.2 and later probes
+/// `fs.azure.sas.fixed.token.<container>.<host>` first, with the URL's host
and port as
+/// `containerConf` builds it, then the account-scoped and global keys, as
+/// `account_scoped_entry` probes them. The WASB driver's
+/// `fs.azure.sas.<container>.<account>[.<endpoint-suffix>]` variants come
last as a fallback.
+fn sas_token(
+ configs: &HashMap<String, String>,
+ account: Option<&str>,
+ container: Option<UrlContainer<'_>>,
+) -> Option<(String, String)> {
+ let container_fixed = || {
+ let UrlContainer { name, host } = container?;
+ let key = format!("{HADOOP_SAS_FIXED_TOKEN}.{name}.{host}");
+ configs.get(&key).map(|value| (key, value.clone()))
+ };
+ let container_scoped = || match (container, account) {
+ (Some(UrlContainer { name, .. }), Some(account)) => {
+ let mut candidates: Vec<String> = ENDPOINT_SUFFIXES
+ .iter()
+ .map(|suffix|
format!("{HADOOP_SAS_PREFIX}{name}.{account}.{suffix}"))
+ .collect();
+ candidates.push(format!("{HADOOP_SAS_PREFIX}{name}.{account}"));
+ first_entry(configs, candidates)
+ }
+ _ => None,
+ };
+ container_fixed()
Review Comment:
[P2] Respect the runtime Hadoop version when selecting container-scoped SAS
tokens. This unconditional lookup applies Hadoop 3.4.2 semantics to Hadoop
3.4.1, which Spark 4.0.4 uses. With `auth.type=SAS`, a valid global fixed token
and an unused blank
`fs.azure.sas.fixed.token.data.myacct.dfs.core.windows.net`, Hadoop ignores the
container key and uses the global token. This head instead rejects the scan as
a blank credential. A nonblank container value similarly replaces Hadoop's
selected token. The base works when the matching legacy WASB token is also
configured. Could token selection and validation follow the runtime Hadoop
behavior, either through JVM resolution or a capability guard? Documenting the
mismatch does not preserve previously working scans.
Evidence: Executed exact base/head `create_store` implementations with
`object_store 0.13.2` and no Azure environment settings. URL:
`abfss://[email protected]/path/file.parquet`. Set
`fs.azure.account.auth.type=SAS`, both `fs.azure.sas.fixed.token` and
`fs.azure.sas.data.myacct.dfs.core.windows.net` to
`sv=2020-08-04&sig=synthetic-global`, and
`fs.azure.sas.fixed.token.data.myacct.dfs.core.windows.net` to two spaces. Base
returns Ok. Head returns `Hadoop configuration for account myacct:
`fs.azure.sas.fixed.token.data.myacct.dfs.core.windows.net` is blank`. Removing
only the container-scoped fixed key makes both succeed. With a nonblank
synthetic-container token, static credential inspection confirms base selects
synthetic-global and head selects synthetic-container. Upstream Hadoop 3.4.1
`AbfsConfiguration.getPasswordString` lines 506–514 probes only account/global
keys, and `getSASTokenProvider` line 1069 uses that resolution. No network
authentication was atte
mpted.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
