[ https://issues.apache.org/jira/browse/HADOOP-13972?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16362312#comment-16362312 ]
ASF GitHub Bot commented on HADOOP-13972: ----------------------------------------- Github user steveloughran commented on a diff in the pull request: https://github.com/apache/hadoop/pull/339#discussion_r167862774 --- Diff: hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java --- @@ -985,4 +994,65 @@ public void setUserGroupRepresentationAsUPN(boolean enableUPN) { oidOrUpn = enableUPN ? UserGroupRepresentation.UPN : UserGroupRepresentation.OID; } + + /** + * Gets ADL account name from ADL FQDN + * @param accountFQDN ADL account fqdn + * @return ADL account name + */ + public static String getAccountNameFromFQDN(String accountFQDN) { + return accountFQDN.contains(".") + ? accountFQDN.substring(0, accountFQDN.indexOf(".")) + : accountFQDN; + } + + /** + * Propagates account-specific settings into generic ADL configuration keys. + * This is done by propagating the values of the form + * {@code fs.adl.account.${account_name}.key} to + * {@code fs.adl.key}, for all values of "key" + * + * The source of the updated property is set to the key name of the account + * property, to aid in diagnostics of where things came from. + * + * Returns a new configuration. Why the clone? + * You can use the same conf for different filesystems, and the original + * values are not updated. + * + * + * @param source Source Configuration object + * @param accountName account name. Must not be empty + * @return a (potentially) patched clone of the original + * @throws IOException + */ + public static Configuration propagateAccountOptions( + Configuration source, String accountName) throws IOException { + if (StringUtils.isEmpty(accountName)) { + throw new IOException("Account name cannot be empty"); + } + + final String accountPrefix = AZURE_AD_ACCOUNT_PREFIX + accountName +'.'; + LOG.debug("Propagating entries under {}", accountPrefix); + final Configuration dest = new Configuration(source); + for (Map.Entry<String, String> entry : source) { + final String key = entry.getKey(); + // get the (unexpanded) value. + final String value = entry.getValue(); + if (!key.startsWith(accountPrefix) || accountPrefix.equals(key)) { + continue; + } + // there's a account prefix, so strip it + final String stripped = key.substring(accountPrefix.length()); + + // propagate the value, building a new origin field. + // to track overwrites, the generic key is overwritten even if + // already matches the new one. + String origin = "[" + StringUtils.join( + source.getPropertySources(key), ", ") +"]"; + final String generic = AZURE_AD_PREFIX + stripped; + LOG.debug("Updating {} from {}", generic, origin); + dest.set(generic, value, key + " via " + origin); --- End diff -- nice! > ADLS to support per-store configuration > --------------------------------------- > > Key: HADOOP-13972 > URL: https://issues.apache.org/jira/browse/HADOOP-13972 > Project: Hadoop Common > Issue Type: Improvement > Components: fs/adl > Affects Versions: 3.0.0-alpha2 > Reporter: John Zhuge > Priority: Major > > Useful when distcp needs to access 2 Data Lake stores with different SPIs. > Of course, a workaround is to grant the same SPI access permission to both > stores, but sometimes it might not be feasible. > One idea is to embed the store name in the configuration property names, > e.g., {{dfs.adls.oauth2.<store>.client.id}}. Per-store keys will be consulted > first, then fall back to the global keys. -- This message was sent by Atlassian JIRA (v7.6.3#76005) --------------------------------------------------------------------- To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org