This is an automated email from the ASF dual-hosted git repository. ajay pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new 95c94dc HADOOP-16350. Ability to tell HDFS client not to request KMS Information from NameNode. Ccontributed by Greg Senia, Ajay Kumar. 95c94dc is described below commit 95c94dcca71a41e56a4c2989cf2aefdaf9923e13 Author: Ajay Kumar <a...@apache.org> AuthorDate: Mon Jun 24 11:38:43 2019 -0700 HADOOP-16350. Ability to tell HDFS client not to request KMS Information from NameNode. Ccontributed by Greg Senia, Ajay Kumar. --- .../org/apache/hadoop/fs/CommonConfigurationKeys.java | 13 +++++++++++++ .../hadoop-common/src/main/resources/core-default.xml | 14 ++++++++++++++ .../main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java | 18 ++++++++++++++---- .../org/apache/hadoop/hdfs/TestEncryptionZones.java | 11 +++++++++++ 4 files changed, 52 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index 2e6b132..958113c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -403,4 +403,17 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { public static final Class<? extends DomainNameResolver> HADOOP_DOMAINNAME_RESOLVER_IMPL_DEFAULT = DNSDomainNameResolver.class; + /* + * Ignore KMS default URI returned from NameNode. + * When set to true, kms uri is searched in the following order: + * 1. If there is a mapping in Credential's secrets map for namenode uri. + * 2. Fallback to local conf. + * If client choose to ignore KMS uri provided by NameNode then client + * should set KMS URI using 'hadoop.security.key.provider.path' to access + * the right KMS for encrypted files. + * */ + public static final String DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI = + "dfs.client.ignore.namenode.default.kms.uri"; + public static final boolean + DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI_DEFAULT = false; } diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 4e22e0a..5ae60d7 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -3480,4 +3480,18 @@ with the input domain name of the services by querying the underlying DNS. </description> </property> + + <property> + <name>dfs.client.ignore.namenode.default.kms.uri</name> + <value>false</value> + <description> + Ignore KMS default URI returned from NameNode. + When set to true, kms uri is searched in the following order: + 1. If there is a mapping in Credential's secrets map for namenode uri. + 2. Fallback to local conf. (i.e hadoop.security.key.provider.path) + If client choose to ignore KMS uri provided by NameNode then client + should set KMS URI using 'hadoop.security.key.provider.path' to access + the right KMS for encrypted files. + </description> + </property> </configuration> diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java index 30e8aa7..d35b23f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsKMSUtil.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs; +import static org.apache.hadoop.fs.CommonConfigurationKeys.DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI; +import static org.apache.hadoop.fs.CommonConfigurationKeys.DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX; import java.io.IOException; @@ -141,11 +143,19 @@ public final class HdfsKMSUtil { URI.create(DFSUtilClient.bytes2String(keyProviderUriBytes)); } if (keyProviderUri == null) { - // NN is old and doesn't report provider, so use conf. - if (keyProviderUriStr == null) { + // Check if NN provided uri is not null and ignore property is false. + if (keyProviderUriStr != null && !conf.getBoolean( + DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI, + DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI_DEFAULT)) { + if (!keyProviderUriStr.isEmpty()) { + keyProviderUri = URI.create(keyProviderUriStr); + } + } + // Fallback to configuration. + if (keyProviderUri == null) { + // Either NN is old and doesn't report provider or ignore NN KMS + // provider property is set to true, so use conf. keyProviderUri = KMSUtil.getKeyProviderUri(conf, keyProviderUriKeyName); - } else if (!keyProviderUriStr.isEmpty()) { - keyProviderUri = URI.create(keyProviderUriStr); } if (keyProviderUri != null) { credentials.addSecretKey( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index d401380..78fcc15 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -111,6 +111,7 @@ import org.junit.Test; import org.junit.rules.Timeout; import org.mockito.Mockito; +import static org.apache.hadoop.fs.CommonConfigurationKeys.DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI; import static org.junit.Assert.assertNotNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; @@ -1950,6 +1951,16 @@ public class TestEncryptionZones { Assert.assertEquals("Key Provider for client and namenode are different", namenodeKeyProviderUri, cluster.getFileSystem().getClient() .getKeyProviderUri()); + + // Ignore the key provider from NN. + clusterConf.setBoolean( + DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI, true); + Assert.assertEquals("Expecting Key Provider for client config", + "dummy://foo:bar@test_provider1", cluster.getFileSystem().getClient() + .getKeyProviderUri().toString()); + Assert.assertNotEquals("Key Provider for client and namenode is different", + namenodeKeyProviderUri, cluster.getFileSystem().getClient() + .getKeyProviderUri().toString()); } /** --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org