HDFS-9364. Unnecessary DNS resolution attempts when creating NameNodeProxies. Contributed by Xiao Chen.
Change-Id: I9e42f724f27924cf73891425a832de37ce014a1e Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/73b94d78 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/73b94d78 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/73b94d78 Branch: refs/heads/yarn-2877 Commit: 73b94d789969354bb9a6872d99976763ca8470d7 Parents: 0eb9c60 Author: Zhe Zhang <z...@apache.org> Authored: Tue Nov 10 09:55:29 2015 -0800 Committer: Zhe Zhang <z...@apache.org> Committed: Tue Nov 10 09:55:29 2015 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 22 +++++++++++- .../hadoop/hdfs/NameNodeProxiesClient.java | 2 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/TestDFSClientFailover.java | 35 ++++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/73b94d78/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java index c14d5d3..62c5d81 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java @@ -615,7 +615,7 @@ public class DFSUtilClient { public static InetSocketAddress getNNAddress(Configuration conf) { URI filesystemURI = FileSystem.getDefaultUri(conf); - return getNNAddress(filesystemURI); + return getNNAddressCheckLogical(conf, filesystemURI); } /** @@ -638,6 +638,26 @@ public class DFSUtilClient { return getNNAddress(authority); } + /** + * Get the NN address from the URI. If the uri is logical, default address is + * returned. Otherwise return the DNS-resolved address of the URI. + * + * @param conf configuration + * @param filesystemURI URI of the file system + * @return address of file system + */ + public static InetSocketAddress getNNAddressCheckLogical(Configuration conf, + URI filesystemURI) { + InetSocketAddress retAddr; + if (HAUtilClient.isLogicalUri(conf, filesystemURI)) { + retAddr = InetSocketAddress.createUnresolved(filesystemURI.getAuthority(), + HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); + } else { + retAddr = getNNAddress(filesystemURI); + } + return retAddr; + } + public static URI getNNUri(InetSocketAddress namenode) { int port = namenode.getPort(); String portString = http://git-wip-us.apache.org/repos/asf/hadoop/blob/73b94d78/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java index 39b188f..5ca7030 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java @@ -320,7 +320,7 @@ public class NameNodeProxiesClient { DFSUtilClient.getNNAddress(nameNodeUri)); } return new ProxyAndInfo<>(proxy, dtService, - DFSUtilClient.getNNAddress(nameNodeUri)); + DFSUtilClient.getNNAddressCheckLogical(conf, nameNodeUri)); } public static ClientProtocol createNonHAProxyWithClientProtocol( http://git-wip-us.apache.org/repos/asf/hadoop/blob/73b94d78/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8625ecf..336db03 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2285,6 +2285,9 @@ Release 2.8.0 - UNRELEASED HDFS-9401. Fix findbugs warnings in BlockRecoveryWorker. (Brahma Reddy Battula via waltersu4549) + HDFS-9364. Unnecessary DNS resolution attempts when creating NameNodeProxies. + (Xiao Chen via zhz) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/73b94d78/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java index b098711..60ffe30 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.hdfs.server.namenode.ha.IPFailoverProxyProvider; +import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.retry.FailoverProxyProvider; import org.apache.hadoop.net.ConnectTimeoutException; @@ -291,6 +292,40 @@ public class TestDFSClientFailover { Mockito.verify(spyNS, Mockito.never()).lookupAllHostAddr(Mockito.eq(logicalHost)); } + /** + * Test that creating proxy doesn't ever try to DNS-resolve the logical URI. + * Regression test for HDFS-9364. + */ + @Test(timeout=60000) + public void testCreateProxyDoesntDnsResolveLogicalURI() throws IOException { + final NameService spyNS = spyOnNameService(); + final Configuration conf = new HdfsConfiguration(); + final String service = "nameservice1"; + final String namenode = "namenode113"; + conf.set(DFSConfigKeys.DFS_NAMESERVICES, service); + conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "hdfs://" + service); + conf.set( + HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX + "." + service, + "org.apache.hadoop.hdfs.server.namenode.ha." + + "ConfiguredFailoverProxyProvider"); + conf.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + service, + namenode); + conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + service + "." + + namenode, "localhost:8020"); + + // call createProxy implicitly and explicitly + Path p = new Path("/"); + p.getFileSystem(conf); + NameNodeProxiesClient.createProxyWithClientProtocol(conf, + FileSystem.getDefaultUri(conf), null); + NameNodeProxies.createProxy(conf, FileSystem.getDefaultUri(conf), + NamenodeProtocol.class, null); + + // Ensure that the logical hostname was never resolved. + Mockito.verify(spyNS, Mockito.never()).lookupAllHostAddr( + Mockito.eq(service)); + } + /** Dummy implementation of plain FailoverProxyProvider */ public static class DummyLegacyFailoverProxyProvider<T> implements FailoverProxyProvider<T> {
