Repository: hadoop Updated Branches: refs/heads/HADOOP-11890 fa64e68fd -> 009e67021
YARN-4283 Avoid unsafe split and append on fields that might be IPv6 literals Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/009e6702 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/009e6702 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/009e6702 Branch: refs/heads/HADOOP-11890 Commit: 009e67021f5b4c20fc70463159a3a3fdce8c7c49 Parents: fa64e68 Author: Elliott Clark <ecl...@apache.org> Authored: Tue Oct 27 18:41:02 2015 -0700 Committer: Elliott Clark <ecl...@apache.org> Committed: Tue Oct 27 18:41:02 2015 -0700 ---------------------------------------------------------------------- .../apache/hadoop/yarn/util/ConverterUtils.java | 20 ++--- .../hadoop/yarn/webapp/util/WebAppUtils.java | 20 ++--- .../hadoop/yarn/conf/TestYarnConfiguration.java | 82 ++++++++++++-------- .../hadoop/yarn/util/TestConverterUtils.java | 14 +++- .../apache/hadoop/yarn/lib/TestZKClient.java | 17 ++-- .../containermanager/ContainerManagerImpl.java | 3 +- .../server/resourcemanager/ResourceManager.java | 6 +- .../yarn/server/resourcemanager/MockNM.java | 7 +- .../yarn/server/webproxy/WebAppProxy.java | 7 +- .../webproxy/amfilter/AmFilterInitializer.java | 6 +- .../server/webproxy/TestWebAppProxyServlet.java | 8 +- 11 files changed, 116 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/009e6702/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java index e9674cf..34cf4d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import com.google.common.net.HostAndPort; /** * This class contains a set of utilities which help converting data structures @@ -152,26 +153,27 @@ public class ConverterUtils { public static String toString(ContainerId cId) { return cId == null ? null : cId.toString(); } - + public static NodeId toNodeIdWithDefaultPort(String nodeIdStr) { - if (nodeIdStr.indexOf(":") < 0) { - return toNodeId(nodeIdStr + ":0"); - } - return toNodeId(nodeIdStr); + HostAndPort hp = HostAndPort.fromString(nodeIdStr); + hp = hp.withDefaultPort(0); + return toNodeId(hp.toString()); } public static NodeId toNodeId(String nodeIdStr) { - String[] parts = nodeIdStr.split(":"); - if (parts.length != 2) { + HostAndPort hp = HostAndPort.fromString(nodeIdStr); + if (!hp.hasPort()) { throw new IllegalArgumentException("Invalid NodeId [" + nodeIdStr + "]. Expected host:port"); } try { + String hostPortStr = hp.toString(); + String host = hostPortStr.substring(0, hostPortStr.lastIndexOf(":")); NodeId nodeId = - NodeId.newInstance(parts[0].trim(), Integer.parseInt(parts[1])); + NodeId.newInstance(host, hp.getPort()); return nodeId; } catch (NumberFormatException e) { - throw new IllegalArgumentException("Invalid port: " + parts[1], e); + throw new IllegalArgumentException("Invalid port: " + hp.getPort(), e); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/009e6702/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index 459c110..1b05f4e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -37,6 +37,8 @@ import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.util.RMHAUtils; +import com.google.common.net.HostAndPort; + @Private @Evolving public class WebAppUtils { @@ -51,15 +53,13 @@ public class WebAppUtils { public static void setRMWebAppPort(Configuration conf, int port) { String hostname = getRMWebAppURLWithoutScheme(conf); - hostname = - (hostname.contains(":")) ? hostname.substring(0, hostname.indexOf(":")) - : hostname; - setRMWebAppHostnameAndPort(conf, hostname, port); + HostAndPort hp = HostAndPort.fromString(hostname); + setRMWebAppHostnameAndPort(conf, hp.getHostText(), port); } public static void setRMWebAppHostnameAndPort(Configuration conf, String hostname, int port) { - String resolvedAddress = hostname + ":" + port; + String resolvedAddress = HostAndPort.fromParts(hostname, port).toString(); if (YarnConfiguration.useHttps(conf)) { conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, resolvedAddress); } else { @@ -69,12 +69,11 @@ public class WebAppUtils { public static void setNMWebAppHostNameAndPort(Configuration conf, String hostName, int port) { + String hostPortString = HostAndPort.fromParts(hostName, port).toString(); if (YarnConfiguration.useHttps(conf)) { - conf.set(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS, - hostName + ":" + port); + conf.set(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS, hostPortString); } else { - conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS, - hostName + ":" + port); + conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS, hostPortString); } } @@ -241,7 +240,8 @@ public class WebAppUtils { String host = conf.getTrimmed(hostProperty); if (host != null && !host.isEmpty()) { if (webAppURLWithoutScheme.contains(":")) { - webAppURLWithoutScheme = host + ":" + webAppURLWithoutScheme.split(":")[1]; + String[] splits = webAppURLWithoutScheme.split(":"); + webAppURLWithoutScheme = host + ":" + splits[splits.length - 1]; } else { throw new YarnRuntimeException("webAppURLWithoutScheme must include port specification but doesn't: " + http://git-wip-us.apache.org/repos/asf/hadoop/blob/009e6702/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java index 1f10810..598d44f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.conf; +import com.google.common.net.HostAndPort; +import org.apache.hadoop.net.NetUtils; import org.junit.Assert; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -53,7 +55,7 @@ public class TestYarnConfiguration { conf.set(YarnConfiguration.RM_ADDRESS, "rmtesting:9999"); String rmWebUrl = WebAppUtils.getRMWebAppURLWithScheme(conf); String[] parts = rmWebUrl.split(":"); - Assert.assertEquals("RM Web URL Port is incrrect", 24543, + Assert.assertEquals("RM Web URL Port is incorrect", 24543, Integer.valueOf(parts[parts.length - 1]).intValue()); Assert.assertNotSame( "RM Web Url not resolved correctly. Should not be rmtesting", @@ -92,9 +94,9 @@ public class TestYarnConfiguration { YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); assertEquals( - new InetSocketAddress( - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0], - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), + new InetSocketAddress( + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0], + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), resourceTrackerAddress); //with address @@ -105,9 +107,9 @@ public class TestYarnConfiguration { YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); assertEquals( - new InetSocketAddress( - "10.0.0.1", - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), + new InetSocketAddress( + "10.0.0.1", + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), resourceTrackerAddress); //address and socket @@ -118,9 +120,23 @@ public class TestYarnConfiguration { YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); assertEquals( - new InetSocketAddress( - "10.0.0.2", - 5001), + new InetSocketAddress( + "10.0.0.2", + 5001), + resourceTrackerAddress); + + // IPv6 address and socket + conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, + "[2401:db00:20:a01e:face:0:5:0]:5001"); + resourceTrackerAddress = conf.getSocketAddr( + YarnConfiguration.RM_BIND_HOST, + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); + assertEquals( + new InetSocketAddress( + "2401:db00:20:a01e:face:0:5:0", + 5001), resourceTrackerAddress); //bind host only @@ -132,9 +148,9 @@ public class TestYarnConfiguration { YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); assertEquals( - new InetSocketAddress( - "10.0.0.3", - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), + new InetSocketAddress( + "10.0.0.3", + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), resourceTrackerAddress); //bind host and address no port @@ -146,9 +162,9 @@ public class TestYarnConfiguration { YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); assertEquals( - new InetSocketAddress( - "0.0.0.0", - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), + new InetSocketAddress( + "0.0.0.0", + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), resourceTrackerAddress); //bind host and address with port @@ -160,9 +176,9 @@ public class TestYarnConfiguration { YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); assertEquals( - new InetSocketAddress( - "0.0.0.0", - 5003), + new InetSocketAddress( + "0.0.0.0", + 5003), resourceTrackerAddress); } @@ -176,9 +192,8 @@ public class TestYarnConfiguration { //no override, old behavior. Won't work on a host named "yo.yo.yo" conf = new YarnConfiguration(); conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo"); - serverAddress = new InetSocketAddress( - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0], - Integer.valueOf(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1])); + serverAddress = newInetSocketAddressFromHostPort( + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS); resourceTrackerConnectAddress = conf.updateConnectAddr( YarnConfiguration.RM_BIND_HOST, @@ -186,15 +201,14 @@ public class TestYarnConfiguration { YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, serverAddress); - assertFalse(resourceTrackerConnectAddress.toString().startsWith("yo.yo.yo")); + assertFalse(NetUtils.getSocketAddressString(resourceTrackerConnectAddress) + .startsWith("yo.yo.yo")); //cause override with address conf = new YarnConfiguration(); conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo"); conf.set(YarnConfiguration.RM_BIND_HOST, "0.0.0.0"); - serverAddress = new InetSocketAddress( - YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0], - Integer.valueOf(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1])); + serverAddress = newInetSocketAddressFromHostPort(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS); resourceTrackerConnectAddress = conf.updateConnectAddr( YarnConfiguration.RM_BIND_HOST, @@ -202,7 +216,8 @@ public class TestYarnConfiguration { YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, serverAddress); - assertTrue(resourceTrackerConnectAddress.toString().startsWith("yo.yo.yo")); + assertTrue(NetUtils.getSocketAddressString(resourceTrackerConnectAddress) + .startsWith("yo.yo.yo")); //tests updateConnectAddr won't add suffix to NM service address configurations conf = new YarnConfiguration(); @@ -211,9 +226,7 @@ public class TestYarnConfiguration { conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); conf.set(YarnConfiguration.RM_HA_ID, "rm1"); - serverAddress = new InetSocketAddress( - YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS.split(":")[0], - Integer.valueOf(YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS.split(":")[1])); + serverAddress = newInetSocketAddressFromHostPort(YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS); InetSocketAddress localizerAddress = conf.updateConnectAddr( YarnConfiguration.NM_BIND_HOST, @@ -221,8 +234,15 @@ public class TestYarnConfiguration { YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS, serverAddress); - assertTrue(localizerAddress.toString().startsWith("yo.yo.yo")); + assertTrue(NetUtils.getSocketAddressString(localizerAddress) + .startsWith("yo.yo.yo")); assertNull(conf.get( HAUtil.addSuffix(YarnConfiguration.NM_LOCALIZER_ADDRESS, "rm1"))); } + + private InetSocketAddress newInetSocketAddressFromHostPort( + String hostPort) { + HostAndPort hp = HostAndPort.fromString(hostPort); + return new InetSocketAddress(hp.getHostText(), hp.getPort()); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/009e6702/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java index 3cec38b..863cafb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java @@ -30,7 +30,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.junit.Test; public class TestConverterUtils { - + @Test public void testConvertUrlWithNoPort() throws URISyntaxException { Path expectedPath = new Path("hdfs://foo.com"); @@ -90,14 +90,22 @@ public class TestConverterUtils { @Test public void testNodeIdWithDefaultPort() throws URISyntaxException { NodeId nid; - + nid = ConverterUtils.toNodeIdWithDefaultPort("node:10"); assertEquals(nid.getPort(), 10); assertEquals(nid.getHost(), "node"); - + nid = ConverterUtils.toNodeIdWithDefaultPort("node"); assertEquals(nid.getPort(), 0); assertEquals(nid.getHost(), "node"); + + nid = ConverterUtils.toNodeIdWithDefaultPort("[2401:db00:20:a01e:face:0:5:0]:10"); + assertEquals(nid.getPort(), 10); + assertEquals(nid.getHost(), "[2401:db00:20:a01e:face:0:5:0]"); + + nid = ConverterUtils.toNodeIdWithDefaultPort("[2401:db00:20:a01e:face:0:5:0]"); + assertEquals(nid.getPort(), 0); + assertEquals(nid.getHost(), "[2401:db00:20:a01e:face:0:5:0]"); } @Test(expected = IllegalArgumentException.class) http://git-wip-us.apache.org/repos/asf/hadoop/blob/009e6702/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/lib/TestZKClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/lib/TestZKClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/lib/TestZKClient.java index d228563..ce0c3bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/lib/TestZKClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/lib/TestZKClient.java @@ -26,6 +26,7 @@ import java.io.OutputStream; import java.net.InetSocketAddress; import java.net.Socket; +import com.google.common.net.HostAndPort; import org.junit.Assert; import org.apache.hadoop.yarn.lib.ZKClient; @@ -82,8 +83,9 @@ public class TestZKClient { long start = System.currentTimeMillis(); while (true) { try { - String host = hp.split(":")[0]; - int port = Integer.parseInt(hp.split(":")[1]); + HostAndPort hap = HostAndPort.fromString(hp); + String host = hap.getHostText(); + int port = hap.getPort(); send4LetterWord(host, port, "stat"); } catch (IOException e) { return true; @@ -106,8 +108,9 @@ public class TestZKClient { long start = System.currentTimeMillis(); while (true) { try { - String host = hp.split(":")[0]; - int port = Integer.parseInt(hp.split(":")[1]); + HostAndPort hap = HostAndPort.fromString(hp); + String host = hap.getHostText(); + int port = hap.getPort(); // if there are multiple hostports, just take the first one String result = send4LetterWord(host, port, "stat"); if (result.startsWith("Zookeeper version:")) { @@ -146,7 +149,8 @@ public class TestZKClient { } File dataDir = createTmpDir(BASETEST); zks = new ZooKeeperServer(dataDir, dataDir, 3000); - final int PORT = Integer.parseInt(hostPort.split(":")[1]); + HostAndPort hp = HostAndPort.fromString(hostPort); + final int PORT = hp.getPort(); if (factory == null) { factory = new NIOServerCnxnFactory(); factory.configure(new InetSocketAddress(PORT), maxCnxns); @@ -167,7 +171,8 @@ public class TestZKClient { zkDb.close(); } catch (IOException ie) { } - final int PORT = Integer.parseInt(hostPort.split(":")[1]); + HostAndPort hp = HostAndPort.fromString(hostPort); + final int PORT = hp.getPort(); Assert.assertTrue("waiting for server down", waitForServerDown("127.0.0.1:" + PORT, http://git-wip-us.apache.org/repos/asf/hadoop/blob/009e6702/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index f44de59..2e6f4e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -37,6 +37,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; +import com.google.common.net.HostAndPort; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -474,7 +475,7 @@ public class ContainerManagerImpl extends CompositeService implements //hostname found when querying for our hostname with the specified //address, combine the specified address with the actual port listened //on by the server - hostOverride = nmAddress.split(":")[0]; + hostOverride = HostAndPort.fromString(nmAddress).getHostText(); } // setup node ID http://git-wip-us.apache.org/repos/asf/hadoop/blob/009e6702/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 01a1c8f..f1de990 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager; import com.google.common.annotations.VisibleForTesting; +import com.google.common.net.HostAndPort; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -962,9 +963,8 @@ public class ResourceManager extends CompositeService implements Recoverable { builder.withServlet(ProxyUriUtils.PROXY_SERVLET_NAME, ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class); builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher); - String[] proxyParts = proxyHostAndPort.split(":"); - builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE, proxyParts[0]); - + builder.withAttribute(WebAppProxy.PROXY_HOST_ATTRIBUTE, + HostAndPort.fromString(proxyHostAndPort).getHostText()); } webApp = builder.start(new RMWebApp(this)); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/009e6702/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java index 4233cd4..41147f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import com.google.common.net.HostAndPort; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -75,8 +76,10 @@ public class MockNM { this.vCores = vcores; this.resourceTracker = resourceTracker; this.version = version; - String[] splits = nodeIdStr.split(":"); - nodeId = BuilderUtils.newNodeId(splits[0], Integer.parseInt(splits[1])); + HostAndPort hostAndPort = HostAndPort.fromString(nodeIdStr); + String hostPortStr = hostAndPort.toString(); + String host = hostPortStr.substring(0, hostPortStr.lastIndexOf(":")); + nodeId = BuilderUtils.newNodeId(host, hostAndPort.getPort()); } public NodeId getNodeId() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/009e6702/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java index 5935327..3b3f620 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.webproxy; import java.io.IOException; import java.net.URI; +import com.google.common.net.HostAndPort; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.security.authorize.AccessControlList; @@ -56,7 +57,8 @@ public class WebAppProxy extends AbstractService { @Override protected void serviceInit(Configuration conf) throws Exception { - String auth = conf.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION); + String auth = conf.get( + CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION); if (auth == null || "simple".equals(auth)) { isSecurityEnabled = false; } else if ("kerberos".equals(auth)) { @@ -67,8 +69,7 @@ public class WebAppProxy extends AbstractService { " of " + auth); } String proxy = WebAppUtils.getProxyHostAndPort(conf); - String[] proxyParts = proxy.split(":"); - proxyHost = proxyParts[0]; + proxyHost = HostAndPort.fromString(proxy).getHostText(); fetcher = new AppReportFetcher(conf); bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS); http://git-wip-us.apache.org/repos/asf/hadoop/blob/009e6702/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java index 6841616..3c97162 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java @@ -29,18 +29,20 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.annotations.VisibleForTesting; +import com.google.common.net.HostAndPort; public class AmFilterInitializer extends FilterInitializer { private static final String FILTER_NAME = "AM_PROXY_FILTER"; private static final String FILTER_CLASS = AmIpFilter.class.getCanonicalName(); - + @Override public void initFilter(FilterContainer container, Configuration conf) { Map<String, String> params = new HashMap<>(); List<String> proxies = WebAppUtils.getProxyHostsAndPortsForAmFilter(conf); StringBuilder sb = new StringBuilder(); for (String proxy : proxies) { - sb.append(proxy.split(":")[0]).append(AmIpFilter.PROXY_HOSTS_DELIMITER); + sb.append(HostAndPort.fromString(proxy).getHostText()) + .append(AmIpFilter.PROXY_HOSTS_DELIMITER); } sb.setLength(sb.length() - 1); params.put(AmIpFilter.PROXY_HOSTS, sb.toString()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/009e6702/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java index 68d1258..251c5d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java @@ -41,6 +41,7 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import com.google.common.net.HostAndPort; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.http.HttpServer2; @@ -442,11 +443,10 @@ public class TestWebAppProxyServlet { proxyServer.setAttribute(FETCHER_ATTRIBUTE, appReportFetcher ); proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, Boolean.TRUE); - + String proxy = WebAppUtils.getProxyHostAndPort(conf); - String[] proxyParts = proxy.split(":"); - String proxyHost = proxyParts[0]; - + String proxyHost = HostAndPort.fromString(proxy).getHostText(); + proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost); proxyServer.start(); LOG.info("Proxy server is started at port {}",
