Repository: hadoop Updated Branches: refs/heads/HADOOP-11890 009e67021 -> 9fe251d15
MAPREDUCE-6519 Avoid unsafe split and append on fields that might be IPv6 literals Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9fe251d1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9fe251d1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9fe251d1 Branch: refs/heads/HADOOP-11890 Commit: 9fe251d154e49b1bf27464465d0523c2880afe6c Parents: 009e670 Author: Elliott Clark <ecl...@apache.org> Authored: Wed Nov 4 13:01:27 2015 -0800 Committer: Elliott Clark <ecl...@apache.org> Committed: Wed Nov 4 13:01:27 2015 -0800 ---------------------------------------------------------------------- .../apache/hadoop/mapred/FileInputFormat.java | 23 ++++++++++---------- .../apache/hadoop/mapreduce/util/HostUtil.java | 6 ++--- .../mapreduce/v2/hs/HistoryClientService.java | 11 ++++++---- .../hadoop/ipc/TestMRCJCSocketFactory.java | 8 ++++--- .../apache/hadoop/mapred/ReliabilityTest.java | 13 ++--------- .../hadoop/mapred/TestClientRedirect.java | 8 ++++--- .../org/apache/hadoop/mapred/UtilsForTests.java | 16 -------------- .../mapreduce/MiniHadoopClusterManager.java | 6 +++-- 8 files changed, 37 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe251d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java index 2c58ebe..65dbdd2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java @@ -44,6 +44,7 @@ import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.mapreduce.security.TokenCache; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.NodeBase; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StopWatch; @@ -685,19 +686,19 @@ public abstract class FileInputFormat<K, V> implements InputFormat<K, V> { private String[] identifyHosts(int replicationFactor, Map<Node,NodeInfo> racksMap) { - + String [] retVal = new String[replicationFactor]; - - List <NodeInfo> rackList = new LinkedList<NodeInfo>(); + + List <NodeInfo> rackList = new LinkedList<NodeInfo>(); rackList.addAll(racksMap.values()); - + // Sort the racks based on their contribution to this split sortInDescendingOrder(rackList); - + boolean done = false; int index = 0; - + // Get the host list for all our aggregated items, sort // them and return the top entries for (NodeInfo ni: rackList) { @@ -706,27 +707,27 @@ public abstract class FileInputFormat<K, V> implements InputFormat<K, V> { List<NodeInfo>hostList = new LinkedList<NodeInfo>(); hostList.addAll(hostSet); - + // Sort the hosts in this rack based on their contribution sortInDescendingOrder(hostList); for (NodeInfo host: hostList) { // Strip out the port number from the host name - retVal[index++] = host.node.getName().split(":")[0]; + retVal[index++] = NetUtils.getHostFromHostPort(host.node.getName()); if (index == replicationFactor) { done = true; break; } } - + if (done == true) { break; } } return retVal; } - - private String[] fakeRacks(BlockLocation[] blkLocations, int index) + + private String[] fakeRacks(BlockLocation[] blkLocations, int index) throws IOException { String[] allHosts = blkLocations[index].getHosts(); String[] allTopos = new String[allHosts.length]; http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe251d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java index ad279ee..1ba4387 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java @@ -20,6 +20,7 @@ package org.apache.hadoop.mapreduce.util; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.net.NetUtils; @Private @Unstable @@ -56,10 +57,7 @@ public class HostUtil { public static String convertTrackerNameToHostName(String trackerName) { // Ugly! // Convert the trackerName to its host name - int indexOfColon = trackerName.indexOf(":"); - String trackerHostName = (indexOfColon == -1) ? - trackerName : - trackerName.substring(0, indexOfColon); + String trackerHostName = NetUtils.getHostFromHostPort(trackerName); return trackerHostName.substring("tracker_".length()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe251d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java index 3751ad9..1d5ed5f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java @@ -90,6 +90,7 @@ import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; import com.google.common.annotations.VisibleForTesting; +import com.google.common.net.HostAndPort; /** * This module is responsible for talking to the @@ -161,10 +162,12 @@ public class HistoryClientService extends AbstractService { .withHttpSpnegoPrincipalKey( JHAdminConfig.MR_WEBAPP_SPNEGO_USER_NAME_KEY) .at(NetUtils.getHostPortString(bindAddress)).start(webApp); - - String connectHost = MRWebAppUtil.getJHSWebappURLWithoutScheme(conf).split(":")[0]; - MRWebAppUtil.setJHSWebappURLWithoutScheme(conf, - connectHost + ":" + webApp.getListenerAddress().getPort()); + + String connectHost = MRWebAppUtil.getJHSWebappURLWithoutScheme(conf); + + MRWebAppUtil.setJHSWebappURLWithoutScheme(conf, HostAndPort.fromParts( + HostAndPort.fromString(connectHost).getHostText(), + webApp.getListenerAddress().getPort()).toString()); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe251d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestMRCJCSocketFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestMRCJCSocketFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestMRCJCSocketFactory.java index 0274298..dc0b09a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestMRCJCSocketFactory.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestMRCJCSocketFactory.java @@ -36,6 +36,8 @@ import org.apache.hadoop.net.StandardSocketFactory; import org.junit.Assert; import org.junit.Test; +import com.google.common.net.HostAndPort; + /** * This class checks that RPCs can use specialized socket factories. */ @@ -88,9 +90,9 @@ public class TestMRCJCSocketFactory { "org.apache.hadoop.ipc.DummySocketFactory"); jconf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); String rmAddress = jconf.get("yarn.resourcemanager.address"); - String[] split = rmAddress.split(":"); - jconf.set("yarn.resourcemanager.address", split[0] + ':' - + (Integer.parseInt(split[1]) + 10)); + HostAndPort hp = HostAndPort.fromString(rmAddress); + jconf.set("yarn.resourcemanager.address", hp.getHostText() + ':' + + (hp.getPort() + 10)); client = new JobClient(jconf); JobStatus[] jobs = client.jobsToComplete(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe251d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java index e6e12eb..1d1ca2a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java @@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.util.HostUtil; import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.Shell; @@ -340,7 +341,7 @@ public class ReliabilityTest extends Configured implements Tool { LOG.info(new Date() + " Stopping a few trackers"); for (String tracker : trackerNamesList) { - String host = convertTrackerNameToHostName(tracker); + String host = HostUtil.convertTrackerNameToHostName(tracker); LOG.info(new Date() + " Marking tracker on host: " + host); fos.write((host + "\n").getBytes()); if (count++ >= trackerNamesList.size()/2) { @@ -378,16 +379,6 @@ public class ReliabilityTest extends Configured implements Tool { LOG.info(output); } } - - private String convertTrackerNameToHostName(String trackerName) { - // Convert the trackerName to it's host name - int indexOfColon = trackerName.indexOf(":"); - String trackerHostName = (indexOfColon == -1) ? - trackerName : - trackerName.substring(0, indexOfColon); - return trackerHostName.substring("tracker_".length()); - } - } private class KillTaskThread extends Thread { http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe251d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java index 8ab3304..51b030c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java @@ -133,6 +133,8 @@ import org.apache.hadoop.yarn.ipc.YarnRPC; import org.junit.Assert; import org.junit.Test; +import com.google.common.net.HostAndPort; + public class TestClientRedirect { static { @@ -306,9 +308,9 @@ public class TestClientRedirect { application.setYarnApplicationState(YarnApplicationState.FINISHED); application.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED); } - String[] split = AMHOSTADDRESS.split(":"); - application.setHost(split[0]); - application.setRpcPort(Integer.parseInt(split[1])); + HostAndPort hp = HostAndPort.fromString(AMHOSTADDRESS); + application.setHost(hp.getHostText()); + application.setRpcPort(hp.getPort()); application.setUser("TestClientRedirect-user"); application.setName("N/A"); application.setQueue("N/A"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe251d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java index 972391c..3e7a596 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java @@ -797,20 +797,4 @@ public class UtilsForTests { file.close(); return file; } - - /** - * This formats the long tasktracker name to just the FQDN - * @param taskTrackerLong String The long format of the tasktracker string - * @return String The FQDN of the tasktracker - * @throws Exception - */ - public static String getFQDNofTT (String taskTrackerLong) throws Exception { - //Getting the exact FQDN of the tasktracker from the tasktracker string. - String[] firstSplit = taskTrackerLong.split("_"); - String tmpOutput = firstSplit[1]; - String[] secondSplit = tmpOutput.split(":"); - String tmpTaskTracker = secondSplit[0]; - return tmpTaskTracker; - } - } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe251d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java index 2e8ba5e..cae01e7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java @@ -47,6 +47,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.mortbay.util.ajax.JSON; +import com.google.common.net.HostAndPort; + /** * This class drives the creation of a mini-cluster on the local machine. By * default, a MiniDFSCluster and MiniMRCluster are spawned on the first @@ -188,8 +190,8 @@ public class MiniHadoopClusterManager { map.put("namenode_port", dfs.getNameNodePort()); } if (mr != null) { - map.put("resourcemanager_port", mr.getConfig().get( - YarnConfiguration.RM_ADDRESS).split(":")[1]); + map.put("resourcemanager_port", HostAndPort.fromString( + mr.getConfig().get(YarnConfiguration.RM_ADDRESS)).getPort()); } FileWriter fw = new FileWriter(new File(writeDetails)); fw.write(new JSON().toJSON(map));
