This is an automated email from the ASF dual-hosted git repository.
brahma pushed a commit to branch HADOOP-17800
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/HADOOP-17800 by this push:
new 2c9b22f MAPREDUCE-6519. Avoid unsafe split and append on fields that
might be IPv6 literals. Contributed by Nemanja Matkovic And Hemanth Boyina.
2c9b22f is described below
commit 2c9b22f15cdb8b9498a82846d3b4a48d93ef7d17
Author: Brahma Reddy Battula <bra...@apache.org>
AuthorDate: Sat Jul 31 22:09:26 2021 +0530
MAPREDUCE-6519. Avoid unsafe split and append on fields that might be IPv6
literals. Contributed by Nemanja Matkovic And Hemanth Boyina.
---
.../org/apache/hadoop/mapred/FileInputFormat.java | 21 +++++++++++----------
.../org/apache/hadoop/mapreduce/util/HostUtil.java | 6 ++----
.../mapreduce/v2/hs/HistoryClientService.java | 11 +++++++----
.../apache/hadoop/ipc/TestMRCJCSocketFactory.java | 8 +++++---
.../org/apache/hadoop/mapred/ReliabilityTest.java | 12 ++----------
.../apache/hadoop/mapred/TestClientRedirect.java | 8 +++++---
.../org/apache/hadoop/mapred/UtilsForTests.java | 16 ----------------
.../hadoop/mapreduce/MiniHadoopClusterManager.java | 5 +++--
8 files changed, 35 insertions(+), 52 deletions(-)
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
index 91151f0..d18a722 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StopWatch;
@@ -712,19 +713,19 @@ public abstract class FileInputFormat<K, V> implements
InputFormat<K, V> {
private String[] identifyHosts(int replicationFactor,
Map<Node,NodeInfo> racksMap) {
-
+
String [] retVal = new String[replicationFactor];
-
- List <NodeInfo> rackList = new LinkedList<NodeInfo>();
+
+ List <NodeInfo> rackList = new LinkedList<NodeInfo>();
rackList.addAll(racksMap.values());
-
+
// Sort the racks based on their contribution to this split
sortInDescendingOrder(rackList);
boolean done = false;
int index = 0;
-
+
// Get the host list for all our aggregated items, sort
// them and return the top entries
for (NodeInfo ni: rackList) {
@@ -733,27 +734,27 @@ public abstract class FileInputFormat<K, V> implements
InputFormat<K, V> {
List<NodeInfo>hostList = new LinkedList<NodeInfo>();
hostList.addAll(hostSet);
-
+
// Sort the hosts in this rack based on their contribution
sortInDescendingOrder(hostList);
for (NodeInfo host: hostList) {
// Strip out the port number from the host name
- retVal[index++] = host.node.getName().split(":")[0];
+ retVal[index++] = NetUtils.getHostFromHostPort(host.node.getName());
if (index == replicationFactor) {
done = true;
break;
}
}
-
+
if (done == true) {
break;
}
}
return retVal;
}
-
- private String[] fakeRacks(BlockLocation[] blkLocations, int index)
+
+ private String[] fakeRacks(BlockLocation[] blkLocations, int index)
throws IOException {
String[] allHosts = blkLocations[index].getHosts();
String[] allTopos = new String[allHosts.length];
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java
index ad279ee..1ba4387 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapreduce.util;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.net.NetUtils;
@Private
@Unstable
@@ -56,10 +57,7 @@ public class HostUtil {
public static String convertTrackerNameToHostName(String trackerName) {
// Ugly!
// Convert the trackerName to its host name
- int indexOfColon = trackerName.indexOf(":");
- String trackerHostName = (indexOfColon == -1) ?
- trackerName :
- trackerName.substring(0, indexOfColon);
+ String trackerHostName = NetUtils.getHostFromHostPort(trackerName);
return trackerHostName.substring("tracker_".length());
}
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
index 40faf06..c6f5b53 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
@@ -91,6 +91,7 @@ import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
import
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
+import com.google.common.net.HostAndPort;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -173,10 +174,12 @@ public class HistoryClientService extends AbstractService
{
.withXFSProtection(JHAdminConfig.MR_HISTORY_XFS_PREFIX)
.withAppClientProtocol(appClientProtocol)
.at(NetUtils.getHostPortString(bindAddress)).start(webApp);
-
- String connectHost =
MRWebAppUtil.getJHSWebappURLWithoutScheme(conf).split(":")[0];
- MRWebAppUtil.setJHSWebappURLWithoutScheme(conf,
- connectHost + ":" + webApp.getListenerAddress().getPort());
+
+ String connectHost = MRWebAppUtil.getJHSWebappURLWithoutScheme(conf);
+
+ MRWebAppUtil.setJHSWebappURLWithoutScheme(conf, HostAndPort
+ .fromParts(HostAndPort.fromString(connectHost).getHost(),
+ webApp.getListenerAddress().getPort()).toString());
}
@Override
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestMRCJCSocketFactory.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestMRCJCSocketFactory.java
index 123947a..e89b62f 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestMRCJCSocketFactory.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestMRCJCSocketFactory.java
@@ -37,6 +37,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.Assert;
import org.junit.Test;
+import com.google.common.net.HostAndPort;
+
/**
* This class checks that RPCs can use specialized socket factories.
*/
@@ -89,9 +91,9 @@ public class TestMRCJCSocketFactory {
"org.apache.hadoop.ipc.DummySocketFactory");
jconf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
String rmAddress = jconf.get(YarnConfiguration.RM_ADDRESS);
- String[] split = rmAddress.split(":");
- jconf.set(YarnConfiguration.RM_ADDRESS, split[0] + ':'
- + (Integer.parseInt(split[1]) + 10));
+ HostAndPort hp = HostAndPort.fromString(rmAddress);
+ jconf.set("yarn.resourcemanager.address",
+ hp.getHost() + ':' + (hp.getPort() + 10));
client = new JobClient(jconf);
JobStatus[] jobs = client.jobsToComplete();
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
index 303857b..1f53743 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.util.HostUtil;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.Shell;
@@ -342,7 +343,7 @@ public class ReliabilityTest extends Configured implements
Tool {
LOG.info(new Date() + " Stopping a few trackers");
for (String tracker : trackerNamesList) {
- String host = convertTrackerNameToHostName(tracker);
+ String host = HostUtil.convertTrackerNameToHostName(tracker);
LOG.info(new Date() + " Marking tracker on host: " + host);
fos.write((host + "\n").getBytes());
if (count++ >= trackerNamesList.size()/2) {
@@ -381,15 +382,6 @@ public class ReliabilityTest extends Configured implements
Tool {
}
}
- private String convertTrackerNameToHostName(String trackerName) {
- // Convert the trackerName to it's host name
- int indexOfColon = trackerName.indexOf(":");
- String trackerHostName = (indexOfColon == -1) ?
- trackerName :
- trackerName.substring(0, indexOfColon);
- return trackerHostName.substring("tracker_".length());
- }
-
}
private class KillTaskThread extends Thread {
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
index 5972f65..d948146 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
@@ -151,6 +151,8 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.net.HostAndPort;
+
public class TestClientRedirect {
static {
@@ -325,9 +327,9 @@ public class TestClientRedirect {
application.setYarnApplicationState(YarnApplicationState.FINISHED);
application.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
}
- String[] split = AMHOSTADDRESS.split(":");
- application.setHost(split[0]);
- application.setRpcPort(Integer.parseInt(split[1]));
+ HostAndPort hp = HostAndPort.fromString(AMHOSTADDRESS);
+ application.setHost(hp.getHost());
+ application.setRpcPort(hp.getPort());
application.setUser("TestClientRedirect-user");
application.setName("N/A");
application.setQueue("N/A");
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
index 4a7c328..a52cf08 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
@@ -834,20 +834,4 @@ public class UtilsForTests {
file.close();
return file;
}
-
- /**
- * This formats the long tasktracker name to just the FQDN
- * @param taskTrackerLong String The long format of the tasktracker string
- * @return String The FQDN of the tasktracker
- * @throws Exception
- */
- public static String getFQDNofTT (String taskTrackerLong) throws Exception {
- //Getting the exact FQDN of the tasktracker from the tasktracker string.
- String[] firstSplit = taskTrackerLong.split("_");
- String tmpOutput = firstSplit[1];
- String[] secondSplit = tmpOutput.split(":");
- String tmpTaskTracker = secondSplit[0];
- return tmpTaskTracker;
- }
-
}
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
index 1d1c083..28bae49 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
@@ -44,6 +44,7 @@ import
org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.eclipse.jetty.util.ajax.JSON;
+import com.google.common.net.HostAndPort;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -193,8 +194,8 @@ public class MiniHadoopClusterManager {
map.put("namenode_port", dfs.getNameNodePort());
}
if (mr != null) {
- map.put("resourcemanager_port", mr.getConfig().get(
- YarnConfiguration.RM_ADDRESS).split(":")[1]);
+ map.put("resourcemanager_port", HostAndPort.fromString(
+ mr.getConfig().get(YarnConfiguration.RM_ADDRESS)).getPort());
}
FileWriter fw = new FileWriter(new File(writeDetails));
fw.write(new JSON().toJSON(map));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
