This is an automated email from the ASF dual-hosted git repository. brahma pushed a commit to branch HADOOP-17800 in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/HADOOP-17800 by this push: new e28e1cb HADOOP-12432. Add support for include/exclude lists on IPv6 setup. Contributed by Nemanja Matkovic And Hemanth Boyina. e28e1cb is described below commit e28e1cbf5c2ad14226c669eb4f4dba83aab858be Author: Brahma Reddy Battula <bra...@apache.org> AuthorDate: Fri Jul 30 08:31:31 2021 +0530 HADOOP-12432. Add support for include/exclude lists on IPv6 setup. Contributed by Nemanja Matkovic And Hemanth Boyina. --- .../server/blockmanagement/HostFileManager.java | 9 ++-- .../blockmanagement/TestHostFileManager.java | 49 +++++++++++++++------- .../hdfs/server/namenode/TestHostsFiles.java | 9 ++-- .../apache/hadoop/hdfs/util/HostsFileWriter.java | 11 +++-- 4 files changed, 49 insertions(+), 29 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostFileManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostFileManager.java index 57b6902..dcbd131 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostFileManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HostFileManager.java @@ -23,12 +23,11 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.HostsFileReader; import java.io.IOException; import java.net.InetSocketAddress; -import java.net.URI; -import java.net.URISyntaxException; import java.util.HashSet; /** @@ -89,16 +88,14 @@ public class HostFileManager extends HostConfigManager { @VisibleForTesting static InetSocketAddress parseEntry(String type, String fn, String line) { try { - URI uri = new URI("dummy", line, null, null, null); - int port = uri.getPort() == -1 ? 0 : uri.getPort(); - InetSocketAddress addr = new InetSocketAddress(uri.getHost(), port); + InetSocketAddress addr = NetUtils.createSocketAddr(line, 0); if (addr.isUnresolved()) { LOG.warn(String.format("Failed to resolve address `%s` in `%s`. " + "Ignoring in the %s list.", line, fn, type)); return null; } return addr; - } catch (URISyntaxException e) { + } catch (IllegalArgumentException e) { LOG.warn(String.format("Failed to parse `%s` in `%s`. " + "Ignoring in " + "the %s list.", line, fn, type)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHostFileManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHostFileManager.java index 38d0905..2139ac5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHostFileManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHostFileManager.java @@ -110,13 +110,19 @@ public class TestHostFileManager { includedNodes.add(entry("127.0.0.1:12345")); includedNodes.add(entry("localhost:12345")); includedNodes.add(entry("127.0.0.1:12345")); + + includedNodes.add(entry("[::1]:42")); + includedNodes.add(entry("[0:0:0:0:0:0:0:1]:42")); + includedNodes.add(entry("[::1]:42")); + includedNodes.add(entry("127.0.0.2")); excludedNodes.add(entry("127.0.0.1:12346")); excludedNodes.add(entry("127.0.30.1:12346")); + excludedNodes.add(entry("[::1]:24")); - Assert.assertEquals(2, includedNodes.size()); - Assert.assertEquals(2, excludedNodes.size()); + Assert.assertEquals(3, includedNodes.size()); + Assert.assertEquals(3, excludedNodes.size()); hm.refresh(includedNodes, excludedNodes); @@ -125,20 +131,33 @@ public class TestHostFileManager { Map<String, DatanodeDescriptor> dnMap = (Map<String, DatanodeDescriptor>) Whitebox.getInternalState(dm, "datanodeMap"); - // After the de-duplication, there should be only one DN from the included + // After the de-duplication, there should be three DN from the included // nodes declared as dead. - Assert.assertEquals(2, dm.getDatanodeListForReport(HdfsConstants - .DatanodeReportType.ALL).size()); - Assert.assertEquals(2, dm.getDatanodeListForReport(HdfsConstants - .DatanodeReportType.DEAD).size()); - dnMap.put("uuid-foo", new DatanodeDescriptor(new DatanodeID("127.0.0.1", - "localhost", "uuid-foo", 12345, 1020, 1021, 1022))); - Assert.assertEquals(1, dm.getDatanodeListForReport(HdfsConstants - .DatanodeReportType.DEAD).size()); - dnMap.put("uuid-bar", new DatanodeDescriptor(new DatanodeID("127.0.0.2", - "127.0.0.2", "uuid-bar", 12345, 1020, 1021, 1022))); - Assert.assertEquals(0, dm.getDatanodeListForReport(HdfsConstants - .DatanodeReportType.DEAD).size()); + Assert.assertEquals(3, + dm.getDatanodeListForReport(HdfsConstants.DatanodeReportType.ALL) + .size()); + Assert.assertEquals(3, + dm.getDatanodeListForReport(HdfsConstants.DatanodeReportType.DEAD) + .size()); + dnMap.put("uuid-foo", new DatanodeDescriptor( + new DatanodeID("127.0.0.1", "localhost", "uuid-foo", 12345, 1020, 1021, + 1022))); + Assert.assertEquals(2, + dm.getDatanodeListForReport(HdfsConstants.DatanodeReportType.DEAD) + .size()); + dnMap.put("uuid-bar", new DatanodeDescriptor( + new DatanodeID("127.0.0.2", "127.0.0.2", "uuid-bar", 12345, 1020, 1021, + 1022))); + Assert.assertEquals(1, + dm.getDatanodeListForReport(HdfsConstants.DatanodeReportType.DEAD) + .size()); + dnMap.put("uuid-baz", new DatanodeDescriptor( + new DatanodeID("[::1]", "localhost", "uuid-baz", 42, 1020, 1021, + 1022))); + Assert.assertEquals(0, + dm.getDatanodeListForReport(HdfsConstants.DatanodeReportType.DEAD) + .size()); + DatanodeDescriptor spam = new DatanodeDescriptor(new DatanodeID("127.0.0" + ".3", "127.0.0.3", "uuid-spam", 12345, 1020, 1021, 1022)); DFSTestUtil.setDatanodeDead(spam); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java index e86413d..4fa5a3f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestHostsFiles.java @@ -151,14 +151,15 @@ public class TestHostsFiles { HostsFileWriter hostsFileWriter = new HostsFileWriter(); hostsFileWriter.initialize(conf, "temp/decommission"); - hostsFileWriter.initIncludeHosts(new String[] - {"localhost:52","127.0.0.1:7777"}); + hostsFileWriter.initIncludeHosts( + new String[] {"localhost:52", "127.0.0.1:7777", "[::1]:42", + "[0:0:0:0:0:0:0:1]:24"}); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); final FSNamesystem ns = cluster.getNameNode().getNamesystem(); - assertTrue(ns.getNumDeadDataNodes() == 2); + assertTrue(ns.getNumDeadDataNodes() == 4); assertTrue(ns.getNumLiveDataNodes() == 0); // Testing using MBeans @@ -166,7 +167,7 @@ public class TestHostsFiles { ObjectName mxbeanName = new ObjectName( "Hadoop:service=NameNode,name=FSNamesystemState"); String nodes = mbs.getAttribute(mxbeanName, "NumDeadDataNodes") + ""; - assertTrue((Integer) mbs.getAttribute(mxbeanName, "NumDeadDataNodes") == 2); + assertTrue((Integer) mbs.getAttribute(mxbeanName, "NumDeadDataNodes") == 4); assertTrue((Integer) mbs.getAttribute(mxbeanName, "NumLiveDataNodes") == 0); } finally { if (cluster != null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/HostsFileWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/HostsFileWriter.java index e171e2b..d1c779c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/HostsFileWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/HostsFileWriter.java @@ -141,11 +141,14 @@ public class HostsFileWriter { includeHosts.toString()); } else { HashSet<DatanodeAdminProperties> allDNs = new HashSet<>(); - for(String hostNameAndPort : hostNameAndPorts) { - String[] hostAndPort = hostNameAndPort.split(":"); + for (String hostNameAndPort : hostNameAndPorts) { + int i = hostNameAndPort.lastIndexOf(':'); + String port = + hostNameAndPort.substring(hostNameAndPort.lastIndexOf(":") + 1); + String addr = hostNameAndPort.substring(0, i); DatanodeAdminProperties dn = new DatanodeAdminProperties(); - dn.setHostName(hostAndPort[0]); - dn.setPort(Integer.parseInt(hostAndPort[1])); + dn.setHostName(addr); + dn.setPort(Integer.parseInt(port)); allDNs.add(dn); } CombinedHostsFileWriter.writeFile(combinedFile.toString(), allDNs); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org