This is an automated email from the ASF dual-hosted git repository. surendralilhore pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push: new 14beaae HDFS-14909. DFSNetworkTopology#chooseRandomWithStorageType() should not decrease storage count for excluded node which is already part of excluded scope. Contributed by Surendra Singh Lilhore. 14beaae is described below commit 14beaae65e68725635e428c237c79ba434b8b076 Author: Surendra Singh Lilhore <surendralilh...@apache.org> AuthorDate: Thu Oct 17 22:28:30 2019 +0530 HDFS-14909. DFSNetworkTopology#chooseRandomWithStorageType() should not decrease storage count for excluded node which is already part of excluded scope. Contributed by Surendra Singh Lilhore. (cherry picked from commit 54dc6b7d720851eb6017906d664aa0fda2698225) --- .../apache/hadoop/hdfs/net/DFSNetworkTopology.java | 4 +++ .../apache/hadoop/hdfs/TestMissingBlocksAlert.java | 30 ++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java index 0884fc0..aea1ff6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java @@ -212,6 +212,10 @@ public class DFSNetworkTopology extends NetworkTopology { } if (excludedNodes != null) { for (Node excludedNode : excludedNodes) { + if (excludeRoot != null + && excludedNode.getNetworkLocation().startsWith(excludedScope)) { + continue; + } if (excludedNode instanceof DatanodeDescriptor) { availableCount -= ((DatanodeDescriptor) excludedNode) .hasStorageType(type) ? 1 : 0; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java index ca2fe92..7370a28 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMissingBlocksAlert.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceBlockPlacementPolicy; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.junit.Assert; import org.junit.Test; @@ -145,4 +146,33 @@ public class TestMissingBlocksAlert { } } } + + @Test + public void testMissReplicatedBlockwithTwoRack() throws Exception { + Configuration conf = new Configuration(); + //Start cluster with rack /default/rack1 + String[] hosts = new String[] {"host0", "host1", "host2", "host3"}; + String[] racks = new String[] {"/default/rack1", "/default/rack1", + "/default/rack1", "/default/rack1"}; + conf.set(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY, + AvailableSpaceBlockPlacementPolicy.class.getName()); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4) + .hosts(hosts).racks(racks).build(); + Path file = new Path("/file2"); + try { + DistributedFileSystem dfs = cluster.getFileSystem(); + DFSTestUtil.createFile(dfs, file, 1024, (short) 2, 0); + dfs.getFileStatus(file); + //Add one more rack /default/rack2 + cluster.startDataNodes(conf, 2, true, null, + new String[] {"/default/rack2", "/default/rack2"}, + new String[] {"host4", "host5"}, null); + dfs.setReplication(file, (short) 3); + // wait for block replication + DFSTestUtil.waitForReplication(dfs, file, (short) 3, 60000); + } finally { + cluster.shutdown(); + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org