This is an automated email from the ASF dual-hosted git repository. ayushsaxena pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push: new b1821be HDFS-13736. BlockPlacementPolicyDefault can not choose favored nodes when 'dfs.namenode.block-placement-policy.default.prefer-local-node' set to false. Contributed by hu xiaodong. b1821be is described below commit b1821be1ec20bb7d9a11789e97c59b29939cee9b Author: Ayush Saxena <ayushsax...@apache.org> AuthorDate: Fri Nov 1 22:41:50 2019 +0530 HDFS-13736. BlockPlacementPolicyDefault can not choose favored nodes when 'dfs.namenode.block-placement-policy.default.prefer-local-node' set to false. Contributed by hu xiaodong. --- .../BlockPlacementPolicyDefault.java | 44 +++++++++++++++++----- .../blockmanagement/TestReplicationPolicy.java | 23 +++++++++++ 2 files changed, 58 insertions(+), 9 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 5c68c93..239ce9d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -239,9 +239,10 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { DatanodeDescriptor favoredNode = favoredNodes.get(i); // Choose a single node which is local to favoredNode. // 'results' is updated within chooseLocalNode - final DatanodeStorageInfo target = - chooseLocalStorage(favoredNode, favoriteAndExcludedNodes, blocksize, - maxNodesPerRack, results, avoidStaleNodes, storageTypes, false); + final DatanodeStorageInfo target = chooseLocalOrFavoredStorage( + favoredNode, true, favoriteAndExcludedNodes, blocksize, + maxNodesPerRack, results, avoidStaleNodes, storageTypes); + if (target == null) { LOG.warn("Could not find a target for file " + src + " with favored node " + favoredNode); @@ -550,16 +551,41 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { List<DatanodeStorageInfo> results, boolean avoidStaleNodes, EnumMap<StorageType, Integer> storageTypes) throws NotEnoughReplicasException { + return chooseLocalOrFavoredStorage(localMachine, false, + excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, + storageTypes); + } + + /** + * Choose storage of local or favored node. + * @param localOrFavoredNode local or favored node + * @param isFavoredNode if target node is favored node + * @param excludedNodes datanodes that should not be considered as targets + * @param blocksize size of the data to be written + * @param maxNodesPerRack max nodes allowed per rack + * @param results the target nodes already chosen + * @param avoidStaleNodes avoid stale nodes in replica choosing + * @param storageTypes storage type to be considered for target + * @return storage of local or favored node (not chosen node) + * @throws NotEnoughReplicasException + */ + protected DatanodeStorageInfo chooseLocalOrFavoredStorage( + Node localOrFavoredNode, boolean isFavoredNode, Set<Node> excludedNodes, + long blocksize, int maxNodesPerRack, List<DatanodeStorageInfo> results, + boolean avoidStaleNodes, EnumMap<StorageType, Integer> storageTypes) + throws NotEnoughReplicasException { // if no local machine, randomly choose one node - if (localMachine == null) { + if (localOrFavoredNode == null) { return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes); } - if (preferLocalNode && localMachine instanceof DatanodeDescriptor - && clusterMap.contains(localMachine)) { - DatanodeDescriptor localDatanode = (DatanodeDescriptor) localMachine; + if ((preferLocalNode || isFavoredNode) + && localOrFavoredNode instanceof DatanodeDescriptor + && clusterMap.contains(localOrFavoredNode)) { + DatanodeDescriptor localDatanode = + (DatanodeDescriptor) localOrFavoredNode; // otherwise try local machine first - if (excludedNodes.add(localMachine) // was not in the excluded list + if (excludedNodes.add(localOrFavoredNode) // was not in the excluded list && isGoodDatanode(localDatanode, maxNodesPerRack, false, results, avoidStaleNodes)) { for (Iterator<Map.Entry<StorageType, Integer>> iter = storageTypes @@ -579,7 +605,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { return localStorage; } } - } + } } return null; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index f08fa13..a6db359 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -1519,6 +1519,29 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest { } } + @Test + public void testChooseFromFavoredNodesWhenPreferLocalSetToFalse() { + ((BlockPlacementPolicyDefault) replicator).setPreferLocalNode(false); + try { + DatanodeStorageInfo[] targets; + List<DatanodeDescriptor> expectedTargets = new ArrayList<>(); + expectedTargets.add(dataNodes[0]); + expectedTargets.add(dataNodes[2]); + List<DatanodeDescriptor> favouredNodes = new ArrayList<>(); + favouredNodes.add(dataNodes[0]); + favouredNodes.add(dataNodes[2]); + targets = chooseTarget(2, dataNodes[3], null, + favouredNodes); + assertEquals(targets.length, 2); + for (int i = 0; i < targets.length; i++) { + assertTrue("Target should be a part of Expected Targets", + expectedTargets.contains(targets[i].getDatanodeDescriptor())); + } + } finally { + ((BlockPlacementPolicyDefault) replicator).setPreferLocalNode(true); + } + } + private DatanodeStorageInfo[] chooseTarget(int numOfReplicas, DatanodeDescriptor writer, Set<Node> excludedNodes, List<DatanodeDescriptor> favoredNodes) { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org