This is an automated email from the ASF dual-hosted git repository. aajisaka pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit a24c98163fc4d2d9f9cf2621105da8a98177c436 Author: huhaiyang <[email protected]> AuthorDate: Sat Dec 4 01:19:36 2021 +0800 HDFS-16314. Support to make dfs.namenode.block-placement-policy.exclude-slow-nodes.enabled reconfigurable (#3664) Reviewed-by: Fei Hui <[email protected]> Signed-off-by: Akira Ajisaka <[email protected]> (cherry picked from commit c0f405a46b402b4bdf5b7771f5124790fc866965) --- .../hdfs/server/blockmanagement/BlockManager.java | 10 ++++++++ .../blockmanagement/BlockPlacementPolicy.java | 12 ++++++++++ .../BlockPlacementPolicyDefault.java | 12 +++++++++- .../hadoop/hdfs/server/namenode/NameNode.java | 27 ++++++++++++++++------ .../server/namenode/TestNameNodeReconfigure.java | 22 ++++++++++++++++-- .../org/apache/hadoop/hdfs/tools/TestDFSAdmin.java | 6 +++-- 6 files changed, 77 insertions(+), 12 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 2f156b3..97cb313 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -5319,4 +5319,14 @@ public class BlockManager implements BlockStatsMXBean { public StoragePolicySatisfyManager getSPSManager() { return spsManager; } + + public void setExcludeSlowNodesEnabled(boolean enable) { + placementPolicies.getPolicy(CONTIGUOUS).setExcludeSlowNodesEnabled(enable); + placementPolicies.getPolicy(STRIPED).setExcludeSlowNodesEnabled(enable); + } + + @VisibleForTesting + public boolean getExcludeSlowNodesEnabled(BlockType blockType) { + return placementPolicies.getPolicy(blockType).getExcludeSlowNodesEnabled(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java index 07df43b..8752410 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java @@ -261,4 +261,16 @@ public abstract class BlockPlacementPolicy { } } } + + /** + * Updates the value used for excludeSlowNodesEnabled, which is set by + * {@code DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY} + * initially. + * + * @param enable true, we will filter out slow nodes + * when choosing targets for blocks, otherwise false not filter. + */ + public abstract void setExcludeSlowNodesEnabled(boolean enable); + + public abstract boolean getExcludeSlowNodesEnabled(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 2d0b721..bd10f99 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -103,7 +103,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { protected double considerLoadFactor; private boolean preferLocalNode; private boolean dataNodePeerStatsEnabled; - private boolean excludeSlowNodesEnabled; + private volatile boolean excludeSlowNodesEnabled; protected NetworkTopology clusterMap; protected Host2NodesMap host2datanodeMap; private FSClusterStats stats; @@ -1359,5 +1359,15 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { void setPreferLocalNode(boolean prefer) { this.preferLocalNode = prefer; } + + @Override + public void setExcludeSlowNodesEnabled(boolean enable) { + this.excludeSlowNodesEnabled = enable; + } + + @Override + public boolean getExcludeSlowNodesEnabled() { + return excludeSlowNodesEnabled; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 05648bb..9bb6cd8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -188,6 +188,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAM import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT; import static org.apache.hadoop.util.ExitUtil.terminate; import static org.apache.hadoop.util.ToolRunner.confirmPrompt; @@ -330,7 +332,8 @@ public class NameNode extends ReconfigurableBase implements DFS_BLOCK_REPLICATOR_CLASSNAME_KEY, DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY, DFS_IMAGE_PARALLEL_LOAD_KEY, - DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY)); + DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY, + DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY)); private static final String USAGE = "Usage: hdfs namenode [" + StartupOption.BACKUP.getName() + "] | \n\t[" @@ -2196,7 +2199,8 @@ public class NameNode extends ReconfigurableBase implements return newVal; } else if (property.equals(DFS_IMAGE_PARALLEL_LOAD_KEY)) { return reconfigureParallelLoad(newVal); - } else if (property.equals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY)) { + } else if (property.equals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY) + || (property.equals(DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY))) { return reconfigureSlowNodesParameters(datanodeManager, property, newVal); } else { throw new ReconfigurationException(property, newVal, getConf().get( @@ -2230,7 +2234,7 @@ public class NameNode extends ReconfigurableBase implements newSetting = bm.getBlocksReplWorkMultiplier(); } else { throw new IllegalArgumentException("Unexpected property " + - property + "in reconfReplicationParameters"); + property + " in reconfReplicationParameters"); } LOG.info("RECONFIGURE* changed {} to {}", property, newSetting); return String.valueOf(newSetting); @@ -2386,15 +2390,24 @@ public class NameNode extends ReconfigurableBase implements String reconfigureSlowNodesParameters(final DatanodeManager datanodeManager, final String property, final String newVal) throws ReconfigurationException { + BlockManager bm = namesystem.getBlockManager(); namesystem.writeLock(); boolean enable; try { - if (newVal == null) { - enable = DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_DEFAULT; + if (property.equals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY)) { + enable = (newVal == null ? DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_DEFAULT : + Boolean.parseBoolean(newVal)); + datanodeManager.setAvoidSlowDataNodesForReadEnabled(enable); + } else if (property.equals( + DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY)) { + enable = (newVal == null ? + DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_DEFAULT : + Boolean.parseBoolean(newVal)); + bm.setExcludeSlowNodesEnabled(enable); } else { - enable = Boolean.parseBoolean(newVal); + throw new IllegalArgumentException("Unexpected property " + + property + " in reconfigureSlowNodesParameters"); } - datanodeManager.setAvoidSlowDataNodesForReadEnabled(enable); LOG.info("RECONFIGURE* changed {} to {}", property, newVal); return Boolean.toString(enable); } catch (IllegalArgumentException e) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java index eed442c..da9b447 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java @@ -34,10 +34,12 @@ import org.apache.hadoop.conf.ReconfigurationException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants.StoragePolicySatisfierMode; +import org.apache.hadoop.hdfs.protocol.BlockType; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfyManager; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.test.GenericTestUtils; @@ -52,6 +54,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MODE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_BACKOFF_ENABLE_DEFAULT; public class TestNameNodeReconfigure { @@ -399,8 +402,8 @@ public class TestNameNodeReconfigure { public void testEnableSlowNodesParametersAfterReconfigured() throws ReconfigurationException { final NameNode nameNode = cluster.getNameNode(); - final DatanodeManager datanodeManager = nameNode.namesystem - .getBlockManager().getDatanodeManager(); + final BlockManager blockManager = nameNode.namesystem.getBlockManager(); + final DatanodeManager datanodeManager = blockManager.getDatanodeManager(); // By default, avoidSlowDataNodesForRead is false. assertEquals(false, datanodeManager.getEnableAvoidSlowDataNodesForRead()); @@ -410,6 +413,21 @@ public class TestNameNodeReconfigure { // After reconfigured, avoidSlowDataNodesForRead is true. assertEquals(true, datanodeManager.getEnableAvoidSlowDataNodesForRead()); + + // By default, excludeSlowNodesEnabled is false. + assertEquals(false, blockManager. + getExcludeSlowNodesEnabled(BlockType.CONTIGUOUS)); + assertEquals(false, blockManager. + getExcludeSlowNodesEnabled(BlockType.STRIPED)); + + nameNode.reconfigureProperty( + DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY, Boolean.toString(true)); + + // After reconfigured, excludeSlowNodesEnabled is true. + assertEquals(true, blockManager. + getExcludeSlowNodesEnabled(BlockType.CONTIGUOUS)); + assertEquals(true, blockManager. + getExcludeSlowNodesEnabled(BlockType.STRIPED)); } @After diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java index 5e47334..b719dd6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java @@ -26,6 +26,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHEC import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY; import java.util.function.Supplier; import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; @@ -423,13 +424,14 @@ public class TestDFSAdmin { final List<String> outs = Lists.newArrayList(); final List<String> errs = Lists.newArrayList(); getReconfigurableProperties("namenode", address, outs, errs); - assertEquals(14, outs.size()); + assertEquals(15, outs.size()); assertEquals(DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY, outs.get(1)); assertEquals(DFS_BLOCK_REPLICATOR_CLASSNAME_KEY, outs.get(2)); assertEquals(DFS_HEARTBEAT_INTERVAL_KEY, outs.get(3)); assertEquals(DFS_IMAGE_PARALLEL_LOAD_KEY, outs.get(4)); assertEquals(DFS_NAMENODE_AVOID_SLOW_DATANODE_FOR_READ_KEY, outs.get(5)); - assertEquals(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, outs.get(6)); + assertEquals(DFS_NAMENODE_BLOCKPLACEMENTPOLICY_EXCLUDE_SLOW_NODES_ENABLED_KEY, outs.get(6)); + assertEquals(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, outs.get(7)); assertEquals(errs.size(), 0); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
