Author: cmccabe Date: Tue Oct 29 21:00:34 2013 New Revision: 1536885 URL: http://svn.apache.org/r1536885 Log: HDFS-4657. Limit the number of blocks logged by the NN after a block report to a configurable value. (atm via cmccabe)
Modified: hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Modified: hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1536885&r1=1536884&r2=1536885&view=diff ============================================================================== --- hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Oct 29 21:00:34 2013 @@ -15,6 +15,10 @@ Release 2.2.1 - UNRELEASED HDFS-5331. make SnapshotDiff.java to a o.a.h.util.Tool interface implementation. (Vinayakumar B via umamahesh) + HDFS-4657. Limit the number of blocks logged by the NN after a block + report to a configurable value. (Aaron Twinning Meyers via Colin Patrick + McCabe) + OPTIMIZATIONS BUG FIXES Modified: hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1536885&r1=1536884&r2=1536885&view=diff ============================================================================== --- hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original) +++ hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Tue Oct 29 21:00:34 2013 @@ -490,6 +490,9 @@ public class DFSConfigKeys extends Commo public static final int DFS_QJOURNAL_NEW_EPOCH_TIMEOUT_DEFAULT = 120000; public static final int DFS_QJOURNAL_WRITE_TXNS_TIMEOUT_DEFAULT = 20000; + public static final String DFS_MAX_NUM_BLOCKS_TO_LOG_KEY = "dfs.namenode.max-num-blocks-to-log"; + public static final long DFS_MAX_NUM_BLOCKS_TO_LOG_DEFAULT = 1000l; + public static final String DFS_NAMENODE_ENABLE_RETRY_CACHE_KEY = "dfs.namenode.enable.retrycache"; public static final boolean DFS_NAMENODE_ENABLE_RETRY_CACHE_DEFAULT = true; public static final String DFS_NAMENODE_RETRY_CACHE_EXPIRYTIME_MILLIS_KEY = "dfs.namenode.retrycache.expirytime.millis"; Modified: hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1536885&r1=1536884&r2=1536885&view=diff ============================================================================== --- hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original) +++ hadoop/common/branches/branch-2.2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Tue Oct 29 21:00:34 2013 @@ -216,6 +216,9 @@ public class BlockManager { // whether or not to issue block encryption keys. final boolean encryptDataTransfer; + + // Max number of blocks to log info about during a block report. + private final long maxNumBlocksToLog; /** * When running inside a Standby node, the node may receive block reports @@ -297,6 +300,10 @@ public class BlockManager { conf.getBoolean(DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY, DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_DEFAULT); + this.maxNumBlocksToLog = + conf.getLong(DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_KEY, + DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_DEFAULT); + LOG.info("defaultReplication = " + defaultReplication); LOG.info("maxReplication = " + maxReplication); LOG.info("minReplication = " + minReplication); @@ -304,6 +311,7 @@ public class BlockManager { LOG.info("shouldCheckForEnoughRacks = " + shouldCheckForEnoughRacks); LOG.info("replicationRecheckInterval = " + replicationRecheckInterval); LOG.info("encryptDataTransfer = " + encryptDataTransfer); + LOG.info("maxNumBlocksToLog = " + maxNumBlocksToLog); } private static BlockTokenSecretManager createBlockTokenSecretManager( @@ -1700,8 +1708,14 @@ public class BlockManager { for (Block b : toRemove) { removeStoredBlock(b, node); } + int numBlocksLogged = 0; for (BlockInfo b : toAdd) { - addStoredBlock(b, node, null, true); + addStoredBlock(b, node, null, numBlocksLogged < maxNumBlocksToLog); + numBlocksLogged++; + } + if (numBlocksLogged > maxNumBlocksToLog) { + blockLog.info("BLOCK* processReport: logged info for " + maxNumBlocksToLog + + " of " + numBlocksLogged + " reported."); } for (Block b : toInvalidate) { blockLog.info("BLOCK* processReport: " @@ -2641,8 +2655,14 @@ assert storedBlock.findDatanode(dn) < 0 for (StatefulBlockInfo b : toUC) { addStoredBlockUnderConstruction(b.storedBlock, node, b.reportedState); } + long numBlocksLogged = 0; for (BlockInfo b : toAdd) { - addStoredBlock(b, node, delHintNode, true); + addStoredBlock(b, node, delHintNode, numBlocksLogged < maxNumBlocksToLog); + numBlocksLogged++; + } + if (numBlocksLogged > maxNumBlocksToLog) { + blockLog.info("BLOCK* addBlock: logged info for " + maxNumBlocksToLog + + " of " + numBlocksLogged + " reported."); } for (Block b : toInvalidate) { blockLog.info("BLOCK* addBlock: block "