Repository: hadoop Updated Branches: refs/heads/trunk e0b570dff -> 10e84c6a6
HDFS-10694. processReport() should print blockReportId in each log message. Contributed by Yuanbo Liu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10e84c6a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10e84c6a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10e84c6a Branch: refs/heads/trunk Commit: 10e84c6a6e831fe2bea061fb21bd0dfe32bc9953 Parents: e0b570d Author: Yuanbo Liu <liuyuanb...@gmail.com> Authored: Wed Aug 10 10:48:42 2016 -0700 Committer: Konstantin V Shvachko <s...@apache.org> Committed: Wed Aug 10 11:07:42 2016 -0700 ---------------------------------------------------------------------- .../server/blockmanagement/BlockManager.java | 45 +++++++++++++------- 1 file changed, 29 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/10e84c6a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 0e7a59b..9e3b935 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -2153,6 +2153,8 @@ public class BlockManager implements BlockStatsMXBean { final long endTime; DatanodeDescriptor node; Collection<Block> invalidatedBlocks = Collections.emptyList(); + String strBlockReportId = + context != null ? Long.toHexString(context.getReportId()) : ""; try { node = datanodeManager.getDatanode(nodeID); @@ -2171,9 +2173,10 @@ public class BlockManager implements BlockStatsMXBean { } if (namesystem.isInStartupSafeMode() && storageInfo.getBlockReportCount() > 0) { - blockLog.info("BLOCK* processReport: " + blockLog.info("BLOCK* processReport 0x{}: " + "discarded non-initial block report from {}" - + " because namenode still in startup phase", nodeID); + + " because namenode still in startup phase", + strBlockReportId, nodeID); blockReportLeaseManager.removeLease(node); return !node.hasStaleStorages(); } @@ -2187,13 +2190,14 @@ public class BlockManager implements BlockStatsMXBean { if (storageInfo.getBlockReportCount() == 0) { // The first block report can be processed a lot more efficiently than // ordinary block reports. This shortens restart times. - LOG.info("Processing first storage report for " + - storageInfo.getStorageID() + " from datanode " + + blockLog.info("BLOCK* processReport 0x{}: Processing first " + + "storage report for {} from datanode {}", + strBlockReportId, + storageInfo.getStorageID(), nodeID.getDatanodeUuid()); processFirstBlockReport(storageInfo, newReport); } else { - invalidatedBlocks = processReport(storageInfo, newReport, - context != null ? context.isSorted() : false); + invalidatedBlocks = processReport(storageInfo, newReport, context); } storageInfo.receivedBlockReport(); @@ -2229,8 +2233,8 @@ public class BlockManager implements BlockStatsMXBean { } for (Block b : invalidatedBlocks) { - blockLog.debug("BLOCK* processReport: {} on node {} size {} does not " + - "belong to any file", b, node, b.getNumBytes()); + blockLog.debug("BLOCK* processReport 0x{}: {} on node {} size {} does not" + + " belong to any file", strBlockReportId, b, node, b.getNumBytes()); } // Log the block report processing stats from Namenode perspective @@ -2238,10 +2242,10 @@ public class BlockManager implements BlockStatsMXBean { if (metrics != null) { metrics.addBlockReport((int) (endTime - startTime)); } - blockLog.info("BLOCK* processReport: from storage {} node {}, " + + blockLog.info("BLOCK* processReport 0x{}: from storage {} node {}, " + "blocks: {}, hasStaleStorage: {}, processing time: {} msecs, " + - "invalidatedBlocks: {}", storage.getStorageID(), nodeID, - newReport.getNumberOfBlocks(), + "invalidatedBlocks: {}", strBlockReportId, storage.getStorageID(), + nodeID, newReport.getNumberOfBlocks(), node.hasStaleStorages(), (endTime - startTime), invalidatedBlocks.size()); return !node.hasStaleStorages(); @@ -2355,7 +2359,8 @@ public class BlockManager implements BlockStatsMXBean { private Collection<Block> processReport( final DatanodeStorageInfo storageInfo, - final BlockListAsLongs report, final boolean sorted) throws IOException { + final BlockListAsLongs report, + BlockReportContext context) throws IOException { // Normal case: // Modify the (block-->datanode) map, according to the difference // between the old and new block report. @@ -2366,13 +2371,21 @@ public class BlockManager implements BlockStatsMXBean { Collection<BlockToMarkCorrupt> toCorrupt = new LinkedList<>(); Collection<StatefulBlockInfo> toUC = new LinkedList<>(); + boolean sorted = false; + String strBlockReportId = ""; + if (context != null) { + sorted = context.isSorted(); + strBlockReportId = Long.toHexString(context.getReportId()); + } + Iterable<BlockReportReplica> sortedReport; if (!sorted) { - blockLog.warn("BLOCK* processReport: Report from the DataNode ({}) is " - + "unsorted. This will cause overhead on the NameNode " + blockLog.warn("BLOCK* processReport 0x{}: Report from the DataNode ({}) " + + "is unsorted. This will cause overhead on the NameNode " + "which needs to sort the Full BR. Please update the " + "DataNode to the same version of Hadoop HDFS as the " + "NameNode ({}).", + strBlockReportId, storageInfo.getDatanodeDescriptor().getDatanodeUuid(), VersionInfo.getVersion()); Set<BlockReportReplica> set = new FoldedTreeSet<>(); @@ -2403,8 +2416,8 @@ public class BlockManager implements BlockStatsMXBean { numBlocksLogged++; } if (numBlocksLogged > maxNumBlocksToLog) { - blockLog.info("BLOCK* processReport: logged info for {} of {} " + - "reported.", maxNumBlocksToLog, numBlocksLogged); + blockLog.info("BLOCK* processReport 0x{}: logged info for {} of {} " + + "reported.", strBlockReportId, maxNumBlocksToLog, numBlocksLogged); } for (Block b : toInvalidate) { addToInvalidates(b, node); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
