HDFS-7579. Improve log reporting during block report rpc failure. Contributed by Charles Lamb.
(cherry picked from commit 7e2d9a32426d04b5f08c2835f61882b053612a20) (cherry picked from commit f0acb7c2a284db61640efee15a1648c6c26d24f5) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33fb7b45 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33fb7b45 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33fb7b45 Branch: refs/heads/sjlee/hdfs-merge Commit: 33fb7b45195a89b6464e4a1cb3fbf6bbad2bcecb Parents: ba28192 Author: cnauroth <cnaur...@apache.org> Authored: Thu Jan 8 15:12:56 2015 -0800 Committer: Sangjin Lee <sj...@apache.org> Committed: Wed Aug 12 22:15:04 2015 -0700 ---------------------------------------------------------------------- .../hdfs/server/datanode/BPServiceActor.java | 76 ++++++++++++-------- .../hdfs/server/protocol/ServerCommand.java | 8 +++ 2 files changed, 54 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/33fb7b45/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 2961698..6bdb68a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -22,7 +22,10 @@ import static org.apache.hadoop.util.Time.now; import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketTimeoutException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; import com.google.common.base.Joiner; import org.apache.commons.logging.Log; @@ -457,7 +460,7 @@ class BPServiceActor implements Runnable { return null; } - ArrayList<DatanodeCommand> cmds = new ArrayList<DatanodeCommand>(); + final ArrayList<DatanodeCommand> cmds = new ArrayList<DatanodeCommand>(); // Flush any block information that precedes the block report. Otherwise // we have a chance that we will miss the delHint information @@ -484,40 +487,54 @@ class BPServiceActor implements Runnable { } // Send the reports to the NN. - int numReportsSent; + int numReportsSent = 0; + int numRPCs = 0; + boolean success = false; long brSendStartTime = now(); - if (totalBlockCount < dnConf.blockReportSplitThreshold) { - // Below split threshold, send all reports in a single message. - numReportsSent = 1; - DatanodeCommand cmd = - bpNamenode.blockReport(bpRegistration, bpos.getBlockPoolId(), reports); - if (cmd != null) { - cmds.add(cmd); - } - } else { - // Send one block report per message. - numReportsSent = i; - for (StorageBlockReport report : reports) { - StorageBlockReport singleReport[] = { report }; + try { + if (totalBlockCount < dnConf.blockReportSplitThreshold) { + // Below split threshold, send all reports in a single message. DatanodeCommand cmd = bpNamenode.blockReport( - bpRegistration, bpos.getBlockPoolId(), singleReport); + bpRegistration, bpos.getBlockPoolId(), reports); + numRPCs = 1; + numReportsSent = reports.length; if (cmd != null) { cmds.add(cmd); } + } else { + // Send one block report per message. + for (StorageBlockReport report : reports) { + StorageBlockReport singleReport[] = { report }; + DatanodeCommand cmd = bpNamenode.blockReport( + bpRegistration, bpos.getBlockPoolId(), singleReport); + numReportsSent++; + numRPCs++; + if (cmd != null) { + cmds.add(cmd); + } + } } + success = true; + } finally { + // Log the block report processing stats from Datanode perspective + long brSendCost = now() - brSendStartTime; + long brCreateCost = brSendStartTime - brCreateStartTime; + dn.getMetrics().addBlockReport(brSendCost); + final int nCmds = cmds.size(); + LOG.info((success ? "S" : "Uns") + + "uccessfully sent " + numReportsSent + + " of " + reports.length + + " blockreports for " + totalBlockCount + + " total blocks using " + numRPCs + + " RPCs. This took " + brCreateCost + + " msec to generate and " + brSendCost + + " msecs for RPC and NN processing." + + " Got back " + + ((nCmds == 0) ? "no commands" : + ((nCmds == 1) ? "one command: " + cmds.get(0) : + (nCmds + " commands: " + Joiner.on("; ").join(cmds)))) + + "."); } - - // Log the block report processing stats from Datanode perspective - long brSendCost = now() - brSendStartTime; - long brCreateCost = brSendStartTime - brCreateStartTime; - dn.getMetrics().addBlockReport(brSendCost); - LOG.info("Sent " + numReportsSent + " blockreports " + totalBlockCount + - " blocks total. Took " + brCreateCost + - " msec to generate and " + brSendCost + - " msecs for RPC and NN processing. " + - " Got back commands " + - (cmds.size() == 0 ? "none" : Joiner.on("; ").join(cmds))); - scheduleNextBlockReport(startTime); return cmds.size() == 0 ? null : cmds; } @@ -967,7 +984,6 @@ class BPServiceActor implements Runnable { /** * Add pending incremental block report for a single block. - * @param blockID * @param blockInfo */ void putBlockInfo(ReceivedDeletedBlockInfo blockInfo) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/33fb7b45/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java index 8d7544d..eed9a6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java @@ -52,4 +52,12 @@ public abstract class ServerCommand { public int getAction() { return this.action; } + + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()); + sb.append("/"); + sb.append(action); + return sb.toString(); + } }
