HDFS-7579. Improve log reporting during block report rpc failure. Contributed by Charles Lamb.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22af33bf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22af33bf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22af33bf Branch: refs/heads/HDFS-EC Commit: 22af33bfb42b50e4294c37540f09dd74e5a4f502 Parents: 786108d Author: cnauroth <[email protected]> Authored: Thu Jan 8 15:12:56 2015 -0800 Committer: Zhe Zhang <[email protected]> Committed: Mon Jan 12 10:18:02 2015 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/server/datanode/BPServiceActor.java | 76 ++++++++++++-------- .../hdfs/server/protocol/ServerCommand.java | 8 +++ 3 files changed, 57 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/22af33bf/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9c1be4a..1358edc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -656,6 +656,9 @@ Release 2.7.0 - UNRELEASED HDFS-7589. Break the dependency between libnative_mini_dfs and libhdfs. (Zhanwei Wang via cnauroth) + HDFS-7579. Improve log reporting during block report rpc failure. + (Charles Lamb via cnauroth) + Release 2.6.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/22af33bf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index d94d056..e6409ab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -22,7 +22,10 @@ import static org.apache.hadoop.util.Time.now; import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketTimeoutException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; import com.google.common.base.Joiner; import org.apache.commons.logging.Log; @@ -458,7 +461,7 @@ class BPServiceActor implements Runnable { return null; } - ArrayList<DatanodeCommand> cmds = new ArrayList<DatanodeCommand>(); + final ArrayList<DatanodeCommand> cmds = new ArrayList<DatanodeCommand>(); // Flush any block information that precedes the block report. Otherwise // we have a chance that we will miss the delHint information @@ -485,40 +488,54 @@ class BPServiceActor implements Runnable { } // Send the reports to the NN. - int numReportsSent; + int numReportsSent = 0; + int numRPCs = 0; + boolean success = false; long brSendStartTime = now(); - if (totalBlockCount < dnConf.blockReportSplitThreshold) { - // Below split threshold, send all reports in a single message. - numReportsSent = 1; - DatanodeCommand cmd = - bpNamenode.blockReport(bpRegistration, bpos.getBlockPoolId(), reports); - if (cmd != null) { - cmds.add(cmd); - } - } else { - // Send one block report per message. - numReportsSent = i; - for (StorageBlockReport report : reports) { - StorageBlockReport singleReport[] = { report }; + try { + if (totalBlockCount < dnConf.blockReportSplitThreshold) { + // Below split threshold, send all reports in a single message. DatanodeCommand cmd = bpNamenode.blockReport( - bpRegistration, bpos.getBlockPoolId(), singleReport); + bpRegistration, bpos.getBlockPoolId(), reports); + numRPCs = 1; + numReportsSent = reports.length; if (cmd != null) { cmds.add(cmd); } + } else { + // Send one block report per message. + for (StorageBlockReport report : reports) { + StorageBlockReport singleReport[] = { report }; + DatanodeCommand cmd = bpNamenode.blockReport( + bpRegistration, bpos.getBlockPoolId(), singleReport); + numReportsSent++; + numRPCs++; + if (cmd != null) { + cmds.add(cmd); + } + } } + success = true; + } finally { + // Log the block report processing stats from Datanode perspective + long brSendCost = now() - brSendStartTime; + long brCreateCost = brSendStartTime - brCreateStartTime; + dn.getMetrics().addBlockReport(brSendCost); + final int nCmds = cmds.size(); + LOG.info((success ? "S" : "Uns") + + "uccessfully sent " + numReportsSent + + " of " + reports.length + + " blockreports for " + totalBlockCount + + " total blocks using " + numRPCs + + " RPCs. This took " + brCreateCost + + " msec to generate and " + brSendCost + + " msecs for RPC and NN processing." + + " Got back " + + ((nCmds == 0) ? "no commands" : + ((nCmds == 1) ? "one command: " + cmds.get(0) : + (nCmds + " commands: " + Joiner.on("; ").join(cmds)))) + + "."); } - - // Log the block report processing stats from Datanode perspective - long brSendCost = now() - brSendStartTime; - long brCreateCost = brSendStartTime - brCreateStartTime; - dn.getMetrics().addBlockReport(brSendCost); - LOG.info("Sent " + numReportsSent + " blockreports " + totalBlockCount + - " blocks total. Took " + brCreateCost + - " msec to generate and " + brSendCost + - " msecs for RPC and NN processing. " + - " Got back commands " + - (cmds.size() == 0 ? "none" : Joiner.on("; ").join(cmds))); - scheduleNextBlockReport(startTime); return cmds.size() == 0 ? null : cmds; } @@ -968,7 +985,6 @@ class BPServiceActor implements Runnable { /** * Add pending incremental block report for a single block. - * @param blockID * @param blockInfo */ void putBlockInfo(ReceivedDeletedBlockInfo blockInfo) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/22af33bf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java index 8d7544d..eed9a6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java @@ -52,4 +52,12 @@ public abstract class ServerCommand { public int getAction() { return this.action; } + + public String toString() { + final StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()); + sb.append("/"); + sb.append(action); + return sb.toString(); + } }
