HDFS-10178. Permanent write failures can happen if pipeline recoveries occur for the first packet. Contributed by Kihwal Lee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a7d1fb0c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a7d1fb0c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a7d1fb0c Branch: refs/heads/HDFS-7240 Commit: a7d1fb0cd2fdbf830602eb4dbbd9bbe62f4d5584 Parents: 154d253 Author: Kihwal Lee <kih...@apache.org> Authored: Mon Apr 4 16:39:23 2016 -0500 Committer: Kihwal Lee <kih...@apache.org> Committed: Mon Apr 4 16:40:00 2016 -0500 ---------------------------------------------------------------------- .../hdfs/server/datanode/BlockReceiver.java | 2 + .../hdfs/server/datanode/BlockSender.java | 6 ++- .../server/datanode/DataNodeFaultInjector.java | 2 + .../TestClientProtocolForPipelineRecovery.java | 53 ++++++++++++++++++++ 4 files changed, 62 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7d1fb0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index 2e4ee02..fb0c1c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -574,6 +574,8 @@ class BlockReceiver implements Closeable { if (mirrorOut != null && !mirrorError) { try { long begin = Time.monotonicNow(); + // For testing. Normally no-op. + DataNodeFaultInjector.get().stopSendingPacketDownstream(); packetReceiver.mirrorPacketTo(mirrorOut); mirrorOut.flush(); long now = Time.monotonicNow(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7d1fb0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java index 773a64c..398935d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java @@ -300,11 +300,15 @@ class BlockSender implements java.io.Closeable { // The meta file will contain only the header if the NULL checksum // type was used, or if the replica was written to transient storage. + // Also, when only header portion of a data packet was transferred + // and then pipeline breaks, the meta file can contain only the + // header and 0 byte in the block data file. // Checksum verification is not performed for replicas on transient // storage. The header is important for determining the checksum // type later when lazy persistence copies the block to non-transient // storage and computes the checksum. - if (metaIn.getLength() > BlockMetadataHeader.getHeaderSize()) { + if (!replica.isOnTransientStorage() && + metaIn.getLength() >= BlockMetadataHeader.getHeaderSize()) { checksumIn = new DataInputStream(new BufferedInputStream( metaIn, IO_FILE_BUFFER_SIZE)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7d1fb0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java index 0e38694..7327420 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeFaultInjector.java @@ -50,5 +50,7 @@ public class DataNodeFaultInjector { return false; } + public void stopSendingPacketDownstream() throws IOException {} + public void noRegistration() throws IOException { } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7d1fb0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java index d7aa79a..0eeb3b7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java @@ -375,4 +375,57 @@ public class TestClientProtocolForPipelineRecovery { } } } + + /** + * Test to make sure the checksum is set correctly after pipeline + * recovery transfers 0 byte partial block. If fails the test case + * will say "java.io.IOException: Failed to replace a bad datanode + * on the existing pipeline due to no more good datanodes being + * available to try." This indicates there was a real failure + * after the staged failure. + */ + @Test + public void testZeroByteBlockRecovery() throws Exception { + // Make the first datanode fail once. With 3 nodes and a block being + // created with 2 replicas, anything more than this planned failure + // will cause a test failure. + DataNodeFaultInjector dnFaultInjector = new DataNodeFaultInjector() { + int tries = 1; + @Override + public void stopSendingPacketDownstream() throws IOException { + if (tries > 0) { + tries--; + try { + Thread.sleep(60000); + } catch (InterruptedException ie) { + throw new IOException("Interrupted while sleeping. Bailing out."); + } + } + } + }; + DataNodeFaultInjector oldDnInjector = DataNodeFaultInjector.get(); + DataNodeFaultInjector.set(dnFaultInjector); + + Configuration conf = new HdfsConfiguration(); + conf.set(HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, "1000"); + conf.set(HdfsClientConfigKeys. + BlockWrite.ReplaceDatanodeOnFailure.POLICY_KEY, "ALWAYS"); + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); + cluster.waitActive(); + + FileSystem fs = cluster.getFileSystem(); + FSDataOutputStream out = fs.create(new Path("noheartbeat.dat"), (short)2); + out.write(0x31); + out.hflush(); + out.close(); + + } finally { + if (cluster != null) { + cluster.shutdown(); + } + DataNodeFaultInjector.set(oldDnInjector); + } + } }
