HDFS-13539. DFSStripedInputStream NPE when reportCheckSumFailure.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/960940e0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/960940e0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/960940e0 Branch: refs/heads/HDDS-4 Commit: 960940e0e08f7839775f2d8a352b444d104d36b4 Parents: fc5d49c Author: Xiao Chen <x...@apache.org> Authored: Mon May 14 09:28:09 2018 -0700 Committer: Xiao Chen <x...@apache.org> Committed: Mon May 14 09:28:39 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/DFSInputStream.java | 13 +++++++++- .../hadoop/hdfs/DFSStripedInputStream.java | 8 ++++--- .../hadoop/hdfs/TestDFSStripedInputStream.java | 25 ++++++++++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/960940e0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index d3d6669..b38e629 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -790,13 +790,24 @@ public class DFSInputStream extends FSInputStream // Check if need to report block replicas corruption either read // was successful or ChecksumException occurred. reportCheckSumFailure(corruptedBlocks, - currentLocatedBlock.getLocations().length, false); + getCurrentBlockLocationsLength(), false); } } } return -1; } + protected int getCurrentBlockLocationsLength() { + int len = 0; + if (currentLocatedBlock == null) { + DFSClient.LOG.info("Found null currentLocatedBlock. pos={}, " + + "blockEnd={}, fileLength={}", pos, blockEnd, getFileLength()); + } else { + len = currentLocatedBlock.getLocations().length; + } + return len; + } + /** * Read the entire buffer. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/960940e0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java index 339a02c..f3b16e0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.ReadOption; import org.apache.hadoop.hdfs.protocol.BlockType; @@ -160,7 +161,8 @@ public class DFSStripedInputStream extends DFSInputStream { * When seeking into a new block group, create blockReader for each internal * block in the group. */ - private synchronized void blockSeekTo(long target) throws IOException { + @VisibleForTesting + synchronized void blockSeekTo(long target) throws IOException { if (target >= getFileLength()) { throw new IOException("Attempted to read past end of file"); } @@ -400,8 +402,8 @@ public class DFSStripedInputStream extends DFSInputStream { } finally { // Check if need to report block replicas corruption either read // was successful or ChecksumException occurred. - reportCheckSumFailure(corruptedBlocks, - currentLocatedBlock.getLocations().length, true); + reportCheckSumFailure(corruptedBlocks, getCurrentBlockLocationsLength(), + true); } } return -1; http://git-wip-us.apache.org/repos/asf/hadoop/blob/960940e0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java index de276a9..cdebee0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java @@ -35,6 +35,7 @@ import org.apache.hadoop.io.erasurecode.ErasureCodeNative; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -51,7 +52,12 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; public class TestDFSStripedInputStream { @@ -504,4 +510,23 @@ public class TestDFSStripedInputStream { in.close(); } } + + @Test + public void testReadFailToGetCurrentBlock() throws Exception { + DFSTestUtil.writeFile(cluster.getFileSystem(), filePath, "test"); + try (DFSStripedInputStream in = (DFSStripedInputStream) fs.getClient() + .open(filePath.toString())) { + final DFSStripedInputStream spy = spy(in); + final String msg = "Injected exception for testReadNPE"; + doThrow(new IOException(msg)).when(spy).blockSeekTo(anyLong()); + assertNull(in.getCurrentBlock()); + try { + spy.read(); + fail("read should have failed"); + } catch (IOException expected) { + LOG.info("Exception caught", expected); + GenericTestUtils.assertExceptionContains(msg, expected); + } + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org