Author: cmccabe Date: Mon Mar 10 02:48:04 2014 New Revision: 1575798 URL: http://svn.apache.org/r1575798 Log: HDFS-6071. BlockReaderLocal does not return -1 on EOF when doing a zero-length read on a short file. (cmccabe)
Added: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java - copied unchanged from r1575797, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/ (props changed) hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/ (props changed) hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed) hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project:r1575797 Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1575797 Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1575798&r1=1575797&r2=1575798&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Mar 10 02:48:04 2014 @@ -320,6 +320,9 @@ Release 2.4.0 - UNRELEASED HDFS-6078. TestIncrementalBlockReports is flaky. (Arpit Agarwal) + HDFS-6071. BlockReaderLocal doesn't return -1 on EOF when doing a + zero-length read on a short file (cmccabe) + BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9) Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1575797 Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java?rev=1575798&r1=1575797&r2=1575798&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java Mon Mar 10 02:48:04 2014 @@ -478,7 +478,7 @@ class BlockReaderLocal implements BlockR total += bb; if (buf.remaining() == 0) return total; } - boolean eof = false; + boolean eof = true, done = false; do { if (buf.isDirect() && (buf.remaining() >= maxReadaheadLength) && ((dataPos % bytesPerChecksum) == 0)) { @@ -493,20 +493,24 @@ class BlockReaderLocal implements BlockR buf.limit(oldLimit); } if (nRead < maxReadaheadLength) { - eof = true; + done = true; + } + if (nRead > 0) { + eof = false; } total += nRead; } else { // Slow lane: refill bounce buffer. if (fillDataBuf(canSkipChecksum)) { - eof = true; + done = true; } bb = drainDataBuf(buf); // drain bounce buffer if possible if (bb >= 0) { + eof = false; total += bb; } } - } while ((!eof) && (buf.remaining() > 0)); + } while ((!done) && (buf.remaining() > 0)); return (eof && total == 0) ? -1 : total; } Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1575798&r1=1575797&r2=1575798&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java Mon Mar 10 02:48:04 2014 @@ -57,11 +57,14 @@ import org.apache.hadoop.hdfs.server.pro import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.net.unix.DomainSocket; +import org.apache.hadoop.net.unix.TemporarySocketDirectory; import org.apache.hadoop.security.ShellBasedUnixGroupsMapping; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.VersionInfo; +import org.junit.Assume; import java.io.*; import java.net.*; @@ -1157,4 +1160,43 @@ public class DFSTestUtil { long c = (val + factor - 1) / factor; return c * factor; } + + /** + * A short-circuit test context which makes it easier to get a short-circuit + * configuration and set everything up. + */ + public static class ShortCircuitTestContext implements Closeable { + private final String testName; + private final TemporarySocketDirectory sockDir; + private boolean closed = false; + private boolean formerTcpReadsDisabled; + + public ShortCircuitTestContext(String testName) { + this.testName = testName; + this.sockDir = new TemporarySocketDirectory(); + DomainSocket.disableBindPathValidation(); + formerTcpReadsDisabled = DFSInputStream.tcpReadsDisabledForTesting; + Assume.assumeTrue(DomainSocket.getLoadingFailureReason() == null); + } + + public Configuration newConfiguration() { + Configuration conf = new Configuration(); + conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true); + conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, + new File(sockDir.getDir(), + testName + "._PORT.sock").getAbsolutePath()); + return conf; + } + + public String getTestName() { + return testName; + } + + public void close() throws IOException { + if (closed) return; + closed = true; + DFSInputStream.tcpReadsDisabledForTesting = formerTcpReadsDisabled; + sockDir.close(); + } + } }