Author: cmccabe
Date: Mon Mar 10 02:48:11 2014
New Revision: 1575799
URL: http://svn.apache.org/r1575799
Log:
HDFS-6071. BlockReaderLocal does not return -1 on EOF when doing a zero-length
read on a short file. (cmccabe)
Added:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java
- copied unchanged from r1575797,
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/ (props
changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project:r1575797
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1575797
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1575799&r1=1575798&r2=1575799&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Mon Mar 10 02:48:11 2014
@@ -334,6 +334,9 @@ Release 2.4.0 - UNRELEASED
HDFS-6078. TestIncrementalBlockReports is flaky. (Arpit Agarwal)
+ HDFS-6071. BlockReaderLocal doesn't return -1 on EOF when doing a
+ zero-length read on a short file (cmccabe)
+
BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS
HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9)
Propchange:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1575797
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java?rev=1575799&r1=1575798&r2=1575799&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
Mon Mar 10 02:48:11 2014
@@ -478,7 +478,7 @@ class BlockReaderLocal implements BlockR
total += bb;
if (buf.remaining() == 0) return total;
}
- boolean eof = false;
+ boolean eof = true, done = false;
do {
if (buf.isDirect() && (buf.remaining() >= maxReadaheadLength)
&& ((dataPos % bytesPerChecksum) == 0)) {
@@ -493,20 +493,24 @@ class BlockReaderLocal implements BlockR
buf.limit(oldLimit);
}
if (nRead < maxReadaheadLength) {
- eof = true;
+ done = true;
+ }
+ if (nRead > 0) {
+ eof = false;
}
total += nRead;
} else {
// Slow lane: refill bounce buffer.
if (fillDataBuf(canSkipChecksum)) {
- eof = true;
+ done = true;
}
bb = drainDataBuf(buf); // drain bounce buffer if possible
if (bb >= 0) {
+ eof = false;
total += bb;
}
}
- } while ((!eof) && (buf.remaining() > 0));
+ } while ((!done) && (buf.remaining() > 0));
return (eof && total == 0) ? -1 : total;
}
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1575799&r1=1575798&r2=1575799&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
Mon Mar 10 02:48:11 2014
@@ -57,11 +57,14 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.unix.DomainSocket;
+import org.apache.hadoop.net.unix.TemporarySocketDirectory;
import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.VersionInfo;
+import org.junit.Assume;
import java.io.*;
import java.net.*;
@@ -1157,4 +1160,43 @@ public class DFSTestUtil {
long c = (val + factor - 1) / factor;
return c * factor;
}
+
+ /**
+ * A short-circuit test context which makes it easier to get a short-circuit
+ * configuration and set everything up.
+ */
+ public static class ShortCircuitTestContext implements Closeable {
+ private final String testName;
+ private final TemporarySocketDirectory sockDir;
+ private boolean closed = false;
+ private boolean formerTcpReadsDisabled;
+
+ public ShortCircuitTestContext(String testName) {
+ this.testName = testName;
+ this.sockDir = new TemporarySocketDirectory();
+ DomainSocket.disableBindPathValidation();
+ formerTcpReadsDisabled = DFSInputStream.tcpReadsDisabledForTesting;
+ Assume.assumeTrue(DomainSocket.getLoadingFailureReason() == null);
+ }
+
+ public Configuration newConfiguration() {
+ Configuration conf = new Configuration();
+ conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true);
+ conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY,
+ new File(sockDir.getDir(),
+ testName + "._PORT.sock").getAbsolutePath());
+ return conf;
+ }
+
+ public String getTestName() {
+ return testName;
+ }
+
+ public void close() throws IOException {
+ if (closed) return;
+ closed = true;
+ DFSInputStream.tcpReadsDisabledForTesting = formerTcpReadsDisabled;
+ sockDir.close();
+ }
+ }
}
