Author: cmccabe Date: Fri Mar 7 01:19:24 2014 New Revision: 1575111 URL: http://svn.apache.org/r1575111 Log: HDFS-6065. HDFS zero-copy reads should return null on EOF when doing ZCR (cmccabe)
Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/ (props changed) hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/ (props changed) hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed) hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitReplica.java hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed) hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/expect.h hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project:r1575109 Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1575109 Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1575111&r1=1575110&r2=1575111&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Mar 7 01:19:24 2014 @@ -309,6 +309,9 @@ Release 2.4.0 - UNRELEASED HDFS-6067. TestPread.testMaxOutHedgedReadPool is flaky (cmccabe) + HDFS-6065. HDFS zero-copy reads should return null on EOF when doing ZCR + (cmccabe) + BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9) Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1575109 Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java?rev=1575111&r1=1575110&r2=1575111&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java Fri Mar 7 01:19:24 2014 @@ -1556,13 +1556,27 @@ implements ByteBufferReadable, CanSetDro closeCurrentBlockReader(); } + /** + * The immutable empty buffer we return when we reach EOF when doing a + * zero-copy read. + */ + private static final ByteBuffer EMPTY_BUFFER = + ByteBuffer.allocateDirect(0).asReadOnlyBuffer(); + @Override public synchronized ByteBuffer read(ByteBufferPool bufferPool, int maxLength, EnumSet<ReadOption> opts) throws IOException, UnsupportedOperationException { - assert(maxLength > 0); - if (((blockReader == null) || (blockEnd == -1)) && - (pos < getFileLength())) { + if (maxLength == 0) { + return EMPTY_BUFFER; + } else if (maxLength < 0) { + throw new IllegalArgumentException("can't read a negative " + + "number of bytes."); + } + if ((blockReader == null) || (blockEnd == -1)) { + if (pos >= getFileLength()) { + return null; + } /* * If we don't have a blockReader, or the one we have has no more bytes * left to read, we call seekToBlockSource to get a new blockReader and @@ -1645,6 +1659,7 @@ implements ByteBufferReadable, CanSetDro @Override public synchronized void releaseBuffer(ByteBuffer buffer) { + if (buffer == EMPTY_BUFFER) return; Object val = extendedReadBuffers.remove(buffer); if (val == null) { throw new IllegalArgumentException("tried to release a buffer " + Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitReplica.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitReplica.java?rev=1575111&r1=1575110&r2=1575111&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitReplica.java (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ShortCircuitReplica.java Fri Mar 7 01:19:24 2014 @@ -277,7 +277,11 @@ public class ShortCircuitReplica { MappedByteBuffer loadMmapInternal() { try { FileChannel channel = dataStream.getChannel(); - return channel.map(MapMode.READ_ONLY, 0, channel.size()); + MappedByteBuffer mmap = channel.map(MapMode.READ_ONLY, 0, channel.size()); + if (LOG.isTraceEnabled()) { + LOG.trace(this + ": created mmap of size " + channel.size()); + } + return mmap; } catch (IOException e) { LOG.warn(this + ": mmap error", e); return null; Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1575109 Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/expect.h URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/expect.h?rev=1575111&r1=1575110&r2=1575111&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/expect.h (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/expect.h Fri Mar 7 01:19:24 2014 @@ -38,7 +38,7 @@ struct hdfsFile_internal; #define EXPECT_NULL(x) \ do { \ - void* __my_ret__ = x; \ + const void* __my_ret__ = x; \ int __my_errno__ = errno; \ if (__my_ret__ != NULL) { \ fprintf(stderr, "TEST_ERROR: failed on %s:%d (errno: %d): " \ @@ -50,7 +50,7 @@ struct hdfsFile_internal; #define EXPECT_NONNULL(x) \ do { \ - void* __my_ret__ = x; \ + const void* __my_ret__ = x; \ int __my_errno__ = errno; \ if (__my_ret__ == NULL) { \ fprintf(stderr, "TEST_ERROR: failed on %s:%d (errno: %d): " \ Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h?rev=1575111&r1=1575110&r2=1575111&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h Fri Mar 7 01:19:24 2014 @@ -746,12 +746,16 @@ extern "C" { * @param maxLength The maximum length to read. We may read fewer bytes * than this length. * - * @return On success, returns a new hadoopRzBuffer. + * @return On success, we will return a new hadoopRzBuffer. * This buffer will continue to be valid and readable * until it is released by readZeroBufferFree. Failure to * release a buffer will lead to a memory leak. + * You can access the data within the hadoopRzBuffer with + * hadoopRzBufferGet. If you have reached EOF, the data + * within the hadoopRzBuffer will be NULL. You must still + * free hadoopRzBuffer instances containing NULL. * - * NULL plus an errno code on an error. + * On failure, we will return NULL plus an errno code. * errno = EOPNOTSUPP indicates that we could not do a * zero-copy read, and there was no ByteBufferPool * supplied. Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c?rev=1575111&r1=1575110&r2=1575111&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c Fri Mar 7 01:19:24 2014 @@ -38,6 +38,9 @@ #define TEST_ZEROCOPY_LAST_BLOCK_SIZE 3215 #define TEST_ZEROCOPY_NUM_BLOCKS 6 #define SMALL_READ_LEN 16 +#define TEST_ZEROCOPY_FILE_LEN \ + (((TEST_ZEROCOPY_NUM_BLOCKS - 1) * TEST_ZEROCOPY_FULL_BLOCK_SIZE) + \ + TEST_ZEROCOPY_LAST_BLOCK_SIZE) #define ZC_BUF_LEN 32768 @@ -165,6 +168,22 @@ static int doTestZeroCopyReads(hdfsFS fs EXPECT_ZERO(memcmp(block, hadoopRzBufferGet(buffer) + (TEST_ZEROCOPY_FULL_BLOCK_SIZE - SMALL_READ_LEN), SMALL_READ_LEN)); hadoopRzBufferFree(file, buffer); + + /* Check the result of a zero-length read. */ + buffer = hadoopReadZero(file, opts, 0); + EXPECT_NONNULL(buffer); + EXPECT_NONNULL(hadoopRzBufferGet(buffer)); + EXPECT_INT_EQ(0, hadoopRzBufferLength(buffer)); + hadoopRzBufferFree(file, buffer); + + /* Check the result of reading past EOF */ + EXPECT_INT_EQ(0, hdfsSeek(fs, file, TEST_ZEROCOPY_FILE_LEN)); + buffer = hadoopReadZero(file, opts, 1); + EXPECT_NONNULL(buffer); + EXPECT_NULL(hadoopRzBufferGet(buffer)); + hadoopRzBufferFree(file, buffer); + + /* Cleanup */ free(block); hadoopRzOptionsFree(opts); EXPECT_ZERO(hdfsCloseFile(fs, file)); Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java?rev=1575111&r1=1575110&r2=1575111&view=diff ============================================================================== --- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java (original) +++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestEnhancedByteBufferAccess.java Fri Mar 7 01:19:24 2014 @@ -753,6 +753,10 @@ public class TestEnhancedByteBufferAcces fsIn = fs.open(TEST_PATH); ByteBuffer buf = fsIn.read(null, 1, EnumSet.of(ReadOption.SKIP_CHECKSUMS)); fsIn.releaseBuffer(buf); + // Test EOF behavior + IOUtils.skipFully(fsIn, TEST_FILE_LENGTH - 1); + buf = fsIn.read(null, 1, EnumSet.of(ReadOption.SKIP_CHECKSUMS)); + Assert.assertEquals(null, buf); } finally { if (fsIn != null) fsIn.close(); if (fs != null) fs.close();