HADOOP-14072. AliyunOSS: Failed to read from stream when seek beyond the download size. Contributed by Genmao Yu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cd3e59a3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cd3e59a3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cd3e59a3 Branch: refs/heads/HADOOP-13345 Commit: cd3e59a3dcc69f68711777d448da5228a55846b3 Parents: 8acb376 Author: Kai Zheng <kai.zh...@intel.com> Authored: Wed Feb 15 16:34:30 2017 +0800 Committer: Kai Zheng <kai.zh...@intel.com> Committed: Wed Feb 15 16:34:30 2017 +0800 ---------------------------------------------------------------------- .../fs/aliyun/oss/AliyunOSSInputStream.java | 4 ++- .../oss/contract/TestAliyunOSSContractSeek.java | 26 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd3e59a3/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java index a3af7ce..72ba619 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java @@ -224,8 +224,10 @@ public class AliyunOSSInputStream extends FSInputStream { if (position == pos) { return; } else if (pos > position && pos < position + partRemaining) { - AliyunOSSUtils.skipFully(wrappedStream, pos - position); + long len = pos - position; + AliyunOSSUtils.skipFully(wrappedStream, len); position = pos; + partRemaining -= len; } else { reopen(pos); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd3e59a3/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java index b247ab1..d9b3674 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java @@ -19,8 +19,15 @@ package org.apache.hadoop.fs.aliyun.oss.contract; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.AbstractContractSeekTest; import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.Test; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; +import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; /** * Aliyun OSS contract seeking tests. @@ -31,4 +38,23 @@ public class TestAliyunOSSContractSeek extends AbstractContractSeekTest { protected AbstractFSContract createContract(Configuration conf) { return new AliyunOSSContract(conf); } + + @Test + public void testSeekBeyondDownloadSize() throws Throwable { + describe("seek and read beyond download size."); + + Path byteFile = path("byte_file.txt"); + // 'fs.oss.multipart.download.size' = 100 * 1024 + byte[] block = dataset(100 * 1024 + 10, 0, 255); + FileSystem fs = getFileSystem(); + createFile(fs, byteFile, true, block); + + FSDataInputStream instream = getFileSystem().open(byteFile); + instream.seek(100 * 1024 - 1); + assertEquals(100 * 1024 - 1, instream.getPos()); + assertEquals(144, instream.read()); + instream.seek(100 * 1024 + 1); + assertEquals(100 * 1024 + 1, instream.getPos()); + assertEquals(146, instream.read()); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org