HDFS-7681. Change ReplicaInputStreams constructor to take InputStream(s) instead of FileDescriptor(s). Contributed by Joe Pallas
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ced72c8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ced72c8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ced72c8 Branch: refs/heads/HDFS-EC Commit: 8ced72c81a01a126a37078e7f55fd631b6bef7ab Parents: dd5946a Author: Tsz-Wo Nicholas Sze <szets...@hortonworks.com> Authored: Wed Jan 28 15:59:33 2015 -0800 Committer: Zhe Zhang <z...@apache.org> Committed: Thu Jan 29 10:05:26 2015 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../server/datanode/fsdataset/ReplicaInputStreams.java | 6 +++--- .../server/datanode/fsdataset/impl/FsDatasetImpl.java | 10 ++++++++-- .../server/datanode/extdataset/ExternalDatasetImpl.java | 2 +- 4 files changed, 15 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ced72c8/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 4d2b41d..fd29408 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -141,6 +141,9 @@ Trunk (Unreleased) HDFS-7430. Rewrite the BlockScanner to use O(1) memory and use multiple threads (cmccabe) + HDFS-7681. Change ReplicaInputStreams constructor to take InputStream(s) + instead of FileDescriptor(s). (Joe Pallas via szetszwo) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ced72c8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/ReplicaInputStreams.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/ReplicaInputStreams.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/ReplicaInputStreams.java index a8bf622..e0928a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/ReplicaInputStreams.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/ReplicaInputStreams.java @@ -33,11 +33,11 @@ public class ReplicaInputStreams implements Closeable { private final FsVolumeReference volumeRef; /** Create an object with a data input stream and a checksum input stream. */ - public ReplicaInputStreams(FileDescriptor dataFd, FileDescriptor checksumFd, + public ReplicaInputStreams(InputStream dataStream, InputStream checksumStream, FsVolumeReference volumeRef) { this.volumeRef = volumeRef; - this.dataIn = new FileInputStream(dataFd); - this.checksumIn = new FileInputStream(checksumFd); + this.dataIn = dataStream; + this.checksumIn = checksumStream; } /** @return the data input stream. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ced72c8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index c00d467..5c20102 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -642,8 +642,14 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> { if (ckoff > 0) { metaInFile.seek(ckoff); } - return new ReplicaInputStreams( - blockInFile.getFD(), metaInFile.getFD(), ref); + InputStream blockInStream = new FileInputStream(blockInFile.getFD()); + try { + InputStream metaInStream = new FileInputStream(metaInFile.getFD()); + return new ReplicaInputStreams(blockInStream, metaInStream, ref); + } catch (IOException e) { + IOUtils.cleanup(null, blockInStream); + throw e; + } } catch (IOException e) { IOUtils.cleanup(null, ref); throw e; http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ced72c8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java index 8fd51d2..aa868d5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java @@ -137,7 +137,7 @@ public class ExternalDatasetImpl implements FsDatasetSpi<ExternalVolumeImpl> { @Override public ReplicaInputStreams getTmpInputStreams(ExtendedBlock b, long blkoff, long ckoff) throws IOException { - return new ReplicaInputStreams(FileDescriptor.in, FileDescriptor.in, null); + return new ReplicaInputStreams(null, null, null); } @Override