[19/50] [abbrv] hadoop git commit: HDFS-7210. Avoid two separate RPC's namenode.append() and namenode.getFileInfo() for an append call from DFSClient. (Vinayakumar B via umamahesh)
HDFS-7210. Avoid two separate RPC's namenode.append() and namenode.getFileInfo() for an append call from DFSClient. (Vinayakumar B via umamahesh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1556f86a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1556f86a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1556f86a Branch: refs/heads/HDFS-EC Commit: 1556f86a31a54733d6550363aa0e027acca7823b Parents: c1f2bb2 Author: Uma Maheswara Rao G Authored: Fri Nov 28 21:09:16 2014 +0530 Committer: Uma Maheswara Rao G Committed: Fri Nov 28 21:09:16 2014 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../java/org/apache/hadoop/hdfs/DFSClient.java | 10 +++-- .../hadoop/hdfs/protocol/ClientProtocol.java| 5 ++- .../hdfs/protocol/LastBlockWithStatus.java | 46 ...tNamenodeProtocolServerSideTranslatorPB.java | 20 + .../ClientNamenodeProtocolTranslatorPB.java | 11 +++-- .../hdfs/server/namenode/FSEditLogLoader.java | 7 ++- .../hdfs/server/namenode/FSNamesystem.java | 10 +++-- .../hdfs/server/namenode/NameNodeRpcServer.java | 8 ++-- .../src/main/proto/ClientNamenodeProtocol.proto | 1 + .../server/namenode/TestNamenodeRetryCache.java | 4 +- .../namenode/ha/TestRetryCacheWithHA.java | 3 +- 12 files changed, 100 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1556f86a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 936aa35..dfdca8d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -404,6 +404,9 @@ Release 2.7.0 - UNRELEASED HDFS-7310. Mover can give first priority to local DN if it has target storage type available in local DN. (Vinayakumar B via umamahesh) +HDFS-7210. Avoid two separate RPC's namenode.append() and namenode.getFileInfo() +for an append call from DFSClient. (Vinayakumar B via umamahesh) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/1556f86a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 056a1b3..62db1fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -163,6 +163,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; @@ -1770,9 +1771,9 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, /** Method to get stream returned by append call */ private DFSOutputStream callAppend(String src, int buffersize, Progressable progress) throws IOException { -LocatedBlock lastBlock = null; +LastBlockWithStatus lastBlockWithStatus = null; try { - lastBlock = namenode.append(src, clientName); + lastBlockWithStatus = namenode.append(src, clientName); } catch(RemoteException re) { throw re.unwrapRemoteException(AccessControlException.class, FileNotFoundException.class, @@ -1782,9 +1783,10 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, UnresolvedPathException.class, SnapshotAccessControlException.class); } -HdfsFileStatus newStat = getFileInfo(src); +HdfsFileStatus newStat = lastBlockWithStatus.getFileStatus(); return DFSOutputStream.newStreamForAppend(this, src, buffersize, progress, -lastBlock, newStat, dfsClientConf.createChecksum()); +lastBlockWithStatus.getLastBlock(), newStat, +dfsClientConf.createChecksum()); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/1556f86a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java ---
hadoop git commit: HDFS-7210. Avoid two separate RPC's namenode.append() and namenode.getFileInfo() for an append call from DFSClient. (Vinayakumar B via umamahesh) (cherry picked from commit 1556f86a
Repository: hadoop Updated Branches: refs/heads/branch-2 8f9822541 -> f4ab30634 HDFS-7210. Avoid two separate RPC's namenode.append() and namenode.getFileInfo() for an append call from DFSClient. (Vinayakumar B via umamahesh) (cherry picked from commit 1556f86a31a54733d6550363aa0e027acca7823b) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4ab3063 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4ab3063 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4ab3063 Branch: refs/heads/branch-2 Commit: f4ab30634715b975c059561df27185662a8e910a Parents: 8f98225 Author: Uma Maheswara Rao G Authored: Fri Nov 28 21:09:16 2014 +0530 Committer: Uma Maheswara Rao G Committed: Fri Nov 28 21:13:17 2014 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../java/org/apache/hadoop/hdfs/DFSClient.java | 10 +++-- .../hadoop/hdfs/protocol/ClientProtocol.java| 5 ++- .../hdfs/protocol/LastBlockWithStatus.java | 46 ...tNamenodeProtocolServerSideTranslatorPB.java | 20 + .../ClientNamenodeProtocolTranslatorPB.java | 11 +++-- .../hdfs/server/namenode/FSEditLogLoader.java | 7 ++- .../hdfs/server/namenode/FSNamesystem.java | 10 +++-- .../hdfs/server/namenode/NameNodeRpcServer.java | 8 ++-- .../src/main/proto/ClientNamenodeProtocol.proto | 1 + .../server/namenode/TestNamenodeRetryCache.java | 4 +- .../namenode/ha/TestRetryCacheWithHA.java | 3 +- 12 files changed, 100 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4ab3063/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0e353fd..d909497 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -147,6 +147,9 @@ Release 2.7.0 - UNRELEASED HDFS-7310. Mover can give first priority to local DN if it has target storage type available in local DN. (Vinayakumar B via umamahesh) +HDFS-7210. Avoid two separate RPC's namenode.append() and namenode.getFileInfo() +for an append call from DFSClient. (Vinayakumar B via umamahesh) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4ab3063/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 6bf90c8..bd9622d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -164,6 +164,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; @@ -1777,9 +1778,9 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, /** Method to get stream returned by append call */ private DFSOutputStream callAppend(String src, int buffersize, Progressable progress) throws IOException { -LocatedBlock lastBlock = null; +LastBlockWithStatus lastBlockWithStatus = null; try { - lastBlock = namenode.append(src, clientName); + lastBlockWithStatus = namenode.append(src, clientName); } catch(RemoteException re) { throw re.unwrapRemoteException(AccessControlException.class, FileNotFoundException.class, @@ -1789,9 +1790,10 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, UnresolvedPathException.class, SnapshotAccessControlException.class); } -HdfsFileStatus newStat = getFileInfo(src); +HdfsFileStatus newStat = lastBlockWithStatus.getFileStatus(); return DFSOutputStream.newStreamForAppend(this, src, buffersize, progress, -lastBlock, newStat, dfsClientConf.createChecksum()); +lastBlockWithStatus.getLastBlock(), newStat, +dfsClientConf.createChecksum()); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4ab3063
hadoop git commit: HDFS-7210. Avoid two separate RPC's namenode.append() and namenode.getFileInfo() for an append call from DFSClient. (Vinayakumar B via umamahesh)
Repository: hadoop Updated Branches: refs/heads/trunk c1f2bb2d3 -> 1556f86a3 HDFS-7210. Avoid two separate RPC's namenode.append() and namenode.getFileInfo() for an append call from DFSClient. (Vinayakumar B via umamahesh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1556f86a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1556f86a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1556f86a Branch: refs/heads/trunk Commit: 1556f86a31a54733d6550363aa0e027acca7823b Parents: c1f2bb2 Author: Uma Maheswara Rao G Authored: Fri Nov 28 21:09:16 2014 +0530 Committer: Uma Maheswara Rao G Committed: Fri Nov 28 21:09:16 2014 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../java/org/apache/hadoop/hdfs/DFSClient.java | 10 +++-- .../hadoop/hdfs/protocol/ClientProtocol.java| 5 ++- .../hdfs/protocol/LastBlockWithStatus.java | 46 ...tNamenodeProtocolServerSideTranslatorPB.java | 20 + .../ClientNamenodeProtocolTranslatorPB.java | 11 +++-- .../hdfs/server/namenode/FSEditLogLoader.java | 7 ++- .../hdfs/server/namenode/FSNamesystem.java | 10 +++-- .../hdfs/server/namenode/NameNodeRpcServer.java | 8 ++-- .../src/main/proto/ClientNamenodeProtocol.proto | 1 + .../server/namenode/TestNamenodeRetryCache.java | 4 +- .../namenode/ha/TestRetryCacheWithHA.java | 3 +- 12 files changed, 100 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1556f86a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 936aa35..dfdca8d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -404,6 +404,9 @@ Release 2.7.0 - UNRELEASED HDFS-7310. Mover can give first priority to local DN if it has target storage type available in local DN. (Vinayakumar B via umamahesh) +HDFS-7210. Avoid two separate RPC's namenode.append() and namenode.getFileInfo() +for an append call from DFSClient. (Vinayakumar B via umamahesh) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/1556f86a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 056a1b3..62db1fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -163,6 +163,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; @@ -1770,9 +1771,9 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, /** Method to get stream returned by append call */ private DFSOutputStream callAppend(String src, int buffersize, Progressable progress) throws IOException { -LocatedBlock lastBlock = null; +LastBlockWithStatus lastBlockWithStatus = null; try { - lastBlock = namenode.append(src, clientName); + lastBlockWithStatus = namenode.append(src, clientName); } catch(RemoteException re) { throw re.unwrapRemoteException(AccessControlException.class, FileNotFoundException.class, @@ -1782,9 +1783,10 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, UnresolvedPathException.class, SnapshotAccessControlException.class); } -HdfsFileStatus newStat = getFileInfo(src); +HdfsFileStatus newStat = lastBlockWithStatus.getFileStatus(); return DFSOutputStream.newStreamForAppend(this, src, buffersize, progress, -lastBlock, newStat, dfsClientConf.createChecksum()); +lastBlockWithStatus.getLastBlock(), newStat, +dfsClientConf.createChecksum()); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/1556f86a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/proto