Merge commit 'b67ae6d9d741e79ccf2bd6f08a37fce070e6ad77' into HDFS-12943 # Conflicts: # hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java # hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java # hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto # hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/066afcf2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/066afcf2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/066afcf2 Branch: refs/heads/HDFS-12943 Commit: 066afcf291d84d4215b7b447937d9e55f052239e Parents: 90e61bf 3ac07b7 Author: Konstantin V Shvachko <s...@apache.org> Authored: Sat Aug 25 17:13:21 2018 -0700 Committer: Konstantin V Shvachko <s...@apache.org> Committed: Sat Aug 25 17:13:21 2018 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hdfs/DFSClient.java | 19 +- .../hadoop/hdfs/DistributedFileSystem.java | 34 + .../apache/hadoop/hdfs/client/HdfsAdmin.java | 10 + .../hadoop/hdfs/protocol/ClientProtocol.java | 14 + .../hadoop/hdfs/protocol/HdfsConstants.java | 32 + .../ClientNamenodeProtocolTranslatorPB.java | 13 + .../src/main/proto/ClientNamenodeProtocol.proto | 11 +- .../router/RouterNamenodeProtocol.java | 7 + .../federation/router/RouterRpcServer.java | 12 + .../hadoop-hdfs/src/main/bin/hdfs | 7 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 44 + .../java/org/apache/hadoop/hdfs/DFSUtil.java | 2 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 20 +- .../DatanodeProtocolClientSideTranslatorPB.java | 4 +- .../NamenodeProtocolServerSideTranslatorPB.java | 18 + .../NamenodeProtocolTranslatorPB.java | 15 + .../hdfs/server/balancer/NameNodeConnector.java | 36 +- .../blockmanagement/BlockInfoStriped.java | 10 + .../server/blockmanagement/BlockManager.java | 72 + .../blockmanagement/DatanodeDescriptor.java | 1 - .../server/blockmanagement/DatanodeManager.java | 24 +- .../hdfs/server/common/HdfsServerConstants.java | 5 + .../hdfs/server/common/sps/BlockDispatcher.java | 186 ++ .../sps/BlockMovementAttemptFinished.java | 100 ++ .../server/common/sps/BlockMovementStatus.java | 53 + .../common/sps/BlockStorageMovementTracker.java | 91 + .../sps/BlocksMovementsStatusHandler.java | 39 + .../hdfs/server/common/sps/package-info.java | 27 + .../hdfs/server/datanode/BPServiceActor.java | 2 + .../hadoop/hdfs/server/datanode/DataNode.java | 2 +- .../apache/hadoop/hdfs/server/mover/Mover.java | 7 +- .../namenode/FSDirSatisfyStoragePolicyOp.java | 157 ++ .../hdfs/server/namenode/FSDirXAttrOp.java | 7 + .../hdfs/server/namenode/FSDirectory.java | 19 + .../hdfs/server/namenode/FSNamesystem.java | 97 +- .../hdfs/server/namenode/FSTreeTraverser.java | 4 +- .../hadoop/hdfs/server/namenode/NameNode.java | 48 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 47 +- .../hadoop/hdfs/server/namenode/Namesystem.java | 10 + .../server/namenode/ReencryptionHandler.java | 3 +- .../namenode/sps/BlockMoveTaskHandler.java | 43 + .../namenode/sps/BlockMovementListener.java | 40 + .../sps/BlockStorageMovementAttemptedItems.java | 313 ++++ .../sps/BlockStorageMovementNeeded.java | 346 ++++ .../hdfs/server/namenode/sps/Context.java | 141 ++ .../namenode/sps/DatanodeCacheManager.java | 121 ++ .../hdfs/server/namenode/sps/FileCollector.java | 43 + .../hdfs/server/namenode/sps/ItemInfo.java | 82 + .../hdfs/server/namenode/sps/SPSService.java | 127 ++ .../namenode/sps/StoragePolicySatisfier.java | 1226 +++++++++++++ .../sps/StoragePolicySatisfyManager.java | 293 ++++ .../hdfs/server/namenode/sps/package-info.java | 28 + .../protocol/BlockStorageMovementCommand.java | 136 ++ .../BlocksStorageMoveAttemptFinished.java | 48 + .../hdfs/server/protocol/DatanodeProtocol.java | 2 + .../server/protocol/DropSPSWorkCommand.java | 36 + .../hdfs/server/protocol/NamenodeProtocol.java | 10 +- .../sps/ExternalSPSBlockMoveTaskHandler.java | 223 +++ .../hdfs/server/sps/ExternalSPSContext.java | 211 +++ .../sps/ExternalSPSFilePathCollector.java | 174 ++ .../sps/ExternalStoragePolicySatisfier.java | 117 ++ .../hadoop/hdfs/server/sps/package-info.java | 28 + .../hadoop/hdfs/tools/StoragePolicyAdmin.java | 61 +- .../src/main/proto/DatanodeProtocol.proto | 1 + .../src/main/proto/NamenodeProtocol.proto | 21 + .../src/main/resources/hdfs-default.xml | 125 ++ .../src/site/markdown/ArchivalStorage.md | 71 +- .../src/site/markdown/HDFSCommands.md | 2 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 112 ++ .../SimpleBlocksMovementsStatusHandler.java | 88 + .../hadoop/hdfs/server/mover/TestMover.java | 73 +- .../hdfs/server/mover/TestStorageMover.java | 2 + .../hdfs/server/namenode/TestDeadDatanode.java | 2 +- .../namenode/TestNameNodeReconfigure.java | 130 ++ .../TestPersistentStoragePolicySatisfier.java | 531 ++++++ .../TestStoragePolicySatisfierWithHA.java | 112 ++ .../TestBlockStorageMovementAttemptedItems.java | 220 +++ ...stStoragePolicySatisfierWithStripedFile.java | 581 ++++++ .../sps/TestExternalStoragePolicySatisfier.java | 1650 ++++++++++++++++++ .../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 2 +- .../hdfs/tools/TestStoragePolicyCommands.java | 10 +- .../TestStoragePolicySatisfyAdminCommands.java | 112 ++ 82 files changed, 8953 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 84a875a,e8c881b..9e456a5 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@@ -1789,13 -1744,16 +1789,27 @@@ public interface ClientProtocol EnumSet<OpenFilesType> openFilesTypes, String path) throws IOException; /** + * Called by client to wait until the server has reached the state id of the + * client. The client and server state id are given by client side and server + * side alignment context respectively. This can be a blocking call. + * + * @throws IOException + */ + @Idempotent + @ReadOnly + void msync() throws IOException; ++ ++ /** + * Satisfy the storage policy for a file/directory. + * @param path Path of an existing file/directory. + * @throws AccessControlException If access is denied. + * @throws org.apache.hadoop.fs.UnresolvedLinkException if <code>src</code> + * contains a symlink. + * @throws java.io.FileNotFoundException If file/dir <code>src</code> is not + * found. + * @throws org.apache.hadoop.hdfs.server.namenode.SafeModeException append not + * allowed in safemode. + */ + @AtMostOnce + void satisfyStoragePolicy(String path) throws IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 088a9aa,e4bca51..8ca199f --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@@ -1946,12 -1948,13 +1949,22 @@@ public class ClientNamenodeProtocolTran } @Override + public void msync() throws IOException { + MsyncRequestProto.Builder req = MsyncRequestProto.newBuilder(); + try { + rpcProxy.msync(null, req.build()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } ++ + public void satisfyStoragePolicy(String src) throws IOException { + SatisfyStoragePolicyRequestProto req = + SatisfyStoragePolicyRequestProto.newBuilder().setSrc(src).build(); + try { + rpcProxy.satisfyStoragePolicy(null, req); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto index 247e54b,49ea3f3..55113a1 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto @@@ -831,12 -830,14 +830,20 @@@ message ListOpenFilesResponseProto repeated OpenFilesTypeProto types = 3; } +message MsyncRequestProto { +} + +message MsyncResponseProto { +} + + message SatisfyStoragePolicyRequestProto { + required string src = 1; + } + + message SatisfyStoragePolicyResponseProto { + + } + service ClientNamenodeProtocol { rpc getBlockLocations(GetBlockLocationsRequestProto) returns(GetBlockLocationsResponseProto); @@@ -1023,6 -1024,6 +1030,8 @@@ returns(GetQuotaUsageResponseProto); rpc listOpenFiles(ListOpenFilesRequestProto) returns(ListOpenFilesResponseProto); + rpc msync(MsyncRequestProto) + returns(MsyncResponseProto); + rpc satisfyStoragePolicy(SatisfyStoragePolicyRequestProto) + returns(SatisfyStoragePolicyResponseProto); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 81dc0fa,e51529e..8a4d4a1 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@@ -1890,13 -1894,14 +1896,25 @@@ public class ClientNamenodeProtocolServ } @Override + public MsyncResponseProto msync(RpcController controller, + MsyncRequestProto req) throws ServiceException { + try { + server.msync(); + return MsyncResponseProto.newBuilder().build(); + } catch (IOException e) { + throw new ServiceException(e); + } + } ++ ++ @Override + public SatisfyStoragePolicyResponseProto satisfyStoragePolicy( + RpcController controller, + SatisfyStoragePolicyRequestProto request) throws ServiceException { + try { + server.satisfyStoragePolicy(request.getSrc()); + } catch (IOException e) { + throw new ServiceException(e); + } + return VOID_SATISFYSTORAGEPOLICY_RESPONSE; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/066afcf2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org