HDFS-13215. RBF: Move Router to its own module. Contributed by Wei Yan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87700d45 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87700d45 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87700d45 Branch: refs/heads/HDFS-12996 Commit: 87700d453d52118f4139c81e3527496469a6d8c4 Parents: feefeb4 Author: weiy <w...@apache.org> Authored: Mon Mar 19 22:13:40 2018 -0700 Committer: Hanisha Koneru <hanishakon...@apache.org> Committed: Wed Mar 21 16:46:52 2018 -0700 ---------------------------------------------------------------------- dev-support/bin/dist-layout-stitching | 1 + .../dev-support/findbugsExcludeFile.xml | 22 + hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 236 ++ .../hdfs/protocolPB/RouterAdminProtocolPB.java | 44 + ...uterAdminProtocolServerSideTranslatorPB.java | 211 ++ .../RouterAdminProtocolTranslatorPB.java | 208 ++ .../hadoop/hdfs/protocolPB/package-info.java | 18 + .../federation/metrics/FederationMBean.java | 223 ++ .../federation/metrics/FederationMetrics.java | 766 ++++++ .../federation/metrics/FederationRPCMBean.java | 92 + .../metrics/FederationRPCMetrics.java | 250 ++ .../FederationRPCPerformanceMonitor.java | 221 ++ .../federation/metrics/NamenodeBeanMetrics.java | 634 +++++ .../federation/metrics/StateStoreMBean.java | 45 + .../federation/metrics/StateStoreMetrics.java | 149 ++ .../server/federation/metrics/package-info.java | 27 + .../resolver/ActiveNamenodeResolver.java | 118 + .../resolver/FederationNamenodeContext.java | 87 + .../FederationNamenodeServiceState.java | 46 + .../resolver/FederationNamespaceInfo.java | 73 + .../resolver/FileSubclusterResolver.java | 75 + .../resolver/MembershipNamenodeResolver.java | 317 +++ .../federation/resolver/MountTableManager.java | 80 + .../federation/resolver/MountTableResolver.java | 597 +++++ .../MultipleDestinationMountTableResolver.java | 116 + .../resolver/NamenodePriorityComparator.java | 63 + .../resolver/NamenodeStatusReport.java | 406 ++++ .../federation/resolver/PathLocation.java | 212 ++ .../federation/resolver/RemoteLocation.java | 76 + .../resolver/order/DestinationOrder.java | 30 + .../resolver/order/HashFirstResolver.java | 71 + .../federation/resolver/order/HashResolver.java | 137 ++ .../resolver/order/LocalResolver.java | 297 +++ .../resolver/order/OrderedResolver.java | 37 + .../resolver/order/RandomResolver.java | 62 + .../federation/resolver/order/package-info.java | 29 + .../federation/resolver/package-info.java | 41 + .../federation/router/ConnectionContext.java | 104 + .../federation/router/ConnectionManager.java | 435 ++++ .../federation/router/ConnectionPool.java | 337 +++ .../federation/router/ConnectionPoolId.java | 123 + .../server/federation/router/DFSRouter.java | 76 + .../server/federation/router/ErasureCoding.java | 198 ++ .../federation/router/FederationUtil.java | 208 ++ .../router/NamenodeHeartbeatService.java | 362 +++ .../federation/router/PeriodicService.java | 198 ++ .../hdfs/server/federation/router/Quota.java | 208 ++ .../server/federation/router/RBFConfigKeys.java | 229 ++ .../router/RemoteLocationContext.java | 68 + .../server/federation/router/RemoteMethod.java | 164 ++ .../server/federation/router/RemoteParam.java | 71 + .../hdfs/server/federation/router/Router.java | 655 ++++++ .../federation/router/RouterAdminServer.java | 298 +++ .../server/federation/router/RouterClient.java | 80 + .../router/RouterHeartbeatService.java | 166 ++ .../federation/router/RouterHttpServer.java | 124 + .../server/federation/router/RouterMetrics.java | 73 + .../federation/router/RouterMetricsService.java | 108 + .../router/RouterPermissionChecker.java | 82 + .../federation/router/RouterQuotaManager.java | 172 ++ .../router/RouterQuotaUpdateService.java | 227 ++ .../federation/router/RouterQuotaUsage.java | 119 + .../federation/router/RouterRpcClient.java | 1021 ++++++++ .../federation/router/RouterRpcMonitor.java | 108 + .../federation/router/RouterRpcServer.java | 2214 ++++++++++++++++++ .../router/RouterSafeModeException.java | 53 + .../router/RouterSafemodeService.java | 149 ++ .../federation/router/RouterServiceState.java | 31 + .../federation/router/RouterStateManager.java | 50 + .../server/federation/router/package-info.java | 31 + .../federation/store/CachedRecordStore.java | 242 ++ .../federation/store/MembershipStore.java | 126 + .../federation/store/MountTableStore.java | 49 + .../server/federation/store/RecordStore.java | 100 + .../server/federation/store/RouterStore.java | 78 + .../federation/store/StateStoreCache.java | 36 + .../store/StateStoreCacheUpdateService.java | 70 + .../StateStoreConnectionMonitorService.java | 67 + .../federation/store/StateStoreService.java | 450 ++++ .../store/StateStoreUnavailableException.java | 33 + .../federation/store/StateStoreUtils.java | 109 + .../store/driver/StateStoreDriver.java | 203 ++ .../driver/StateStoreRecordOperations.java | 148 ++ .../store/driver/StateStoreSerializer.java | 119 + .../store/driver/impl/StateStoreBaseImpl.java | 86 + .../driver/impl/StateStoreFileBaseImpl.java | 462 ++++ .../store/driver/impl/StateStoreFileImpl.java | 148 ++ .../driver/impl/StateStoreFileSystemImpl.java | 178 ++ .../driver/impl/StateStoreSerializableImpl.java | 98 + .../driver/impl/StateStoreSerializerPBImpl.java | 115 + .../driver/impl/StateStoreZooKeeperImpl.java | 328 +++ .../store/driver/impl/package-info.java | 39 + .../federation/store/driver/package-info.java | 37 + .../store/impl/MembershipStoreImpl.java | 310 +++ .../store/impl/MountTableStoreImpl.java | 154 ++ .../federation/store/impl/RouterStoreImpl.java | 93 + .../federation/store/impl/package-info.java | 31 + .../server/federation/store/package-info.java | 62 + .../protocol/AddMountTableEntryRequest.java | 47 + .../protocol/AddMountTableEntryResponse.java | 42 + .../store/protocol/EnterSafeModeRequest.java | 32 + .../store/protocol/EnterSafeModeResponse.java | 50 + .../protocol/GetMountTableEntriesRequest.java | 49 + .../protocol/GetMountTableEntriesResponse.java | 53 + .../GetNamenodeRegistrationsRequest.java | 52 + .../GetNamenodeRegistrationsResponse.java | 55 + .../store/protocol/GetNamespaceInfoRequest.java | 30 + .../protocol/GetNamespaceInfoResponse.java | 52 + .../protocol/GetRouterRegistrationRequest.java | 47 + .../protocol/GetRouterRegistrationResponse.java | 44 + .../protocol/GetRouterRegistrationsRequest.java | 34 + .../GetRouterRegistrationsResponse.java | 55 + .../store/protocol/GetSafeModeRequest.java | 31 + .../store/protocol/GetSafeModeResponse.java | 49 + .../store/protocol/LeaveSafeModeRequest.java | 32 + .../store/protocol/LeaveSafeModeResponse.java | 50 + .../protocol/NamenodeHeartbeatRequest.java | 52 + .../protocol/NamenodeHeartbeatResponse.java | 49 + .../protocol/RemoveMountTableEntryRequest.java | 49 + .../protocol/RemoveMountTableEntryResponse.java | 42 + .../store/protocol/RouterHeartbeatRequest.java | 50 + .../store/protocol/RouterHeartbeatResponse.java | 49 + .../protocol/UpdateMountTableEntryRequest.java | 51 + .../protocol/UpdateMountTableEntryResponse.java | 43 + .../UpdateNamenodeRegistrationRequest.java | 72 + .../UpdateNamenodeRegistrationResponse.java | 51 + .../pb/AddMountTableEntryRequestPBImpl.java | 84 + .../pb/AddMountTableEntryResponsePBImpl.java | 76 + .../impl/pb/EnterSafeModeRequestPBImpl.java | 62 + .../impl/pb/EnterSafeModeResponsePBImpl.java | 73 + .../impl/pb/FederationProtocolPBTranslator.java | 145 ++ .../pb/GetMountTableEntriesRequestPBImpl.java | 76 + .../pb/GetMountTableEntriesResponsePBImpl.java | 104 + .../GetNamenodeRegistrationsRequestPBImpl.java | 87 + .../GetNamenodeRegistrationsResponsePBImpl.java | 99 + .../impl/pb/GetNamespaceInfoRequestPBImpl.java | 60 + .../impl/pb/GetNamespaceInfoResponsePBImpl.java | 95 + .../pb/GetRouterRegistrationRequestPBImpl.java | 70 + .../pb/GetRouterRegistrationResponsePBImpl.java | 79 + .../pb/GetRouterRegistrationsRequestPBImpl.java | 61 + .../GetRouterRegistrationsResponsePBImpl.java | 102 + .../impl/pb/GetSafeModeRequestPBImpl.java | 62 + .../impl/pb/GetSafeModeResponsePBImpl.java | 73 + .../impl/pb/LeaveSafeModeRequestPBImpl.java | 62 + .../impl/pb/LeaveSafeModeResponsePBImpl.java | 73 + .../impl/pb/NamenodeHeartbeatRequestPBImpl.java | 93 + .../pb/NamenodeHeartbeatResponsePBImpl.java | 71 + .../pb/RemoveMountTableEntryRequestPBImpl.java | 76 + .../pb/RemoveMountTableEntryResponsePBImpl.java | 76 + .../impl/pb/RouterHeartbeatRequestPBImpl.java | 78 + .../impl/pb/RouterHeartbeatResponsePBImpl.java | 70 + .../pb/UpdateMountTableEntryRequestPBImpl.java | 96 + .../pb/UpdateMountTableEntryResponsePBImpl.java | 76 + ...UpdateNamenodeRegistrationRequestPBImpl.java | 95 + ...pdateNamenodeRegistrationResponsePBImpl.java | 73 + .../store/protocol/impl/pb/package-info.java | 29 + .../federation/store/protocol/package-info.java | 31 + .../federation/store/records/BaseRecord.java | 215 ++ .../store/records/MembershipState.java | 336 +++ .../store/records/MembershipStats.java | 130 + .../federation/store/records/MountTable.java | 430 ++++ .../server/federation/store/records/Query.java | 66 + .../federation/store/records/QueryResult.java | 56 + .../federation/store/records/RouterState.java | 168 ++ .../store/records/StateStoreVersion.java | 91 + .../records/impl/pb/MembershipStatePBImpl.java | 335 +++ .../records/impl/pb/MembershipStatsPBImpl.java | 201 ++ .../store/records/impl/pb/MountTablePBImpl.java | 317 +++ .../store/records/impl/pb/PBRecord.java | 47 + .../records/impl/pb/RouterStatePBImpl.java | 202 ++ .../impl/pb/StateStoreVersionPBImpl.java | 79 + .../store/records/impl/pb/package-info.java | 29 + .../federation/store/records/package-info.java | 36 + .../federation/utils/ConsistentHashRing.java | 144 ++ .../server/federation/utils/package-info.java | 28 + .../hdfs/tools/federation/RouterAdmin.java | 627 +++++ .../hdfs/tools/federation/package-info.java | 28 + .../src/main/proto/FederationProtocol.proto | 244 ++ .../src/main/proto/RouterProtocol.proto | 62 + .../src/main/resources/hdfs-rbf-default.xml | 434 ++++ .../src/main/webapps/proto-web.xml | 16 + .../main/webapps/router/federationhealth.html | 424 ++++ .../src/main/webapps/router/federationhealth.js | 381 +++ .../src/main/webapps/router/index.html | 24 + .../src/main/webapps/router/robots.txt | 2 + .../src/site/markdown/HDFSRouterFederation.md | 373 +++ .../src/site/resources/css/site.css | 30 + .../site/resources/images/routerfederation.png | Bin 0 -> 24961 bytes .../hadoop-hdfs-rbf/src/site/site.xml | 28 + .../server/federation/FederationTestUtils.java | 246 ++ .../hdfs/server/federation/MockResolver.java | 313 +++ .../server/federation/RouterConfigBuilder.java | 163 ++ .../server/federation/RouterDFSCluster.java | 1005 ++++++++ .../server/federation/StateStoreDFSCluster.java | 148 ++ .../metrics/TestFederationMetrics.java | 293 +++ .../federation/metrics/TestMetricsBase.java | 196 ++ .../resolver/TestMountTableResolver.java | 528 +++++ .../TestMultipleDestinationResolver.java | 419 ++++ .../resolver/TestNamenodeResolver.java | 284 +++ .../resolver/order/TestLocalResolver.java | 143 ++ .../router/TestConnectionManager.java | 157 ++ .../router/TestNamenodeHeartbeat.java | 168 ++ .../federation/router/TestRBFConfigFields.java | 51 + .../server/federation/router/TestRouter.java | 150 ++ .../federation/router/TestRouterAdmin.java | 340 +++ .../federation/router/TestRouterAdminCLI.java | 477 ++++ .../router/TestRouterHeartbeatService.java | 146 ++ .../federation/router/TestRouterMountTable.java | 143 ++ .../router/TestRouterNamenodeMonitoring.java | 143 ++ .../federation/router/TestRouterQuota.java | 451 ++++ .../router/TestRouterQuotaManager.java | 125 + .../router/TestRouterRPCClientRetries.java | 160 ++ .../server/federation/router/TestRouterRpc.java | 1072 +++++++++ .../router/TestRouterRpcMultiDestination.java | 216 ++ .../federation/router/TestRouterSafemode.java | 200 ++ .../store/FederationStateStoreTestUtils.java | 274 +++ .../federation/store/TestStateStoreBase.java | 81 + .../store/TestStateStoreMembershipState.java | 463 ++++ .../store/TestStateStoreMountTable.java | 250 ++ .../store/TestStateStoreRouterState.java | 195 ++ .../store/driver/TestStateStoreDriverBase.java | 613 +++++ .../store/driver/TestStateStoreFile.java | 76 + .../store/driver/TestStateStoreFileBase.java | 47 + .../store/driver/TestStateStoreFileSystem.java | 94 + .../store/driver/TestStateStoreZK.java | 105 + .../store/records/TestMembershipState.java | 129 + .../store/records/TestMountTable.java | 259 ++ .../store/records/TestRouterState.java | 85 + .../src/test/resources/hdfs-site.xml | 29 + .../src/test/resources/log4j.properties | 23 + .../dev-support/findbugsExcludeFile.xml | 3 - hadoop-hdfs-project/hadoop-hdfs/pom.xml | 6 - .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 204 -- .../hdfs/protocolPB/RouterAdminProtocolPB.java | 44 - ...uterAdminProtocolServerSideTranslatorPB.java | 211 -- .../RouterAdminProtocolTranslatorPB.java | 208 -- .../federation/metrics/FederationMBean.java | 223 -- .../federation/metrics/FederationMetrics.java | 766 ------ .../federation/metrics/FederationRPCMBean.java | 92 - .../metrics/FederationRPCMetrics.java | 250 -- .../FederationRPCPerformanceMonitor.java | 221 -- .../federation/metrics/NamenodeBeanMetrics.java | 634 ----- .../federation/metrics/StateStoreMBean.java | 45 - .../federation/metrics/StateStoreMetrics.java | 149 -- .../server/federation/metrics/package-info.java | 27 - .../resolver/ActiveNamenodeResolver.java | 118 - .../resolver/FederationNamenodeContext.java | 87 - .../FederationNamenodeServiceState.java | 46 - .../resolver/FederationNamespaceInfo.java | 73 - .../resolver/FileSubclusterResolver.java | 75 - .../resolver/MembershipNamenodeResolver.java | 317 --- .../federation/resolver/MountTableManager.java | 80 - .../federation/resolver/MountTableResolver.java | 597 ----- .../MultipleDestinationMountTableResolver.java | 116 - .../resolver/NamenodePriorityComparator.java | 63 - .../resolver/NamenodeStatusReport.java | 406 ---- .../federation/resolver/PathLocation.java | 212 -- .../federation/resolver/RemoteLocation.java | 76 - .../resolver/order/DestinationOrder.java | 30 - .../resolver/order/HashFirstResolver.java | 71 - .../federation/resolver/order/HashResolver.java | 137 -- .../resolver/order/LocalResolver.java | 297 --- .../resolver/order/OrderedResolver.java | 37 - .../resolver/order/RandomResolver.java | 62 - .../federation/resolver/order/package-info.java | 29 - .../federation/resolver/package-info.java | 41 - .../federation/router/ConnectionContext.java | 104 - .../federation/router/ConnectionManager.java | 436 ---- .../federation/router/ConnectionPool.java | 337 --- .../federation/router/ConnectionPoolId.java | 123 - .../server/federation/router/DFSRouter.java | 76 - .../server/federation/router/ErasureCoding.java | 198 -- .../federation/router/FederationUtil.java | 209 -- .../router/NamenodeHeartbeatService.java | 362 --- .../federation/router/PeriodicService.java | 198 -- .../hdfs/server/federation/router/Quota.java | 208 -- .../router/RemoteLocationContext.java | 68 - .../server/federation/router/RemoteMethod.java | 164 -- .../server/federation/router/RemoteParam.java | 71 - .../hdfs/server/federation/router/Router.java | 656 ------ .../federation/router/RouterAdminServer.java | 298 --- .../server/federation/router/RouterClient.java | 80 - .../router/RouterHeartbeatService.java | 167 -- .../federation/router/RouterHttpServer.java | 124 - .../server/federation/router/RouterMetrics.java | 73 - .../federation/router/RouterMetricsService.java | 108 - .../router/RouterPermissionChecker.java | 82 - .../federation/router/RouterQuotaManager.java | 172 -- .../router/RouterQuotaUpdateService.java | 228 -- .../federation/router/RouterQuotaUsage.java | 119 - .../federation/router/RouterRpcClient.java | 1022 -------- .../federation/router/RouterRpcMonitor.java | 108 - .../federation/router/RouterRpcServer.java | 2214 ------------------ .../router/RouterSafeModeException.java | 53 - .../router/RouterSafemodeService.java | 150 -- .../federation/router/RouterServiceState.java | 31 - .../federation/router/RouterStateManager.java | 50 - .../server/federation/router/package-info.java | 31 - .../federation/store/CachedRecordStore.java | 242 -- .../federation/store/MembershipStore.java | 126 - .../federation/store/MountTableStore.java | 49 - .../server/federation/store/RecordStore.java | 100 - .../server/federation/store/RouterStore.java | 78 - .../federation/store/StateStoreCache.java | 36 - .../store/StateStoreCacheUpdateService.java | 70 - .../StateStoreConnectionMonitorService.java | 67 - .../federation/store/StateStoreService.java | 450 ---- .../store/StateStoreUnavailableException.java | 33 - .../federation/store/StateStoreUtils.java | 109 - .../store/driver/StateStoreDriver.java | 203 -- .../driver/StateStoreRecordOperations.java | 148 -- .../store/driver/StateStoreSerializer.java | 119 - .../store/driver/impl/StateStoreBaseImpl.java | 86 - .../driver/impl/StateStoreFileBaseImpl.java | 462 ---- .../store/driver/impl/StateStoreFileImpl.java | 148 -- .../driver/impl/StateStoreFileSystemImpl.java | 178 -- .../driver/impl/StateStoreSerializableImpl.java | 98 - .../driver/impl/StateStoreSerializerPBImpl.java | 115 - .../driver/impl/StateStoreZooKeeperImpl.java | 328 --- .../store/driver/impl/package-info.java | 39 - .../federation/store/driver/package-info.java | 37 - .../store/impl/MembershipStoreImpl.java | 310 --- .../store/impl/MountTableStoreImpl.java | 154 -- .../federation/store/impl/RouterStoreImpl.java | 93 - .../federation/store/impl/package-info.java | 31 - .../server/federation/store/package-info.java | 62 - .../protocol/AddMountTableEntryRequest.java | 47 - .../protocol/AddMountTableEntryResponse.java | 42 - .../store/protocol/EnterSafeModeRequest.java | 32 - .../store/protocol/EnterSafeModeResponse.java | 50 - .../protocol/GetMountTableEntriesRequest.java | 49 - .../protocol/GetMountTableEntriesResponse.java | 53 - .../GetNamenodeRegistrationsRequest.java | 52 - .../GetNamenodeRegistrationsResponse.java | 55 - .../store/protocol/GetNamespaceInfoRequest.java | 30 - .../protocol/GetNamespaceInfoResponse.java | 52 - .../protocol/GetRouterRegistrationRequest.java | 47 - .../protocol/GetRouterRegistrationResponse.java | 44 - .../protocol/GetRouterRegistrationsRequest.java | 34 - .../GetRouterRegistrationsResponse.java | 55 - .../store/protocol/GetSafeModeRequest.java | 31 - .../store/protocol/GetSafeModeResponse.java | 49 - .../store/protocol/LeaveSafeModeRequest.java | 32 - .../store/protocol/LeaveSafeModeResponse.java | 50 - .../protocol/NamenodeHeartbeatRequest.java | 52 - .../protocol/NamenodeHeartbeatResponse.java | 49 - .../protocol/RemoveMountTableEntryRequest.java | 49 - .../protocol/RemoveMountTableEntryResponse.java | 42 - .../store/protocol/RouterHeartbeatRequest.java | 50 - .../store/protocol/RouterHeartbeatResponse.java | 49 - .../protocol/UpdateMountTableEntryRequest.java | 51 - .../protocol/UpdateMountTableEntryResponse.java | 43 - .../UpdateNamenodeRegistrationRequest.java | 72 - .../UpdateNamenodeRegistrationResponse.java | 51 - .../pb/AddMountTableEntryRequestPBImpl.java | 84 - .../pb/AddMountTableEntryResponsePBImpl.java | 76 - .../impl/pb/EnterSafeModeRequestPBImpl.java | 62 - .../impl/pb/EnterSafeModeResponsePBImpl.java | 73 - .../impl/pb/FederationProtocolPBTranslator.java | 145 -- .../pb/GetMountTableEntriesRequestPBImpl.java | 76 - .../pb/GetMountTableEntriesResponsePBImpl.java | 104 - .../GetNamenodeRegistrationsRequestPBImpl.java | 87 - .../GetNamenodeRegistrationsResponsePBImpl.java | 99 - .../impl/pb/GetNamespaceInfoRequestPBImpl.java | 60 - .../impl/pb/GetNamespaceInfoResponsePBImpl.java | 95 - .../pb/GetRouterRegistrationRequestPBImpl.java | 70 - .../pb/GetRouterRegistrationResponsePBImpl.java | 79 - .../pb/GetRouterRegistrationsRequestPBImpl.java | 61 - .../GetRouterRegistrationsResponsePBImpl.java | 102 - .../impl/pb/GetSafeModeRequestPBImpl.java | 62 - .../impl/pb/GetSafeModeResponsePBImpl.java | 73 - .../impl/pb/LeaveSafeModeRequestPBImpl.java | 62 - .../impl/pb/LeaveSafeModeResponsePBImpl.java | 73 - .../impl/pb/NamenodeHeartbeatRequestPBImpl.java | 93 - .../pb/NamenodeHeartbeatResponsePBImpl.java | 71 - .../pb/RemoveMountTableEntryRequestPBImpl.java | 76 - .../pb/RemoveMountTableEntryResponsePBImpl.java | 76 - .../impl/pb/RouterHeartbeatRequestPBImpl.java | 78 - .../impl/pb/RouterHeartbeatResponsePBImpl.java | 70 - .../pb/UpdateMountTableEntryRequestPBImpl.java | 96 - .../pb/UpdateMountTableEntryResponsePBImpl.java | 76 - ...UpdateNamenodeRegistrationRequestPBImpl.java | 95 - ...pdateNamenodeRegistrationResponsePBImpl.java | 73 - .../store/protocol/impl/pb/package-info.java | 29 - .../federation/store/protocol/package-info.java | 31 - .../federation/store/records/BaseRecord.java | 215 -- .../store/records/MembershipState.java | 336 --- .../store/records/MembershipStats.java | 130 - .../federation/store/records/MountTable.java | 430 ---- .../server/federation/store/records/Query.java | 66 - .../federation/store/records/QueryResult.java | 56 - .../federation/store/records/RouterState.java | 168 -- .../store/records/StateStoreVersion.java | 91 - .../records/impl/pb/MembershipStatePBImpl.java | 335 --- .../records/impl/pb/MembershipStatsPBImpl.java | 201 -- .../store/records/impl/pb/MountTablePBImpl.java | 317 --- .../store/records/impl/pb/PBRecord.java | 47 - .../records/impl/pb/RouterStatePBImpl.java | 202 -- .../impl/pb/StateStoreVersionPBImpl.java | 79 - .../store/records/impl/pb/package-info.java | 29 - .../federation/store/records/package-info.java | 36 - .../federation/utils/ConsistentHashRing.java | 144 -- .../server/federation/utils/package-info.java | 28 - .../hdfs/tools/federation/RouterAdmin.java | 627 ----- .../hdfs/tools/federation/package-info.java | 28 - .../src/main/proto/FederationProtocol.proto | 244 -- .../src/main/proto/RouterProtocol.proto | 62 - .../src/main/resources/hdfs-default.xml | 408 ---- .../main/webapps/router/federationhealth.html | 424 ---- .../src/main/webapps/router/federationhealth.js | 381 --- .../src/main/webapps/router/index.html | 24 - .../src/main/webapps/router/robots.txt | 2 - .../src/site/markdown/HDFSRouterFederation.md | 373 --- .../site/resources/images/routerfederation.png | Bin 24961 -> 0 bytes .../server/federation/FederationTestUtils.java | 246 -- .../hdfs/server/federation/MockResolver.java | 313 --- .../server/federation/RouterConfigBuilder.java | 163 -- .../server/federation/RouterDFSCluster.java | 1005 -------- .../server/federation/StateStoreDFSCluster.java | 148 -- .../metrics/TestFederationMetrics.java | 293 --- .../federation/metrics/TestMetricsBase.java | 196 -- .../resolver/TestMountTableResolver.java | 528 ----- .../TestMultipleDestinationResolver.java | 419 ---- .../resolver/TestNamenodeResolver.java | 284 --- .../resolver/order/TestLocalResolver.java | 143 -- .../router/TestConnectionManager.java | 157 -- .../router/TestNamenodeHeartbeat.java | 168 -- .../server/federation/router/TestRouter.java | 150 -- .../federation/router/TestRouterAdmin.java | 340 --- .../federation/router/TestRouterAdminCLI.java | 478 ---- .../router/TestRouterHeartbeatService.java | 147 -- .../federation/router/TestRouterMountTable.java | 143 -- .../router/TestRouterNamenodeMonitoring.java | 143 -- .../federation/router/TestRouterQuota.java | 452 ---- .../router/TestRouterQuotaManager.java | 125 - .../router/TestRouterRPCClientRetries.java | 160 -- .../server/federation/router/TestRouterRpc.java | 1072 --------- .../router/TestRouterRpcMultiDestination.java | 216 -- .../federation/router/TestRouterSafemode.java | 201 -- .../store/FederationStateStoreTestUtils.java | 274 --- .../federation/store/TestStateStoreBase.java | 81 - .../store/TestStateStoreMembershipState.java | 463 ---- .../store/TestStateStoreMountTable.java | 250 -- .../store/TestStateStoreRouterState.java | 195 -- .../store/driver/TestStateStoreDriverBase.java | 613 ----- .../store/driver/TestStateStoreFile.java | 76 - .../store/driver/TestStateStoreFileBase.java | 47 - .../store/driver/TestStateStoreFileSystem.java | 94 - .../store/driver/TestStateStoreZK.java | 105 - .../store/records/TestMembershipState.java | 129 - .../store/records/TestMountTable.java | 259 -- .../store/records/TestRouterState.java | 85 - hadoop-hdfs-project/pom.xml | 1 + hadoop-project/pom.xml | 5 + hadoop-project/src/site/site.xml | 3 +- 455 files changed, 38400 insertions(+), 37908 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/87700d45/dev-support/bin/dist-layout-stitching ---------------------------------------------------------------------- diff --git a/dev-support/bin/dist-layout-stitching b/dev-support/bin/dist-layout-stitching index fdbc83c..f3db542 100755 --- a/dev-support/bin/dist-layout-stitching +++ b/dev-support/bin/dist-layout-stitching @@ -129,6 +129,7 @@ run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${VERSION}" run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-nfs/target/hadoop-hdfs-nfs-${VERSION}" . run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-${VERSION}" . run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-native-client/target/hadoop-hdfs-native-client-${VERSION}" . +run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-rbf/target/hadoop-hdfs-rbf-${VERSION}" . run copy "${ROOT}/hadoop-yarn-project/target/hadoop-yarn-project-${VERSION}" . run copy "${ROOT}/hadoop-mapreduce-project/target/hadoop-mapreduce-${VERSION}" . run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERSION}" . http://git-wip-us.apache.org/repos/asf/hadoop/blob/87700d45/hadoop-hdfs-project/hadoop-hdfs-rbf/dev-support/findbugsExcludeFile.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/dev-support/findbugsExcludeFile.xml new file mode 100644 index 0000000..40771bf --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/dev-support/findbugsExcludeFile.xml @@ -0,0 +1,22 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<FindBugsFilter> + <Match> + <Package name="org.apache.hadoop.hdfs.federation.protocol.proto" /> + </Match> +</FindBugsFilter> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/87700d45/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml new file mode 100644 index 0000000..9683744 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml @@ -0,0 +1,236 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; +xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; +xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 +http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-project-dist</artifactId> + <version>3.2.0-SNAPSHOT</version> + <relativePath>../../hadoop-project-dist</relativePath> + </parent> + <artifactId>hadoop-hdfs-rbf</artifactId> + <version>3.2.0-SNAPSHOT</version> + <description>Apache Hadoop HDFS-RBF</description> + <name>Apache Hadoop HDFS-RBF</name> + <packaging>jar</packaging> + + <properties> + <hadoop.component>hdfs</hadoop.component> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs-client</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-util-ajax</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <scope>test</scope> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-annotations</artifactId> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <scope>test</scope> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-test</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <configuration> + <skipTests>false</skipTests> + </configuration> + <executions> + <execution> + <id>create-web-xmls</id> + <phase>compile</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <copy file="${basedir}/src/main/webapps/proto-web.xml" + tofile="${project.build.directory}/webapps/router/WEB-INF/web.xml" + filtering="true"/> + <copy toDir="${project.build.directory}/webapps"> + <fileset dir="${basedir}/src/main/webapps"> + <exclude name="**/proto-web.xml"/> + </fileset> + </copy> + <replace dir="${project.build.directory}/webapps" value="${release-year}"> + <include name="**/*.html"/> + <replacetoken>{release-year-token}</replacetoken> + </replace> + </target> + </configuration> + </execution> + <execution> + <id>create-log-dir</id> + <phase>process-test-resources</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <copy todir="${project.build.directory}/test-classes/webapps"> + <fileset dir="${project.build.directory}/webapps"> + <exclude name="proto-*-web.xml"/> + <exclude name="**/proto-web.xml"/> + </fileset> + </copy> + </target> + </configuration> + </execution> + <execution> + <phase>pre-site</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <tasks> + <copy file="src/main/resources/hdfs-rbf-default.xml" todir="src/site/resources"/> + </tasks> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-maven-plugins</artifactId> + <executions> + <execution> + <id>compile-protoc</id> + <goals> + <goal>protoc</goal> + </goals> + <configuration> + <protocVersion>${protobuf.version}</protocVersion> + <protocCommand>${protoc.path}</protocCommand> + <imports> + <param>${basedir}/../hadoop-hdfs-client/src/main/proto</param> + <param>${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto</param> + <param>${basedir}/src/main/proto</param> + </imports> + <source> + <directory>${basedir}/src/main/proto</directory> + <includes> + <include>FederationProtocol.proto</include> + <include>RouterProtocol.proto</include> + </includes> + </source> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes> + <exclude>.gitattributes</exclude> + <exclude>.idea/**</exclude> + <exclude>dev-support/findbugsExcludeFile.xml</exclude> + <exclude>src/main/webapps/router/robots.txt</exclude> + <exclude>src/site/resources/images/*</exclude> + </excludes> + </configuration> + </plugin> + <plugin> + <artifactId>maven-clean-plugin</artifactId> + <configuration> + <filesets> + <fileset> + <directory>src/site/resources</directory> + <includes> + <include>hdfs-rbf-default.xml</include> + </includes> + <followSymlinks>false</followSymlinks> + </fileset> + </filesets> + </configuration> + </plugin> + </plugins> + </build> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/87700d45/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolPB.java new file mode 100644 index 0000000..96fa794 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolPB.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocolPB; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.proto.RouterProtocolProtos.RouterAdminProtocolService; +import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector; +import org.apache.hadoop.ipc.ProtocolInfo; +import org.apache.hadoop.security.KerberosInfo; +import org.apache.hadoop.security.token.TokenInfo; + +/** + * Protocol that a clients use to communicate with the NameNode. + * Note: This extends the protocolbuffer service based interface to + * add annotations required for security. + */ +@InterfaceAudience.Private +@InterfaceStability.Stable +@KerberosInfo( + serverPrincipal = DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY) +@TokenInfo(DelegationTokenSelector.class) +@ProtocolInfo(protocolName = HdfsConstants.CLIENT_NAMENODE_PROTOCOL_NAME, + protocolVersion = 1) +public interface RouterAdminProtocolPB extends + RouterAdminProtocolService.BlockingInterface { +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/87700d45/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java new file mode 100644 index 0000000..159d5c2 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java @@ -0,0 +1,211 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocolPB; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryResponseProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeResponseProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesResponseProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeResponseProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeResponseProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryResponseProto; +import org.apache.hadoop.hdfs.server.federation.router.RouterAdminServer; +import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryRequestPBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryResponsePBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.EnterSafeModeRequestPBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.EnterSafeModeResponsePBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetMountTableEntriesRequestPBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetMountTableEntriesResponsePBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetSafeModeRequestPBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetSafeModeResponsePBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.LeaveSafeModeRequestPBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.LeaveSafeModeResponsePBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryRequestPBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryResponsePBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryRequestPBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryResponsePBImpl; + +import com.google.protobuf.RpcController; +import com.google.protobuf.ServiceException; + +/** + * This class is used on the server side. Calls come across the wire for the for + * protocol {@link RouterAdminProtocolPB}. This class translates the PB data + * types to the native data types used inside the HDFS Router as specified in + * the generic RouterAdminProtocol. + */ +@InterfaceAudience.Private +@InterfaceStability.Stable +public class RouterAdminProtocolServerSideTranslatorPB implements + RouterAdminProtocolPB { + + private final RouterAdminServer server; + + /** + * Constructor. + * @param server The NN server. + * @throws IOException + */ + public RouterAdminProtocolServerSideTranslatorPB(RouterAdminServer server) + throws IOException { + this.server = server; + } + + @Override + public AddMountTableEntryResponseProto addMountTableEntry( + RpcController controller, AddMountTableEntryRequestProto request) + throws ServiceException { + + try { + AddMountTableEntryRequest req = + new AddMountTableEntryRequestPBImpl(request); + AddMountTableEntryResponse response = server.addMountTableEntry(req); + AddMountTableEntryResponsePBImpl responsePB = + (AddMountTableEntryResponsePBImpl)response; + return responsePB.getProto(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + /** + * Remove an entry from the mount table. + */ + @Override + public RemoveMountTableEntryResponseProto removeMountTableEntry( + RpcController controller, RemoveMountTableEntryRequestProto request) + throws ServiceException { + try { + RemoveMountTableEntryRequest req = + new RemoveMountTableEntryRequestPBImpl(request); + RemoveMountTableEntryResponse response = + server.removeMountTableEntry(req); + RemoveMountTableEntryResponsePBImpl responsePB = + (RemoveMountTableEntryResponsePBImpl)response; + return responsePB.getProto(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + /** + * Get matching mount table entries. + */ + @Override + public GetMountTableEntriesResponseProto getMountTableEntries( + RpcController controller, GetMountTableEntriesRequestProto request) + throws ServiceException { + try { + GetMountTableEntriesRequest req = + new GetMountTableEntriesRequestPBImpl(request); + GetMountTableEntriesResponse response = server.getMountTableEntries(req); + GetMountTableEntriesResponsePBImpl responsePB = + (GetMountTableEntriesResponsePBImpl)response; + return responsePB.getProto(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + /** + * Update a single mount table entry. + */ + @Override + public UpdateMountTableEntryResponseProto updateMountTableEntry( + RpcController controller, UpdateMountTableEntryRequestProto request) + throws ServiceException { + try { + UpdateMountTableEntryRequest req = + new UpdateMountTableEntryRequestPBImpl(request); + UpdateMountTableEntryResponse response = + server.updateMountTableEntry(req); + UpdateMountTableEntryResponsePBImpl responsePB = + (UpdateMountTableEntryResponsePBImpl)response; + return responsePB.getProto(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public EnterSafeModeResponseProto enterSafeMode(RpcController controller, + EnterSafeModeRequestProto request) throws ServiceException { + try { + EnterSafeModeRequest req = new EnterSafeModeRequestPBImpl(request); + EnterSafeModeResponse response = server.enterSafeMode(req); + EnterSafeModeResponsePBImpl responsePB = + (EnterSafeModeResponsePBImpl) response; + return responsePB.getProto(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public LeaveSafeModeResponseProto leaveSafeMode(RpcController controller, + LeaveSafeModeRequestProto request) throws ServiceException { + try { + LeaveSafeModeRequest req = new LeaveSafeModeRequestPBImpl(request); + LeaveSafeModeResponse response = server.leaveSafeMode(req); + LeaveSafeModeResponsePBImpl responsePB = + (LeaveSafeModeResponsePBImpl) response; + return responsePB.getProto(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public GetSafeModeResponseProto getSafeMode(RpcController controller, + GetSafeModeRequestProto request) throws ServiceException { + try { + GetSafeModeRequest req = new GetSafeModeRequestPBImpl(request); + GetSafeModeResponse response = server.getSafeMode(req); + GetSafeModeResponsePBImpl responsePB = + (GetSafeModeResponsePBImpl) response; + return responsePB.getProto(); + } catch (IOException e) { + throw new ServiceException(e); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/87700d45/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java new file mode 100644 index 0000000..d6210ce --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java @@ -0,0 +1,208 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocolPB; + +import java.io.Closeable; +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.AddMountTableEntryResponseProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.EnterSafeModeResponseProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetMountTableEntriesResponseProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.GetSafeModeResponseProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeResponseProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryResponseProto; +import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager; +import org.apache.hadoop.hdfs.server.federation.router.RouterStateManager; +import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.EnterSafeModeResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.GetSafeModeResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryRequestPBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.AddMountTableEntryResponsePBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.EnterSafeModeResponsePBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetMountTableEntriesRequestPBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetMountTableEntriesResponsePBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.GetSafeModeResponsePBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.LeaveSafeModeResponsePBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryRequestPBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryResponsePBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryRequestPBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryResponsePBImpl; +import org.apache.hadoop.ipc.ProtobufHelper; +import org.apache.hadoop.ipc.ProtocolMetaInterface; +import org.apache.hadoop.ipc.ProtocolTranslator; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ipc.RpcClientUtil; + +import com.google.protobuf.ServiceException; + +/** + * This class forwards NN's ClientProtocol calls as RPC calls to the NN server + * while translating from the parameter types used in ClientProtocol to the + * new PB types. + */ +@InterfaceAudience.Private +@InterfaceStability.Stable +public class RouterAdminProtocolTranslatorPB + implements ProtocolMetaInterface, MountTableManager, + Closeable, ProtocolTranslator, RouterStateManager { + final private RouterAdminProtocolPB rpcProxy; + + public RouterAdminProtocolTranslatorPB(RouterAdminProtocolPB proxy) { + rpcProxy = proxy; + } + + @Override + public void close() { + RPC.stopProxy(rpcProxy); + } + + @Override + public Object getUnderlyingProxyObject() { + return rpcProxy; + } + + @Override + public boolean isMethodSupported(String methodName) throws IOException { + return RpcClientUtil.isMethodSupported(rpcProxy, + RouterAdminProtocolPB.class, RPC.RpcKind.RPC_PROTOCOL_BUFFER, + RPC.getProtocolVersion(RouterAdminProtocolPB.class), methodName); + } + + @Override + public AddMountTableEntryResponse addMountTableEntry( + AddMountTableEntryRequest request) throws IOException { + AddMountTableEntryRequestPBImpl requestPB = + (AddMountTableEntryRequestPBImpl)request; + AddMountTableEntryRequestProto proto = requestPB.getProto(); + try { + AddMountTableEntryResponseProto response = + rpcProxy.addMountTableEntry(null, proto); + return new AddMountTableEntryResponsePBImpl(response); + } catch (ServiceException e) { + throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + } + } + + @Override + public UpdateMountTableEntryResponse updateMountTableEntry( + UpdateMountTableEntryRequest request) throws IOException { + UpdateMountTableEntryRequestPBImpl requestPB = + (UpdateMountTableEntryRequestPBImpl)request; + UpdateMountTableEntryRequestProto proto = requestPB.getProto(); + try { + UpdateMountTableEntryResponseProto response = + rpcProxy.updateMountTableEntry(null, proto); + return new UpdateMountTableEntryResponsePBImpl(response); + } catch (ServiceException e) { + throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + } + } + + @Override + public RemoveMountTableEntryResponse removeMountTableEntry( + RemoveMountTableEntryRequest request) throws IOException { + RemoveMountTableEntryRequestPBImpl requestPB = + (RemoveMountTableEntryRequestPBImpl)request; + RemoveMountTableEntryRequestProto proto = requestPB.getProto(); + try { + RemoveMountTableEntryResponseProto responseProto = + rpcProxy.removeMountTableEntry(null, proto); + return new RemoveMountTableEntryResponsePBImpl(responseProto); + } catch (ServiceException e) { + throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + } + } + + @Override + public GetMountTableEntriesResponse getMountTableEntries( + GetMountTableEntriesRequest request) throws IOException { + GetMountTableEntriesRequestPBImpl requestPB = + (GetMountTableEntriesRequestPBImpl)request; + GetMountTableEntriesRequestProto proto = requestPB.getProto(); + try { + GetMountTableEntriesResponseProto response = + rpcProxy.getMountTableEntries(null, proto); + return new GetMountTableEntriesResponsePBImpl(response); + } catch (ServiceException e) { + throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + } + } + + @Override + public EnterSafeModeResponse enterSafeMode(EnterSafeModeRequest request) + throws IOException { + EnterSafeModeRequestProto proto = + EnterSafeModeRequestProto.newBuilder().build(); + try { + EnterSafeModeResponseProto response = + rpcProxy.enterSafeMode(null, proto); + return new EnterSafeModeResponsePBImpl(response); + } catch (ServiceException e) { + throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + } + } + + @Override + public LeaveSafeModeResponse leaveSafeMode(LeaveSafeModeRequest request) + throws IOException { + LeaveSafeModeRequestProto proto = + LeaveSafeModeRequestProto.newBuilder().build(); + try { + LeaveSafeModeResponseProto response = + rpcProxy.leaveSafeMode(null, proto); + return new LeaveSafeModeResponsePBImpl(response); + } catch (ServiceException e) { + throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + } + } + + @Override + public GetSafeModeResponse getSafeMode(GetSafeModeRequest request) + throws IOException { + GetSafeModeRequestProto proto = + GetSafeModeRequestProto.newBuilder().build(); + try { + GetSafeModeResponseProto response = + rpcProxy.getSafeMode(null, proto); + return new GetSafeModeResponsePBImpl(response); + } catch (ServiceException e) { + throw new IOException(ProtobufHelper.getRemoteException(e).getMessage()); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/87700d45/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java new file mode 100644 index 0000000..6233024 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java @@ -0,0 +1,18 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocolPB; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/87700d45/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java new file mode 100644 index 0000000..79fb3e4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMBean.java @@ -0,0 +1,223 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.metrics; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * JMX interface for the federation statistics. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public interface FederationMBean { + + /** + * Get information about all the namenodes in the federation or null if + * failure. + * @return JSON with all the Namenodes. + */ + String getNamenodes(); + + /** + * Get the latest info for each registered nameservice. + * @return JSON with all the nameservices. + */ + String getNameservices(); + + /** + * Get the mount table for the federated filesystem or null if failure. + * @return JSON with the mount table. + */ + String getMountTable(); + + /** + * Get the latest state of all routers. + * @return JSON with all of the known routers or null if failure. + */ + String getRouters(); + + /** + * Get the total capacity of the federated cluster. + * @return Total capacity of the federated cluster. + */ + long getTotalCapacity(); + + /** + * Get the used capacity of the federated cluster. + * @return Used capacity of the federated cluster. + */ + long getUsedCapacity(); + + /** + * Get the remaining capacity of the federated cluster. + * @return Remaining capacity of the federated cluster. + */ + long getRemainingCapacity(); + + /** + * Get the total remote storage capacity mounted in the federated cluster. + * @return Remote capacity of the federated cluster. + */ + long getProvidedSpace(); + + /** + * Get the number of nameservices in the federation. + * @return Number of nameservices in the federation. + */ + int getNumNameservices(); + + /** + * Get the number of namenodes. + * @return Number of namenodes. + */ + int getNumNamenodes(); + + /** + * Get the number of expired namenodes. + * @return Number of expired namenodes. + */ + int getNumExpiredNamenodes(); + + /** + * Get the number of live datanodes. + * @return Number of live datanodes. + */ + int getNumLiveNodes(); + + /** + * Get the number of dead datanodes. + * @return Number of dead datanodes. + */ + int getNumDeadNodes(); + + /** + * Get the number of decommissioning datanodes. + * @return Number of decommissioning datanodes. + */ + int getNumDecommissioningNodes(); + + /** + * Get the number of live decommissioned datanodes. + * @return Number of live decommissioned datanodes. + */ + int getNumDecomLiveNodes(); + + /** + * Get the number of dead decommissioned datanodes. + * @return Number of dead decommissioned datanodes. + */ + int getNumDecomDeadNodes(); + + /** + * Get Max, Median, Min and Standard Deviation of DataNodes usage. + * @return the DataNode usage information, as a JSON string. + */ + String getNodeUsage(); + + /** + * Get the number of blocks in the federation. + * @return Number of blocks in the federation. + */ + long getNumBlocks(); + + /** + * Get the number of missing blocks in the federation. + * @return Number of missing blocks in the federation. + */ + long getNumOfMissingBlocks(); + + /** + * Get the number of pending replication blocks in the federation. + * @return Number of pending replication blocks in the federation. + */ + long getNumOfBlocksPendingReplication(); + + /** + * Get the number of under replicated blocks in the federation. + * @return Number of under replicated blocks in the federation. + */ + long getNumOfBlocksUnderReplicated(); + + /** + * Get the number of pending deletion blocks in the federation. + * @return Number of pending deletion blocks in the federation. + */ + long getNumOfBlocksPendingDeletion(); + + /** + * Get the number of files in the federation. + * @return Number of files in the federation. + */ + long getNumFiles(); + + /** + * When the router started. + * @return Date as a string the router started. + */ + String getRouterStarted(); + + /** + * Get the version of the router. + * @return Version of the router. + */ + String getVersion(); + + /** + * Get the compilation date of the router. + * @return Compilation date of the router. + */ + String getCompiledDate(); + + /** + * Get the compilation info of the router. + * @return Compilation info of the router. + */ + String getCompileInfo(); + + /** + * Get the host and port of the router. + * @return Host and port of the router. + */ + String getHostAndPort(); + + /** + * Get the identifier of the router. + * @return Identifier of the router. + */ + String getRouterId(); + + /** + * Get the host and port of the router. + * @return Host and port of the router. + */ + String getClusterId(); + + /** + * Get the host and port of the router. + * @return Host and port of the router. + */ + String getBlockPoolId(); + + /** + * Get the current state of the router. + * + * @return String label for the current router state. + */ + String getRouterStatus(); +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
