Merge branch 'HDFS-7240' into trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d6befb2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d6befb2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d6befb2 Branch: refs/heads/trunk Commit: 9d6befb2989a127b5dd7776522a12357b4eb3c61 Parents: 1b9ecc2 9197588 Author: Xiaoyu Yao <x...@apache.org> Authored: Tue Apr 24 12:08:43 2018 -0700 Committer: Xiaoyu Yao <x...@apache.org> Committed: Tue Apr 24 12:08:43 2018 -0700 ---------------------------------------------------------------------- LICENSE.txt | 68 + dev-support/bin/dist-layout-stitching | 22 +- dev-support/docker/Dockerfile | 3 + .../assemblies/hadoop-src-with-hdsl.xml | 56 + .../main/resources/assemblies/hadoop-src.xml | 2 + .../ensure-jars-have-correct-contents.sh | 6 + .../hadoop-client-minicluster/pom.xml | 7 + .../hadoop-client-runtime/pom.xml | 1 + .../src/main/bin/hadoop-functions.sh | 5 + .../hadoop-common/src/main/conf/hadoop-env.sh | 17 + .../src/main/conf/log4j.properties | 34 + .../src/main/resources/core-default.xml | 13 + .../conf/TestCommonConfigurationFields.java | 3 + hadoop-dist/pom.xml | 83 ++ hadoop-dist/src/main/compose/ozone/.env | 17 + .../src/main/compose/ozone/docker-compose.yaml | 61 + .../src/main/compose/ozone/docker-config | 35 + hadoop-hdds/client/pom.xml | 49 + .../apache/hadoop/hdds/scm/XceiverClient.java | 192 +++ .../hadoop/hdds/scm/XceiverClientHandler.java | 202 +++ .../hdds/scm/XceiverClientInitializer.java | 72 + .../hadoop/hdds/scm/XceiverClientManager.java | 218 +++ .../hadoop/hdds/scm/XceiverClientMetrics.java | 92 ++ .../hadoop/hdds/scm/XceiverClientRatis.java | 266 ++++ .../scm/client/ContainerOperationClient.java | 407 ++++++ .../hadoop/hdds/scm/client/HddsClientUtils.java | 232 ++++ .../hadoop/hdds/scm/client/package-info.java | 23 + .../apache/hadoop/hdds/scm/package-info.java | 23 + .../hdds/scm/storage/ChunkInputStream.java | 261 ++++ .../hdds/scm/storage/ChunkOutputStream.java | 227 +++ .../hadoop/hdds/scm/storage/package-info.java | 23 + .../common/dev-support/findbugsExcludeFile.xml | 21 + hadoop-hdds/common/pom.xml | 128 ++ .../org/apache/hadoop/hdds/HddsConfigKeys.java | 23 + .../java/org/apache/hadoop/hdds/HddsUtils.java | 318 +++++ .../apache/hadoop/hdds/client/OzoneQuota.java | 203 +++ .../hadoop/hdds/client/ReplicationFactor.java | 63 + .../hadoop/hdds/client/ReplicationType.java | 28 + .../apache/hadoop/hdds/client/package-info.java | 23 + .../hadoop/hdds/conf/HddsConfServlet.java | 182 +++ .../hadoop/hdds/conf/OzoneConfiguration.java | 162 +++ .../apache/hadoop/hdds/conf/package-info.java | 18 + .../org/apache/hadoop/hdds/package-info.java | 23 + .../hadoop/hdds/protocol/DatanodeDetails.java | 353 +++++ .../hadoop/hdds/protocol/package-info.java | 22 + .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 271 ++++ .../org/apache/hadoop/hdds/scm/ScmInfo.java | 81 ++ .../hadoop/hdds/scm/XceiverClientSpi.java | 129 ++ .../hadoop/hdds/scm/client/ScmClient.java | 139 ++ .../hadoop/hdds/scm/client/package-info.java | 24 + .../hadoop/hdds/scm/container/ContainerID.java | 97 ++ .../common/helpers/AllocatedBlock.java | 77 ++ .../container/common/helpers/ContainerInfo.java | 333 +++++ .../common/helpers/DeleteBlockResult.java | 51 + .../scm/container/common/helpers/Pipeline.java | 253 ++++ .../common/helpers/PipelineChannel.java | 122 ++ .../helpers/StorageContainerException.java | 104 ++ .../container/common/helpers/package-info.java | 22 + .../hadoop/hdds/scm/container/package-info.java | 18 + .../apache/hadoop/hdds/scm/package-info.java | 24 + .../hdds/scm/protocol/LocatedContainer.java | 127 ++ .../scm/protocol/ScmBlockLocationProtocol.java | 72 + .../hdds/scm/protocol/ScmLocatedBlock.java | 100 ++ .../StorageContainerLocationProtocol.java | 124 ++ .../hadoop/hdds/scm/protocol/package-info.java | 19 + ...kLocationProtocolClientSideTranslatorPB.java | 215 +++ .../protocolPB/ScmBlockLocationProtocolPB.java | 35 + ...rLocationProtocolClientSideTranslatorPB.java | 316 +++++ .../StorageContainerLocationProtocolPB.java | 36 + .../hdds/scm/protocolPB/package-info.java | 24 + .../scm/storage/ContainerProtocolCalls.java | 396 ++++++ .../hadoop/hdds/scm/storage/package-info.java | 23 + .../java/org/apache/hadoop/ozone/OzoneAcl.java | 231 ++++ .../apache/hadoop/ozone/OzoneConfigKeys.java | 241 ++++ .../org/apache/hadoop/ozone/OzoneConsts.java | 167 +++ .../apache/hadoop/ozone/common/BlockGroup.java | 87 ++ .../ozone/common/DeleteBlockGroupResult.java | 96 ++ .../InconsistentStorageStateException.java | 51 + .../org/apache/hadoop/ozone/common/Storage.java | 248 ++++ .../apache/hadoop/ozone/common/StorageInfo.java | 183 +++ .../hadoop/ozone/common/package-info.java | 18 + .../InvalidStateTransitionException.java | 42 + .../ozone/common/statemachine/StateMachine.java | 68 + .../ozone/common/statemachine/package-info.java | 21 + .../container/common/helpers/ChunkInfo.java | 185 +++ .../ozone/container/common/helpers/KeyData.java | 170 +++ .../container/common/helpers/package-info.java | 23 + .../org/apache/hadoop/ozone/lease/Lease.java | 189 +++ .../ozone/lease/LeaseAlreadyExistException.java | 46 + .../ozone/lease/LeaseCallbackExecutor.java | 65 + .../hadoop/ozone/lease/LeaseException.java | 45 + .../ozone/lease/LeaseExpiredException.java | 45 + .../apache/hadoop/ozone/lease/LeaseManager.java | 247 ++++ .../lease/LeaseManagerNotRunningException.java | 45 + .../ozone/lease/LeaseNotFoundException.java | 46 + .../apache/hadoop/ozone/lease/package-info.java | 26 + .../org/apache/hadoop/ozone/package-info.java | 35 + ...kLocationProtocolServerSideTranslatorPB.java | 170 +++ ...rLocationProtocolServerSideTranslatorPB.java | 212 +++ .../hadoop/ozone/protocolPB/package-info.java | 24 + .../hadoop/ozone/web/utils/JsonUtils.java | 71 + .../hadoop/ozone/web/utils/package-info.java | 19 + .../apache/hadoop/utils/BackgroundService.java | 161 +++ .../org/apache/hadoop/utils/BackgroundTask.java | 28 + .../hadoop/utils/BackgroundTaskQueue.java | 64 + .../hadoop/utils/BackgroundTaskResult.java | 44 + .../org/apache/hadoop/utils/BatchOperation.java | 90 ++ .../org/apache/hadoop/utils/EntryConsumer.java | 38 + .../org/apache/hadoop/utils/LevelDBStore.java | 380 ++++++ .../apache/hadoop/utils/MetadataKeyFilters.java | 118 ++ .../org/apache/hadoop/utils/MetadataStore.java | 172 +++ .../hadoop/utils/MetadataStoreBuilder.java | 126 ++ .../org/apache/hadoop/utils/RocksDBStore.java | 382 ++++++ .../apache/hadoop/utils/RocksDBStoreMBean.java | 144 ++ .../org/apache/hadoop/utils/package-info.java | 18 + .../main/java/org/apache/ratis/RatisHelper.java | 129 ++ .../java/org/apache/ratis/package-info.java | 22 + .../com/google/protobuf/ShadedProtoUtil.java | 38 + .../com/google/protobuf/package-info.java | 22 + .../main/proto/DatanodeContainerProtocol.proto | 415 ++++++ .../main/proto/ScmBlockLocationProtocol.proto | 166 +++ .../StorageContainerLocationProtocol.proto | 214 +++ hadoop-hdds/common/src/main/proto/hdds.proto | 170 +++ .../common/src/main/resources/ozone-default.xml | 1049 ++++++++++++++ .../apache/hadoop/hdds/scm/package-info.java | 21 + .../apache/hadoop/ozone/TestMetadataStore.java | 414 ++++++ .../org/apache/hadoop/ozone/TestOzoneAcls.java | 141 ++ .../hadoop/ozone/common/TestStateMachine.java | 106 ++ .../hadoop/ozone/lease/TestLeaseManager.java | 374 +++++ .../apache/hadoop/ozone/lease/package-info.java | 21 + .../org/apache/hadoop/ozone/package-info.java | 21 + .../hadoop/utils/TestRocksDBStoreMBean.java | 87 ++ hadoop-hdds/container-service/pom.xml | 103 ++ .../apache/hadoop/hdds/scm/HddsServerUtil.java | 325 +++++ .../org/apache/hadoop/hdds/scm/VersionInfo.java | 81 ++ .../apache/hadoop/hdds/scm/package-info.java | 19 + .../hadoop/ozone/HddsDatanodeService.java | 242 ++++ .../container/common/helpers/ChunkUtils.java | 346 +++++ .../container/common/helpers/ContainerData.java | 326 +++++ .../common/helpers/ContainerMetrics.java | 121 ++ .../common/helpers/ContainerReport.java | 218 +++ .../common/helpers/ContainerUtils.java | 442 ++++++ .../helpers/DeletedContainerBlocksSummary.java | 103 ++ .../container/common/helpers/FileUtils.java | 81 ++ .../container/common/helpers/KeyUtils.java | 148 ++ .../container/common/helpers/package-info.java | 22 + .../container/common/impl/ChunkManagerImpl.java | 235 ++++ .../impl/ContainerLocationManagerImpl.java | 150 ++ .../common/impl/ContainerManagerImpl.java | 1113 +++++++++++++++ .../common/impl/ContainerReportManagerImpl.java | 90 ++ .../container/common/impl/ContainerStatus.java | 217 +++ .../common/impl/ContainerStorageLocation.java | 203 +++ .../ozone/container/common/impl/Dispatcher.java | 713 ++++++++++ .../container/common/impl/KeyManagerImpl.java | 202 +++ .../RandomContainerDeletionChoosingPolicy.java | 70 + .../common/impl/StorageLocationReport.java | 63 + ...NOrderedContainerDeletionChoosingPolicy.java | 91 ++ .../container/common/impl/package-info.java | 22 + .../common/interfaces/ChunkManager.java | 76 ++ .../ContainerDeletionChoosingPolicy.java | 46 + .../common/interfaces/ContainerDispatcher.java | 51 + .../interfaces/ContainerLocationManager.java | 58 + .../ContainerLocationManagerMXBean.java | 36 + .../common/interfaces/ContainerManager.java | 280 ++++ .../interfaces/ContainerReportManager.java | 32 + .../container/common/interfaces/KeyManager.java | 76 ++ .../common/interfaces/package-info.java | 20 + .../ozone/container/common/package-info.java | 28 + .../statemachine/DatanodeStateMachine.java | 397 ++++++ .../statemachine/EndpointStateMachine.java | 294 ++++ .../statemachine/EndpointStateMachineMBean.java | 34 + .../statemachine/SCMConnectionManager.java | 208 +++ .../SCMConnectionManagerMXBean.java | 27 + .../common/statemachine/StateContext.java | 285 ++++ .../background/BlockDeletingService.java | 239 ++++ .../statemachine/background/package-info.java | 18 + .../commandhandler/CloseContainerHandler.java | 112 ++ .../commandhandler/CommandDispatcher.java | 177 +++ .../commandhandler/CommandHandler.java | 59 + .../commandhandler/ContainerReportHandler.java | 114 ++ .../DeleteBlocksCommandHandler.java | 211 +++ .../commandhandler/package-info.java | 18 + .../common/statemachine/package-info.java | 28 + .../container/common/states/DatanodeState.java | 55 + .../states/datanode/InitDatanodeState.java | 164 +++ .../states/datanode/RunningDatanodeState.java | 175 +++ .../common/states/datanode/package-info.java | 21 + .../states/endpoint/HeartbeatEndpointTask.java | 267 ++++ .../states/endpoint/RegisterEndpointTask.java | 208 +++ .../states/endpoint/VersionEndpointTask.java | 68 + .../common/states/endpoint/package-info.java | 20 + .../container/common/states/package-info.java | 18 + .../common/transport/server/XceiverServer.java | 130 ++ .../transport/server/XceiverServerHandler.java | 82 ++ .../server/XceiverServerInitializer.java | 62 + .../transport/server/XceiverServerSpi.java | 43 + .../common/transport/server/package-info.java | 24 + .../server/ratis/ContainerStateMachine.java | 293 ++++ .../server/ratis/XceiverServerRatis.java | 214 +++ .../transport/server/ratis/package-info.java | 23 + .../container/common/utils/ContainerCache.java | 168 +++ .../container/common/utils/package-info.java | 18 + .../container/ozoneimpl/OzoneContainer.java | 277 ++++ .../ozone/container/ozoneimpl/package-info.java | 21 + .../org/apache/hadoop/ozone/package-info.java | 23 + .../StorageContainerDatanodeProtocol.java | 96 ++ .../protocol/StorageContainerNodeProtocol.java | 68 + .../hadoop/ozone/protocol/VersionResponse.java | 150 ++ .../commands/CloseContainerCommand.java | 76 ++ .../protocol/commands/DeleteBlocksCommand.java | 66 + .../protocol/commands/RegisteredCommand.java | 229 ++++ .../protocol/commands/ReregisterCommand.java | 59 + .../ozone/protocol/commands/SCMCommand.java | 41 + .../protocol/commands/SendContainerCommand.java | 80 ++ .../ozone/protocol/commands/package-info.java | 21 + .../hadoop/ozone/protocol/package-info.java | 23 + ...rDatanodeProtocolClientSideTranslatorPB.java | 204 +++ .../StorageContainerDatanodeProtocolPB.java | 34 + ...rDatanodeProtocolServerSideTranslatorPB.java | 119 ++ .../hadoop/ozone/protocolPB/package-info.java | 19 + .../StorageContainerDatanodeProtocol.proto | 353 +++++ ...m.sun.jersey.spi.container.ContainerProvider | 16 + .../container/common/ContainerTestUtils.java | 68 + .../ozone/container/common/SCMTestUtils.java | 120 ++ .../ozone/container/common/ScmTestMock.java | 274 ++++ .../common/TestDatanodeStateMachine.java | 379 +++++ .../testutils/BlockDeletingServiceTestImpl.java | 104 ++ .../ozone/container/testutils/package-info.java | 18 + .../src/test/resources/log4j.properties | 23 + hadoop-hdds/framework/README.md | 24 + hadoop-hdds/framework/pom.xml | 70 + .../hadoop/hdds/server/BaseHttpServer.java | 222 +++ .../apache/hadoop/hdds/server/ServerUtils.java | 139 ++ .../hadoop/hdds/server/ServiceRuntimeInfo.java | 64 + .../hdds/server/ServiceRuntimeInfoImpl.java | 55 + .../apache/hadoop/hdds/server/package-info.java | 23 + .../src/main/resources/webapps/datanode/dn.js | 92 ++ .../webapps/static/angular-1.6.4.min.js | 332 +++++ .../webapps/static/angular-nvd3-1.0.9.min.js | 1 + .../webapps/static/angular-route-1.6.4.min.js | 17 + .../resources/webapps/static/d3-3.5.17.min.js | 5 + .../main/resources/webapps/static/dfs-dust.js | 133 ++ .../resources/webapps/static/nvd3-1.8.5.min.css | 2 + .../webapps/static/nvd3-1.8.5.min.css.map | 1 + .../resources/webapps/static/nvd3-1.8.5.min.js | 11 + .../webapps/static/nvd3-1.8.5.min.js.map | 1 + .../src/main/resources/webapps/static/ozone.css | 60 + .../src/main/resources/webapps/static/ozone.js | 387 ++++++ .../webapps/static/templates/config.html | 91 ++ .../resources/webapps/static/templates/jvm.html | 26 + .../webapps/static/templates/menu.html | 60 + .../webapps/static/templates/overview.html | 39 + .../webapps/static/templates/rpc-metrics.html | 87 ++ .../hadoop/hdds/server/TestBaseHttpServer.java | 98 ++ .../framework/src/test/resources/ozone-site.xml | 24 + hadoop-hdds/pom.xml | 117 ++ hadoop-hdds/server-scm/pom.xml | 159 +++ .../org/apache/hadoop/hdds/scm/SCMMXBean.java | 50 + .../org/apache/hadoop/hdds/scm/SCMStorage.java | 73 + .../hdds/scm/StorageContainerManager.java | 1290 ++++++++++++++++++ .../scm/StorageContainerManagerHttpServer.java | 76 ++ .../hadoop/hdds/scm/block/BlockManager.java | 86 ++ .../hadoop/hdds/scm/block/BlockManagerImpl.java | 530 +++++++ .../hdds/scm/block/BlockmanagerMXBean.java | 30 + .../block/DatanodeDeletedBlockTransactions.java | 132 ++ .../hadoop/hdds/scm/block/DeletedBlockLog.java | 124 ++ .../hdds/scm/block/DeletedBlockLogImpl.java | 356 +++++ .../hdds/scm/block/SCMBlockDeletingService.java | 175 +++ .../hadoop/hdds/scm/block/package-info.java | 22 + .../hdds/scm/container/ContainerMapping.java | 607 ++++++++ .../scm/container/ContainerStateManager.java | 456 +++++++ .../hadoop/hdds/scm/container/Mapping.java | 106 ++ .../scm/container/closer/ContainerCloser.java | 192 +++ .../hdds/scm/container/closer/package-info.java | 23 + .../hadoop/hdds/scm/container/package-info.java | 22 + .../algorithms/ContainerPlacementPolicy.java | 41 + .../placement/algorithms/SCMCommonPolicy.java | 197 +++ .../SCMContainerPlacementCapacity.java | 133 ++ .../algorithms/SCMContainerPlacementRandom.java | 89 ++ .../placement/algorithms/package-info.java | 18 + .../placement/metrics/ContainerStat.java | 165 +++ .../placement/metrics/DatanodeMetric.java | 91 ++ .../container/placement/metrics/LongMetric.java | 162 +++ .../container/placement/metrics/NodeStat.java | 67 + .../container/placement/metrics/SCMMetrics.java | 154 +++ .../placement/metrics/SCMNodeMetric.java | 223 +++ .../placement/metrics/SCMNodeStat.java | 139 ++ .../placement/metrics/package-info.java | 20 + .../scm/container/placement/package-info.java | 19 + .../replication/ContainerSupervisor.java | 343 +++++ .../container/replication/InProgressPool.java | 313 +++++ .../scm/container/replication/PeriodicPool.java | 119 ++ .../scm/container/replication/package-info.java | 23 + .../container/states/ContainerAttribute.java | 245 ++++ .../scm/container/states/ContainerState.java | 96 ++ .../scm/container/states/ContainerStateMap.java | 405 ++++++ .../hdds/scm/container/states/package-info.java | 22 + .../hdds/scm/exceptions/SCMException.java | 119 ++ .../hdds/scm/exceptions/package-info.java | 18 + .../hadoop/hdds/scm/node/CommandQueue.java | 190 +++ .../hdds/scm/node/HeartbeatQueueItem.java | 117 ++ .../hadoop/hdds/scm/node/NodeManager.java | 153 +++ .../hadoop/hdds/scm/node/NodeManagerMXBean.java | 57 + .../hadoop/hdds/scm/node/NodePoolManager.java | 71 + .../hadoop/hdds/scm/node/SCMNodeManager.java | 909 ++++++++++++ .../hdds/scm/node/SCMNodePoolManager.java | 269 ++++ .../hadoop/hdds/scm/node/package-info.java | 31 + .../apache/hadoop/hdds/scm/package-info.java | 22 + .../hdds/scm/pipelines/PipelineManager.java | 175 +++ .../hdds/scm/pipelines/PipelineSelector.java | 230 ++++ .../hadoop/hdds/scm/pipelines/package-info.java | 38 + .../scm/pipelines/ratis/RatisManagerImpl.java | 158 +++ .../hdds/scm/pipelines/ratis/package-info.java | 18 + .../standalone/StandaloneManagerImpl.java | 145 ++ .../scm/pipelines/standalone/package-info.java | 18 + .../hadoop/hdds/scm/ratis/package-info.java | 22 + .../server-scm/src/main/webapps/scm/index.html | 76 ++ .../server-scm/src/main/webapps/scm/main.html | 20 + .../src/main/webapps/scm/scm-overview.html | 60 + .../server-scm/src/main/webapps/scm/scm.js | 54 + .../hadoop/hdds/scm/HddsServerUtilTest.java | 308 +++++ .../TestStorageContainerManagerHttpServer.java | 140 ++ .../org/apache/hadoop/hdds/scm/TestUtils.java | 109 ++ .../hadoop/hdds/scm/block/TestBlockManager.java | 160 +++ .../hdds/scm/block/TestDeletedBlockLog.java | 363 +++++ .../hdds/scm/container/MockNodeManager.java | 520 +++++++ .../scm/container/TestContainerMapping.java | 333 +++++ .../container/closer/TestContainerCloser.java | 228 ++++ .../states/TestContainerAttribute.java | 143 ++ .../hdds/scm/node/TestContainerPlacement.java | 176 +++ .../hadoop/hdds/scm/node/TestNodeManager.java | 1176 ++++++++++++++++ .../hdds/scm/node/TestSCMNodePoolManager.java | 160 +++ .../apache/hadoop/hdds/scm/package-info.java | 21 + .../ozone/container/common/TestEndPoint.java | 458 +++++++ .../placement/TestContainerPlacement.java | 134 ++ .../placement/TestDatanodeMetrics.java | 59 + .../replication/TestContainerSupervisor.java | 272 ++++ .../container/replication/package-info.java | 18 + .../ReplicationDatanodeStateManager.java | 100 ++ .../testutils/ReplicationNodeManagerMock.java | 326 +++++ .../ReplicationNodePoolManagerMock.java | 133 ++ .../ozone/container/testutils/package-info.java | 18 + hadoop-hdds/tools/pom.xml | 65 + .../hadoop/hdds/scm/cli/OzoneBaseCLI.java | 43 + .../hdds/scm/cli/OzoneCommandHandler.java | 87 ++ .../apache/hadoop/hdds/scm/cli/ResultCode.java | 31 + .../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 234 ++++ .../cli/container/CloseContainerHandler.java | 83 ++ .../cli/container/ContainerCommandHandler.java | 129 ++ .../cli/container/CreateContainerHandler.java | 81 ++ .../cli/container/DeleteContainerHandler.java | 92 ++ .../scm/cli/container/InfoContainerHandler.java | 114 ++ .../scm/cli/container/ListContainerHandler.java | 121 ++ .../hdds/scm/cli/container/package-info.java | 19 + .../hadoop/hdds/scm/cli/package-info.java | 19 + .../src/main/webapps/datanode/datanode.html | 46 + .../src/test/resources/log4j.properties | 24 + hadoop-minicluster/pom.xml | 6 + hadoop-ozone/acceptance-test/README.md | 38 + hadoop-ozone/acceptance-test/pom.xml | 86 ++ .../acceptance-test/src/test/compose/.env | 17 + .../src/test/compose/docker-compose.yaml | 61 + .../src/test/compose/docker-config | 35 + .../test/robotframework/acceptance/ozone.robot | 116 ++ hadoop-ozone/client/pom.xml | 43 + .../apache/hadoop/ozone/client/BucketArgs.java | 123 ++ .../apache/hadoop/ozone/client/ObjectStore.java | 211 +++ .../apache/hadoop/ozone/client/OzoneBucket.java | 363 +++++ .../apache/hadoop/ozone/client/OzoneClient.java | 103 ++ .../hadoop/ozone/client/OzoneClientFactory.java | 307 +++++ .../client/OzoneClientInvocationHandler.java | 62 + .../apache/hadoop/ozone/client/OzoneKey.java | 119 ++ .../apache/hadoop/ozone/client/OzoneVolume.java | 295 ++++ .../apache/hadoop/ozone/client/VolumeArgs.java | 128 ++ .../ozone/client/io/ChunkGroupInputStream.java | 319 +++++ .../ozone/client/io/ChunkGroupOutputStream.java | 495 +++++++ .../client/io/OzoneContainerTranslation.java | 51 + .../ozone/client/io/OzoneInputStream.java | 60 + .../ozone/client/io/OzoneOutputStream.java | 64 + .../hadoop/ozone/client/io/package-info.java | 23 + .../hadoop/ozone/client/package-info.java | 23 + .../ozone/client/protocol/ClientProtocol.java | 323 +++++ .../ozone/client/protocol/package-info.java | 23 + .../client/rest/DefaultRestServerSelector.java | 36 + .../ozone/client/rest/OzoneExceptionMapper.java | 45 + .../hadoop/ozone/client/rest/RestClient.java | 799 +++++++++++ .../ozone/client/rest/RestServerSelector.java | 40 + .../client/rest/exceptions/package-info.java | 22 + .../ozone/client/rest/headers/package-info.java | 22 + .../hadoop/ozone/client/rest/package-info.java | 23 + .../hadoop/ozone/client/rpc/RpcClient.java | 567 ++++++++ .../hadoop/ozone/client/rpc/package-info.java | 23 + .../hadoop/ozone/web/client/OzoneBucket.java | 645 +++++++++ .../hadoop/ozone/web/client/OzoneKey.java | 44 + .../ozone/web/client/OzoneRestClient.java | 803 +++++++++++ .../web/client/OzoneRestClientException.java | 54 + .../hadoop/ozone/web/client/OzoneVolume.java | 583 ++++++++ .../hadoop/ozone/web/client/package-info.java | 34 + .../ozone/client/TestHddsClientUtils.java | 105 ++ .../hadoop/ozone/client/package-info.java | 23 + .../common/dev-support/findbugsExcludeFile.xml | 22 + hadoop-ozone/common/pom.xml | 88 ++ hadoop-ozone/common/src/main/bin/ozone | 188 +++ hadoop-ozone/common/src/main/bin/start-ozone.sh | 105 ++ hadoop-ozone/common/src/main/bin/stop-ozone.sh | 99 ++ .../common/src/main/conf/ozone-site.xml | 24 + .../java/org/apache/hadoop/ozone/KsmUtils.java | 87 ++ .../ozone/client/io/LengthInputStream.java | 49 + .../hadoop/ozone/client/io/package-info.java | 22 + .../ozone/client/rest/OzoneException.java | 267 ++++ .../ozone/client/rest/headers/Header.java | 71 + .../ozone/client/rest/headers/package-info.java | 22 + .../hadoop/ozone/client/rest/package-info.java | 22 + .../ozone/client/rest/response/BucketInfo.java | 231 ++++ .../ozone/client/rest/response/KeyInfo.java | 216 +++ .../ozone/client/rest/response/VolumeInfo.java | 215 +++ .../ozone/client/rest/response/VolumeOwner.java | 61 + .../client/rest/response/package-info.java | 24 + .../apache/hadoop/ozone/freon/OzoneGetConf.java | 269 ++++ .../apache/hadoop/ozone/freon/package-info.java | 21 + .../apache/hadoop/ozone/ksm/KSMConfigKeys.java | 81 ++ .../hadoop/ozone/ksm/helpers/KsmBucketArgs.java | 233 ++++ .../hadoop/ozone/ksm/helpers/KsmBucketInfo.java | 235 ++++ .../hadoop/ozone/ksm/helpers/KsmKeyArgs.java | 119 ++ .../hadoop/ozone/ksm/helpers/KsmKeyInfo.java | 243 ++++ .../ozone/ksm/helpers/KsmKeyLocationInfo.java | 136 ++ .../ksm/helpers/KsmKeyLocationInfoGroup.java | 118 ++ .../ozone/ksm/helpers/KsmOzoneAclMap.java | 110 ++ .../hadoop/ozone/ksm/helpers/KsmVolumeArgs.java | 223 +++ .../ozone/ksm/helpers/OpenKeySession.java | 50 + .../hadoop/ozone/ksm/helpers/ServiceInfo.java | 237 ++++ .../hadoop/ozone/ksm/helpers/VolumeArgs.java | 140 ++ .../hadoop/ozone/ksm/helpers/package-info.java | 18 + .../apache/hadoop/ozone/ksm/package-info.java | 21 + .../ksm/protocol/KeySpaceManagerProtocol.java | 245 ++++ .../hadoop/ozone/ksm/protocol/package-info.java | 19 + ...ceManagerProtocolClientSideTranslatorPB.java | 744 ++++++++++ .../protocolPB/KeySpaceManagerProtocolPB.java | 34 + .../ozone/ksm/protocolPB/package-info.java | 19 + .../org/apache/hadoop/ozone/package-info.java | 22 + .../hadoop/ozone/protocolPB/KSMPBHelper.java | 113 ++ .../hadoop/ozone/protocolPB/package-info.java | 24 + .../hadoop/ozone/web/handlers/BucketArgs.java | 214 +++ .../hadoop/ozone/web/handlers/KeyArgs.java | 117 ++ .../hadoop/ozone/web/handlers/ListArgs.java | 142 ++ .../hadoop/ozone/web/handlers/UserArgs.java | 168 +++ .../hadoop/ozone/web/handlers/VolumeArgs.java | 142 ++ .../hadoop/ozone/web/handlers/package-info.java | 22 + .../hadoop/ozone/web/request/OzoneQuota.java | 214 +++ .../hadoop/ozone/web/request/package-info.java | 23 + .../hadoop/ozone/web/response/BucketInfo.java | 325 +++++ .../hadoop/ozone/web/response/KeyInfo.java | 290 ++++ .../hadoop/ozone/web/response/ListBuckets.java | 154 +++ .../hadoop/ozone/web/response/ListKeys.java | 209 +++ .../hadoop/ozone/web/response/ListVolumes.java | 152 +++ .../hadoop/ozone/web/response/VolumeInfo.java | 308 +++++ .../hadoop/ozone/web/response/VolumeOwner.java | 60 + .../hadoop/ozone/web/response/package-info.java | 23 + .../hadoop/ozone/web/utils/OzoneUtils.java | 227 +++ .../hadoop/ozone/web/utils/package-info.java | 18 + .../main/proto/KeySpaceManagerProtocol.proto | 457 +++++++ .../src/main/shellprofile.d/hadoop-ozone.sh | 44 + .../apache/hadoop/ozone/web/TestBucketInfo.java | 81 ++ .../org/apache/hadoop/ozone/web/TestQuota.java | 116 ++ .../org/apache/hadoop/ozone/web/TestUtils.java | 101 ++ .../hadoop/ozone/web/TestVolumeStructs.java | 73 + .../apache/hadoop/ozone/web/package-info.java | 21 + hadoop-ozone/integration-test/pom.xml | 86 ++ .../container/TestContainerStateManager.java | 300 ++++ .../apache/hadoop/ozone/MiniOzoneCluster.java | 322 +++++ .../hadoop/ozone/MiniOzoneClusterImpl.java | 425 ++++++ .../apache/hadoop/ozone/RatisTestHelper.java | 110 ++ .../hadoop/ozone/TestContainerOperations.java | 87 ++ .../hadoop/ozone/TestMiniOzoneCluster.java | 217 +++ .../ozone/TestOzoneConfigurationFields.java | 38 + .../apache/hadoop/ozone/TestOzoneHelper.java | 413 ++++++ .../ozone/TestStorageContainerManager.java | 443 ++++++ .../TestStorageContainerManagerHelper.java | 179 +++ .../ozone/client/rest/TestOzoneRestClient.java | 404 ++++++ .../hadoop/ozone/client/rest/package-info.java | 23 + .../ozone/client/rpc/TestOzoneRpcClient.java | 815 +++++++++++ .../hadoop/ozone/client/rpc/package-info.java | 23 + .../ozone/container/ContainerTestHelper.java | 601 ++++++++ .../common/TestBlockDeletingService.java | 398 ++++++ .../TestContainerDeletionChoosingPolicy.java | 197 +++ .../common/impl/TestContainerPersistence.java | 905 ++++++++++++ .../TestCloseContainerHandler.java | 114 ++ .../container/metrics/TestContainerMetrics.java | 127 ++ .../container/ozoneimpl/TestOzoneContainer.java | 587 ++++++++ .../ozoneimpl/TestOzoneContainerRatis.java | 142 ++ .../container/ozoneimpl/TestRatisManager.java | 128 ++ .../container/server/TestContainerServer.java | 262 ++++ .../hadoop/ozone/freon/TestDataValidate.java | 145 ++ .../apache/hadoop/ozone/freon/TestFreon.java | 130 ++ .../apache/hadoop/ozone/freon/package-info.java | 21 + .../ozone/ksm/TestContainerReportWithKeys.java | 143 ++ .../apache/hadoop/ozone/ksm/TestKSMMetrcis.java | 306 +++++ .../apache/hadoop/ozone/ksm/TestKSMSQLCli.java | 284 ++++ .../hadoop/ozone/ksm/TestKeySpaceManager.java | 1238 +++++++++++++++++ .../ksm/TestKeySpaceManagerRestInterface.java | 134 ++ .../ozone/ksm/TestKsmBlockVersioning.java | 253 ++++ .../ksm/TestMultipleContainerReadWrite.java | 215 +++ .../hadoop/ozone/ozShell/TestOzoneShell.java | 800 +++++++++++ .../hadoop/ozone/scm/TestAllocateContainer.java | 99 ++ .../hadoop/ozone/scm/TestContainerSQLCli.java | 300 ++++ .../ozone/scm/TestContainerSmallFile.java | 147 ++ .../org/apache/hadoop/ozone/scm/TestSCMCli.java | 540 ++++++++ .../apache/hadoop/ozone/scm/TestSCMMXBean.java | 161 +++ .../apache/hadoop/ozone/scm/TestSCMMetrics.java | 258 ++++ .../ozone/scm/TestXceiverClientManager.java | 198 +++ .../ozone/scm/TestXceiverClientMetrics.java | 168 +++ .../hadoop/ozone/scm/node/TestQueryNode.java | 120 ++ .../ozone/web/TestDistributedOzoneVolumes.java | 186 +++ .../hadoop/ozone/web/TestLocalOzoneVolumes.java | 185 +++ .../ozone/web/TestOzoneRestWithMiniCluster.java | 263 ++++ .../hadoop/ozone/web/TestOzoneWebAccess.java | 125 ++ .../hadoop/ozone/web/client/TestBuckets.java | 248 ++++ .../ozone/web/client/TestBucketsRatis.java | 76 ++ .../hadoop/ozone/web/client/TestKeys.java | 688 ++++++++++ .../hadoop/ozone/web/client/TestKeysRatis.java | 127 ++ .../ozone/web/client/TestOzoneClient.java | 305 +++++ .../hadoop/ozone/web/client/TestVolume.java | 433 ++++++ .../ozone/web/client/TestVolumeRatis.java | 129 ++ .../src/test/resources/log4j.properties | 18 + .../src/test/resources/webapps/ksm/.gitkeep | 15 + .../src/test/resources/webapps/scm/.gitkeep | 15 + hadoop-ozone/objectstore-service/pom.xml | 69 + .../server/datanode/ObjectStoreHandler.java | 191 +++ .../hdfs/server/datanode/package-info.java | 22 + .../org/apache/hadoop/ozone/OzoneRestUtils.java | 222 +++ .../org/apache/hadoop/ozone/package-info.java | 22 + .../ozone/web/ObjectStoreApplication.java | 59 + .../ozone/web/OzoneHddsDatanodeService.java | 84 ++ .../hadoop/ozone/web/exceptions/ErrorTable.java | 225 +++ .../ozone/web/exceptions/package-info.java | 22 + .../ozone/web/handlers/BucketHandler.java | 197 +++ .../web/handlers/BucketProcessTemplate.java | 323 +++++ .../hadoop/ozone/web/handlers/KeyHandler.java | 245 ++++ .../ozone/web/handlers/KeyProcessTemplate.java | 235 ++++ .../ozone/web/handlers/ServiceFilter.java | 61 + .../web/handlers/StorageHandlerBuilder.java | 76 ++ .../ozone/web/handlers/UserHandlerBuilder.java | 75 + .../ozone/web/handlers/VolumeHandler.java | 272 ++++ .../web/handlers/VolumeProcessTemplate.java | 276 ++++ .../hadoop/ozone/web/handlers/package-info.java | 22 + .../hadoop/ozone/web/interfaces/Accounting.java | 57 + .../hadoop/ozone/web/interfaces/Bucket.java | 183 +++ .../hadoop/ozone/web/interfaces/Keys.java | 146 ++ .../ozone/web/interfaces/StorageHandler.java | 295 ++++ .../hadoop/ozone/web/interfaces/UserAuth.java | 101 ++ .../hadoop/ozone/web/interfaces/Volume.java | 182 +++ .../ozone/web/interfaces/package-info.java | 22 + .../web/localstorage/LocalStorageHandler.java | 379 +++++ .../web/localstorage/OzoneMetadataManager.java | 1138 +++++++++++++++ .../ozone/web/localstorage/package-info.java | 18 + .../LengthInputStreamMessageBodyWriter.java | 59 + .../web/messages/StringMessageBodyWriter.java | 62 + .../hadoop/ozone/web/messages/package-info.java | 18 + .../web/netty/CloseableCleanupListener.java | 46 + .../web/netty/ObjectStoreChannelHandler.java | 78 ++ .../web/netty/ObjectStoreJerseyContainer.java | 348 +++++ .../ObjectStoreJerseyContainerProvider.java | 40 + .../web/netty/ObjectStoreRestHttpServer.java | 215 +++ .../web/netty/ObjectStoreURLDispatcher.java | 61 + ...RequestContentObjectStoreChannelHandler.java | 117 ++ ...equestDispatchObjectStoreChannelHandler.java | 103 ++ .../hadoop/ozone/web/netty/package-info.java | 26 + .../apache/hadoop/ozone/web/package-info.java | 22 + .../web/storage/DistributedStorageHandler.java | 539 ++++++++ .../hadoop/ozone/web/storage/package-info.java | 27 + .../hadoop/ozone/web/userauth/Simple.java | 169 +++ .../hadoop/ozone/web/userauth/package-info.java | 23 + ...m.sun.jersey.spi.container.ContainerProvider | 16 + .../org/apache/hadoop/ozone/TestErrorCode.java | 53 + hadoop-ozone/ozone-manager/pom.xml | 114 ++ .../apache/hadoop/ozone/ksm/BucketManager.java | 79 ++ .../hadoop/ozone/ksm/BucketManagerImpl.java | 315 +++++ .../org/apache/hadoop/ozone/ksm/KSMMXBean.java | 31 + .../hadoop/ozone/ksm/KSMMetadataManager.java | 253 ++++ .../ozone/ksm/KSMMetadataManagerImpl.java | 525 +++++++ .../org/apache/hadoop/ozone/ksm/KSMMetrics.java | 437 ++++++ .../org/apache/hadoop/ozone/ksm/KSMStorage.java | 90 ++ .../hadoop/ozone/ksm/KeyDeletingService.java | 141 ++ .../org/apache/hadoop/ozone/ksm/KeyManager.java | 165 +++ .../apache/hadoop/ozone/ksm/KeyManagerImpl.java | 512 +++++++ .../hadoop/ozone/ksm/KeySpaceManager.java | 901 ++++++++++++ .../ozone/ksm/KeySpaceManagerHttpServer.java | 78 ++ .../hadoop/ozone/ksm/OpenKeyCleanupService.java | 116 ++ .../ozone/ksm/ServiceListJSONServlet.java | 103 ++ .../apache/hadoop/ozone/ksm/VolumeManager.java | 100 ++ .../hadoop/ozone/ksm/VolumeManagerImpl.java | 391 ++++++ .../ozone/ksm/exceptions/KSMException.java | 116 ++ .../ozone/ksm/exceptions/package-info.java | 19 + .../apache/hadoop/ozone/ksm/package-info.java | 21 + ...ceManagerProtocolServerSideTranslatorPB.java | 539 ++++++++ .../hadoop/ozone/protocolPB/package-info.java | 22 + .../hadoop/ozone/web/ozShell/Handler.java | 79 ++ .../apache/hadoop/ozone/web/ozShell/Shell.java | 415 ++++++ .../web/ozShell/bucket/CreateBucketHandler.java | 96 ++ .../web/ozShell/bucket/DeleteBucketHandler.java | 87 ++ .../web/ozShell/bucket/InfoBucketHandler.java | 94 ++ .../web/ozShell/bucket/ListBucketHandler.java | 113 ++ .../web/ozShell/bucket/UpdateBucketHandler.java | 94 ++ .../ozone/web/ozShell/bucket/package-info.java | 23 + .../web/ozShell/keys/DeleteKeyHandler.java | 96 ++ .../ozone/web/ozShell/keys/GetKeyHandler.java | 116 ++ .../ozone/web/ozShell/keys/InfoKeyHandler.java | 98 ++ .../ozone/web/ozShell/keys/ListKeyHandler.java | 116 ++ .../ozone/web/ozShell/keys/PutKeyHandler.java | 106 ++ .../ozone/web/ozShell/keys/package-info.java | 23 + .../hadoop/ozone/web/ozShell/package-info.java | 27 + .../web/ozShell/volume/CreateVolumeHandler.java | 99 ++ .../web/ozShell/volume/DeleteVolumeHandler.java | 81 ++ .../web/ozShell/volume/InfoVolumeHandler.java | 95 ++ .../web/ozShell/volume/ListVolumeHandler.java | 114 ++ .../web/ozShell/volume/UpdateVolumeHandler.java | 92 ++ .../ozone/web/ozShell/volume/package-info.java | 23 + .../apache/hadoop/ozone/web/package-info.java | 24 + .../src/main/site/markdown/OzoneCommandShell.md | 150 ++ .../site/markdown/OzoneGettingStarted.md.vm | 347 +++++ .../src/main/site/markdown/OzoneMetrics.md | 166 +++ .../src/main/site/markdown/OzoneOverview.md | 88 ++ .../src/main/site/markdown/OzoneRest.md | 549 ++++++++ .../src/main/webapps/ksm/index.html | 70 + .../src/main/webapps/ksm/ksm-metrics.html | 44 + .../ozone-manager/src/main/webapps/ksm/ksm.js | 108 ++ .../ozone-manager/src/main/webapps/ksm/main.css | 23 + .../src/main/webapps/ksm/main.html | 18 + .../hadoop/ozone/ksm/TestBucketManagerImpl.java | 395 ++++++ .../hadoop/ozone/ksm/TestChunkStreams.java | 234 ++++ .../ksm/TestKeySpaceManagerHttpServer.java | 141 ++ .../apache/hadoop/ozone/ksm/package-info.java | 21 + hadoop-ozone/pom.xml | 169 +++ hadoop-ozone/tools/pom.xml | 71 + .../org/apache/hadoop/ozone/freon/Freon.java | 1149 ++++++++++++++++ .../apache/hadoop/ozone/freon/package-info.java | 22 + .../genesis/BenchMarkContainerStateMap.java | 171 +++ .../genesis/BenchMarkDatanodeDispatcher.java | 280 ++++ .../genesis/BenchMarkMetadataStoreReads.java | 67 + .../genesis/BenchMarkMetadataStoreWrites.java | 60 + .../ozone/genesis/BenchMarkRocksDbStore.java | 115 ++ .../apache/hadoop/ozone/genesis/Genesis.java | 54 + .../hadoop/ozone/genesis/GenesisUtil.java | 90 ++ .../hadoop/ozone/genesis/package-info.java | 25 + .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 709 ++++++++++ .../hadoop/ozone/scm/cli/package-info.java | 22 + .../org/apache/hadoop/test/OzoneTestDriver.java | 59 + hadoop-project/pom.xml | 146 ++ hadoop-project/src/site/site.xml | 9 + hadoop-tools/hadoop-ozone/pom.xml | 199 +++ .../org/apache/hadoop/fs/ozone/Constants.java | 48 + .../java/org/apache/hadoop/fs/ozone/OzFs.java | 43 + .../hadoop/fs/ozone/OzoneFSInputStream.java | 79 ++ .../hadoop/fs/ozone/OzoneFSOutputStream.java | 59 + .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 700 ++++++++++ .../apache/hadoop/fs/ozone/package-info.java | 30 + .../hadoop/fs/ozone/TestOzoneFSInputStream.java | 155 +++ .../fs/ozone/TestOzoneFileInterfaces.java | 230 ++++ .../contract/ITestOzoneContractCreate.java | 48 + .../contract/ITestOzoneContractDelete.java | 48 + .../contract/ITestOzoneContractDistCp.java | 50 + .../ITestOzoneContractGetFileStatus.java | 61 + .../ozone/contract/ITestOzoneContractMkdir.java | 48 + .../ozone/contract/ITestOzoneContractOpen.java | 47 + .../contract/ITestOzoneContractRename.java | 49 + .../contract/ITestOzoneContractRootDir.java | 51 + .../ozone/contract/ITestOzoneContractSeek.java | 47 + .../hadoop/fs/ozone/contract/OzoneContract.java | 123 ++ .../src/test/resources/contract/ozone.xml | 113 ++ .../src/test/resources/log4j.properties | 23 + hadoop-tools/hadoop-tools-dist/pom.xml | 15 + hadoop-tools/pom.xml | 12 +- pom.xml | 72 + 673 files changed, 103290 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
