Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3810f7b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3810f7b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3810f7b Branch: refs/heads/HDFS-12943 Commit: a3810f7b6bd822b4243a15c0ac31ab0f99b10b29 Parents: c377e3c 6b5838e Author: Erik Krogen <[email protected]> Authored: Wed Sep 19 14:53:51 2018 -0700 Committer: Erik Krogen <[email protected]> Committed: Wed Sep 19 14:53:51 2018 -0700 ---------------------------------------------------------------------- dev-support/bin/ozone-dist-layout-stitching | 7 +- .../hadoop-annotations/pom.xml | 24 ++ .../org/apache/hadoop/ipc/CallQueueManager.java | 5 +- .../apache/hadoop/log/LogThrottlingHelper.java | 358 +++++++++++++++++++ .../org/apache/hadoop/util/StringUtils.java | 2 +- .../hadoop/log/TestLogThrottlingHelper.java | 172 +++++++++ .../org/apache/hadoop/util/TestStringUtils.java | 9 + .../main/compose/ozone-hdfs/docker-compose.yaml | 2 +- .../src/main/compose/ozone-hdfs/docker-config | 1 + .../src/main/compose/ozone/docker-config | 1 + .../main/compose/ozonefs/docker-compose.yaml | 59 +++ .../src/main/compose/ozonefs/docker-config | 35 ++ .../src/main/compose/ozoneperf/docker-config | 1 + .../src/main/compose/ozonescripts/docker-config | 1 + hadoop-dist/src/main/smoketest/README.md | 30 ++ .../src/main/smoketest/basic/basic.robot | 47 +++ .../src/main/smoketest/basic/ozone-shell.robot | 82 +++++ hadoop-dist/src/main/smoketest/commonlib.robot | 24 ++ .../src/main/smoketest/ozonefs/ozonefs.robot | 35 ++ hadoop-dist/src/main/smoketest/test.sh | 101 ++++++ .../hadoop/hdds/scm/XceiverClientRatis.java | 6 +- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 5 + .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 13 + .../scm/container/common/helpers/Pipeline.java | 22 +- .../container/common/helpers/PipelineID.java | 13 +- .../apache/hadoop/ozone/OzoneConfigKeys.java | 14 + .../main/java/org/apache/ratis/RatisHelper.java | 40 ++- .../main/proto/ScmBlockLocationProtocol.proto | 6 +- hadoop-hdds/common/src/main/proto/hdds.proto | 7 + .../common/src/main/resources/ozone-default.xml | 27 ++ .../apache/hadoop/hdds/scm/HddsServerUtil.java | 21 ++ .../common/report/PipelineReportPublisher.java | 73 ++++ .../common/report/ReportPublisherFactory.java | 4 + .../statemachine/DatanodeStateMachine.java | 19 +- .../common/statemachine/StateContext.java | 42 ++- .../CloseContainerCommandHandler.java | 17 +- .../ReplicateContainerCommandHandler.java | 120 +------ .../states/datanode/InitDatanodeState.java | 12 +- .../states/endpoint/RegisterEndpointTask.java | 8 +- .../transport/server/XceiverServerGrpc.java | 28 +- .../transport/server/XceiverServerSpi.java | 9 + .../server/ratis/XceiverServerRatis.java | 105 +++--- .../container/keyvalue/KeyValueContainer.java | 1 + .../container/ozoneimpl/OzoneContainer.java | 126 +++---- .../replication/ContainerReplicator.java | 27 ++ .../DownloadAndImportReplicator.java | 136 +++++++ .../replication/GrpcReplicationClient.java | 2 +- .../replication/ReplicationSupervisor.java | 142 ++++++++ .../container/replication/ReplicationTask.java | 102 ++++++ .../StorageContainerDatanodeProtocol.java | 10 +- .../protocol/StorageContainerNodeProtocol.java | 6 +- .../commands/CloseContainerCommand.java | 23 +- ...rDatanodeProtocolClientSideTranslatorPB.java | 6 +- ...rDatanodeProtocolServerSideTranslatorPB.java | 5 +- .../StorageContainerDatanodeProtocol.proto | 10 + .../ozone/container/common/ScmTestMock.java | 8 +- .../common/TestDatanodeStateMachine.java | 42 +++ .../TestReplicateContainerCommandHandler.java | 163 --------- .../replication/TestReplicationSupervisor.java | 143 ++++++++ .../container/replication/package-info.java | 22 ++ .../hadoop/hdds/server/events/EventQueue.java | 18 + hadoop-hdds/pom.xml | 7 +- .../org/apache/hadoop/hdds/scm/ScmUtils.java | 45 +++ .../hadoop/hdds/scm/block/BlockManagerImpl.java | 17 +- .../container/CloseContainerEventHandler.java | 90 +++-- .../hdds/scm/container/ContainerMapping.java | 25 -- .../scm/container/ContainerReportHandler.java | 6 +- .../hadoop/hdds/scm/container/Mapping.java | 15 +- .../hadoop/hdds/scm/events/SCMEvents.java | 15 +- .../hdds/scm/exceptions/SCMException.java | 3 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 5 +- .../hadoop/hdds/scm/node/StaleNodeHandler.java | 19 +- .../hdds/scm/node/states/Node2ContainerMap.java | 123 +------ .../hdds/scm/node/states/Node2ObjectsMap.java | 162 +++++++++ .../hdds/scm/node/states/ReportResult.java | 105 +++--- .../hdds/scm/pipelines/Node2PipelineMap.java | 45 +-- .../scm/pipelines/PipelineCloseHandler.java | 24 +- .../hdds/scm/pipelines/PipelineManager.java | 52 ++- .../scm/pipelines/PipelineReportHandler.java | 59 +++ .../hdds/scm/pipelines/PipelineSelector.java | 107 +++--- .../scm/pipelines/ratis/RatisManagerImpl.java | 41 +-- .../standalone/StandaloneManagerImpl.java | 44 +-- .../hdds/scm/server/ChillModePrecheck.java | 12 +- .../hdds/scm/server/SCMChillModeManager.java | 1 + .../scm/server/SCMClientProtocolServer.java | 53 ++- .../server/SCMDatanodeHeartbeatDispatcher.java | 23 ++ .../scm/server/SCMDatanodeProtocolServer.java | 16 +- .../scm/server/StorageContainerManager.java | 129 +++++-- .../org/apache/hadoop/hdds/scm/TestUtils.java | 10 +- .../hdds/scm/container/MockNodeManager.java | 4 +- .../scm/container/TestContainerMapping.java | 54 ++- .../hadoop/hdds/scm/node/TestNodeManager.java | 6 +- .../scm/node/states/TestNode2ContainerMap.java | 28 +- .../scm/server/TestSCMClientProtocolServer.java | 60 ++++ .../ozone/container/common/TestEndPoint.java | 5 +- .../testutils/ReplicationNodeManagerMock.java | 5 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 4 + .../hadoop/fs/http/client/HttpFSFileSystem.java | 2 +- .../hadoop/fs/http/server/FSOperations.java | 3 + .../fs/http/client/BaseTestHttpFSWith.java | 35 +- .../BlockPlacementPolicyDefault.java | 29 +- .../apache/hadoop/hdfs/server/common/Util.java | 3 +- .../hdfs/server/datanode/BlockSender.java | 7 +- .../hdfs/server/namenode/FSNamesystemLock.java | 46 +-- .../hdfs/server/namenode/ha/EditLogTailer.java | 34 +- .../snapshot/DirectorySnapshottableFeature.java | 2 +- .../apache/hadoop/hdfs/MiniDFSNNTopology.java | 17 + .../blockmanagement/TestReplicationPolicy.java | 28 ++ .../TestDataNodeMultipleRegistrations.java | 4 +- .../server/namenode/ha/TestEditLogTailer.java | 98 +++-- .../snapshot/TestSnapshotDiffReport.java | 6 + .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 6 + .../v2/app/speculate/DefaultSpeculator.java | 4 +- .../hadoop/mapreduce/v2/app/MRAppBenchmark.java | 20 +- hadoop-ozone/acceptance-test/README.md | 48 --- .../dev-support/bin/robot-all.sh | 18 - .../dev-support/bin/robot-dnd-all.sh | 57 --- .../acceptance-test/dev-support/bin/robot.sh | 38 -- .../dev-support/docker/Dockerfile | 21 -- .../dev-support/docker/docker-compose.yaml | 23 -- hadoop-ozone/acceptance-test/pom.xml | 65 ---- .../src/test/acceptance/basic/.env | 17 - .../src/test/acceptance/basic/basic.robot | 50 --- .../test/acceptance/basic/docker-compose.yaml | 50 --- .../src/test/acceptance/basic/docker-config | 34 -- .../src/test/acceptance/basic/ozone-shell.robot | 86 ----- .../src/test/acceptance/commonlib.robot | 78 ---- .../src/test/acceptance/ozonefs/.env | 17 - .../test/acceptance/ozonefs/docker-compose.yaml | 59 --- .../src/test/acceptance/ozonefs/docker-config | 35 -- .../src/test/acceptance/ozonefs/ozonefs.robot | 39 -- .../acceptance/ozonefs/ozonesinglenode.robot | 49 --- .../hadoop/ozone/om/helpers/OmBucketInfo.java | 2 +- .../hadoop/ozone/om/helpers/OmVolumeArgs.java | 2 +- .../src/main/proto/OzoneManagerProtocol.proto | 2 +- hadoop-ozone/docs/content/JavaApi.md | 4 + hadoop-ozone/docs/content/OzoneFS.md | 6 +- hadoop-ozone/docs/pom.xml | 5 + .../hdds/scm/pipeline/TestNode2PipelineMap.java | 2 +- .../hdds/scm/pipeline/TestPipelineClose.java | 2 +- .../hdds/scm/pipeline/TestSCMRestart.java | 20 +- .../apache/hadoop/ozone/RatisTestHelper.java | 9 + .../ozone/TestStorageContainerManager.java | 54 +++ .../rpc/TestCloseContainerHandlingByClient.java | 5 +- .../TestCloseContainerByPipeline.java | 5 +- .../hadoop/ozone/om/TestScmChillMode.java | 171 +++++++++ .../hadoop/ozone/scm/TestContainerSQLCli.java | 15 +- .../hadoop/ozone/web/client/TestKeys.java | 1 + .../hadoop/ozone/web/client/TestKeysRatis.java | 2 - .../src/test/resources/log4j.properties | 5 +- .../org/apache/hadoop/ozone/TestErrorCode.java | 53 --- .../apache/hadoop/ozone/web/TestErrorCode.java | 53 +++ .../apache/hadoop/ozone/om/KeyManagerImpl.java | 32 +- .../apache/hadoop/ozone/om/OzoneManager.java | 33 +- .../hadoop/ozone/om/exceptions/OMException.java | 3 +- .../hadoop/ozone/om/TestKeyManagerImpl.java | 165 +++++++++ hadoop-ozone/ozonefs/pom.xml | 14 +- .../ozonefs/src/test/resources/log4j.properties | 1 + hadoop-ozone/pom.xml | 31 +- hadoop-ozone/tools/pom.xml | 2 +- hadoop-project-dist/pom.xml | 34 +- hadoop-project/pom.xml | 6 +- .../yarn/submarine/client/cli/CliConstants.java | 6 + .../yarn/submarine/client/cli/CliUtils.java | 10 +- .../yarn/submarine/client/cli/RunJobCli.java | 43 ++- .../client/cli/param/RunJobParameters.java | 52 ++- .../fs/DefaultRemoteDirectoryManager.java | 21 +- .../common/fs/RemoteDirectoryManager.java | 4 +- .../common/FSBasedSubmarineStorageImpl.java | 4 +- .../yarnservice/YarnServiceJobSubmitter.java | 126 ++++--- .../runtimes/yarnservice/YarnServiceUtils.java | 26 +- .../yarnservice/TestYarnServiceRunJobCli.java | 268 ++++++++++++-- .../common/fs/MockRemoteDirectoryManager.java | 7 +- .../nodemanager/LinuxContainerExecutor.java | 3 +- .../launcher/ContainerLaunch.java | 5 +- .../linux/resources/ResourceHandlerModule.java | 15 + .../runtime/DockerLinuxContainerRuntime.java | 5 +- .../linux/runtime/docker/DockerRmCommand.java | 11 +- .../impl/container-executor.c | 153 +++++++- .../impl/container-executor.h | 8 +- .../main/native/container-executor/impl/main.c | 12 +- .../test/test-container-executor.c | 147 ++++++++ .../launcher/TestContainerLaunch.java | 81 +++++ .../docker/TestDockerCommandExecutor.java | 23 +- .../runtime/docker/TestDockerRmCommand.java | 35 +- .../allocator/RegularContainerAllocator.java | 3 +- .../capacity/TestContainerAllocation.java | 68 ++++ .../site/markdown/PlacementConstraints.md.vm | 2 + .../main/webapp/app/controllers/application.js | 5 +- .../src/main/webapp/app/initializers/loader.js | 3 + .../src/main/webapp/app/routes/application.js | 4 +- pom.xml | 3 +- 192 files changed, 4906 insertions(+), 2132 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3810f7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3810f7b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
