Merge commit '4aacbfff605262aaf3dbd926258afcadc86c72c0' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d53c5997 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d53c5997 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d53c5997 Branch: refs/heads/HDFS-12943 Commit: d53c5997ac0bb08e1b108529faf428e7b7ccbb87 Parents: 066afcf 4aacbff Author: Konstantin V Shvachko <s...@apache.org> Authored: Sat Aug 25 17:15:15 2018 -0700 Committer: Konstantin V Shvachko <s...@apache.org> Committed: Sat Aug 25 17:15:15 2018 -0700 ---------------------------------------------------------------------- README.txt | 2 +- dev-support/bin/ozone-dist-layout-stitching | 2 + .../main/resources/checkstyle/suppressions.xml | 1 - .../hadoop-common/HadoopJNI.cmake | 2 + .../key/kms/LoadBalancingKMSClientProvider.java | 43 +- .../org/apache/hadoop/fs/shell/FsCommand.java | 2 +- .../java/org/apache/hadoop/fs/shell/Touch.java | 85 ---- .../apache/hadoop/fs/shell/TouchCommands.java | 198 ++++++++ .../org/apache/hadoop/util/ServletUtil.java | 2 +- .../src/site/markdown/FileSystemShell.md | 32 ++ .../kms/TestLoadBalancingKMSClientProvider.java | 181 +++++++- .../org/apache/hadoop/fs/TestFsShellTouch.java | 103 +++++ .../fs/contract/AbstractContractRenameTest.java | 4 +- .../fs/contract/AbstractContractSeekTest.java | 2 +- .../fs/contract/AbstractFSContractTestBase.java | 6 +- .../apache/hadoop/http/TestSSLHttpServer.java | 54 ++- .../apache/hadoop/test/GenericTestUtils.java | 8 +- .../src/test/resources/testConf.xml | 51 +++ .../src/main/resources/kms-default.xml | 8 + hadoop-dist/pom.xml | 17 - hadoop-dist/src/main/compose/README.md | 51 +++ .../src/main/compose/ozone/docker-compose.yaml | 6 +- .../main/compose/ozoneperf/docker-compose.yaml | 6 +- .../apache/hadoop/hdds/scm/XceiverClient.java | 5 +- .../hadoop/hdds/scm/XceiverClientGrpc.java | 5 +- .../hadoop/hdds/scm/XceiverClientRatis.java | 14 +- .../scm/client/ContainerOperationClient.java | 5 +- hadoop-hdds/common/pom.xml | 2 +- .../hadoop/hdds/scm/XceiverClientSpi.java | 6 +- .../container/common/helpers/ContainerInfo.java | 25 +- .../scm/container/common/helpers/Pipeline.java | 38 +- .../container/common/helpers/PipelineID.java | 88 ++++ .../org/apache/hadoop/utils/RocksDBStore.java | 2 +- .../main/java/org/apache/ratis/RatisHelper.java | 19 +- hadoop-hdds/common/src/main/proto/hdds.proto | 8 +- .../container/common/impl/ContainerSet.java | 3 +- .../CloseContainerCommandHandler.java | 3 +- .../common/transport/server/XceiverServer.java | 7 +- .../transport/server/XceiverServerGrpc.java | 7 +- .../transport/server/XceiverServerSpi.java | 6 +- .../server/ratis/ContainerStateMachine.java | 323 ++++++++++--- .../server/ratis/XceiverServerRatis.java | 36 +- .../container/ozoneimpl/OzoneContainer.java | 8 +- .../commands/CloseContainerCommand.java | 19 +- .../StorageContainerDatanodeProtocol.proto | 1 + .../hadoop/hdds/scm/block/BlockManagerImpl.java | 4 +- .../hadoop/hdds/scm/block/DeletedBlockLog.java | 6 +- .../hdds/scm/block/DeletedBlockLogImpl.java | 38 +- .../hdds/scm/block/PendingDeleteHandler.java | 38 ++ .../hdds/scm/block/PendingDeleteStatusList.java | 79 ++++ .../hdds/scm/block/SCMBlockDeletingService.java | 20 +- .../container/CloseContainerEventHandler.java | 2 +- .../hdds/scm/container/ContainerMapping.java | 75 ++- .../scm/container/ContainerStateManager.java | 28 +- .../scm/container/closer/ContainerCloser.java | 4 +- .../scm/container/states/ContainerStateMap.java | 18 +- .../hadoop/hdds/scm/events/SCMEvents.java | 9 + .../hdds/scm/exceptions/SCMException.java | 3 +- .../hadoop/hdds/scm/node/DatanodeInfo.java | 8 + .../hdds/scm/pipelines/PipelineManager.java | 36 +- .../hdds/scm/pipelines/PipelineSelector.java | 31 +- .../scm/pipelines/ratis/RatisManagerImpl.java | 18 +- .../standalone/StandaloneManagerImpl.java | 14 +- .../scm/server/StorageContainerManager.java | 5 + .../hadoop/hdds/scm/block/TestBlockManager.java | 25 - .../hdds/scm/block/TestDeletedBlockLog.java | 6 +- .../scm/container/TestContainerMapping.java | 50 ++ .../container/TestContainerReportHandler.java | 4 +- .../replication/TestReplicationManager.java | 5 +- .../hadoop/hdds/scm/node/TestNodeManager.java | 4 +- .../ozone/container/common/TestEndPoint.java | 2 + .../hadoop/hdfs/DistributedFileSystem.java | 6 +- .../hadoop-hdfs-native-client/pom.xml | 105 +++-- .../src/CMakeLists.txt | 48 ++ .../src/main/native/libhdfs/CMakeLists.txt | 6 +- .../main/native/libhdfspp/tests/CMakeLists.txt | 4 + .../hdfs/server/blockmanagement/BlockInfo.java | 4 + .../server/blockmanagement/BlockManager.java | 4 + .../server/blockmanagement/DatanodeManager.java | 7 +- .../hdfs/server/datanode/BPServiceActor.java | 3 +- .../hadoop/hdfs/server/datanode/DataNode.java | 15 + .../hdfs/server/datanode/DataNodeMXBean.java | 7 + .../hdfs/server/datanode/DirectoryScanner.java | 2 +- .../server/namenode/FSImageFormatPBINode.java | 28 +- .../server/namenode/FSImageFormatProtobuf.java | 9 +- .../hdfs/server/namenode/FSNamesystem.java | 10 +- .../server/namenode/FSPermissionChecker.java | 2 +- .../hdfs/server/namenode/SecondaryNameNode.java | 5 + .../namenode/SecondaryNameNodeInfoMXBean.java | 7 + .../org/apache/hadoop/hdfs/tools/DFSck.java | 6 + .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 12 +- .../src/main/native/docs/libhdfs_footer.html | 2 +- .../src/main/webapps/hdfs/explorer.js | 2 +- .../src/site/markdown/HDFSErasureCoding.md | 8 +- .../hadoop/hdfs/TestDFSClientFailover.java | 6 +- .../hadoop/hdfs/TestDistributedFileSystem.java | 7 + .../org/apache/hadoop/hdfs/TestFileAppend4.java | 6 +- .../apache/hadoop/hdfs/TestLeaseRecovery2.java | 65 +++ .../org/apache/hadoop/hdfs/TestReplication.java | 6 +- .../qjournal/client/TestIPCLoggerChannel.java | 6 +- .../server/datanode/BlockReportTestBase.java | 8 +- .../hdfs/server/datanode/DataNodeTestUtils.java | 3 + .../server/datanode/TestDataNodeMXBean.java | 47 +- .../server/datanode/TestDirectoryScanner.java | 42 +- .../hdfs/server/namenode/FSImageTestUtil.java | 13 + .../hdfs/server/namenode/TestCheckpoint.java | 4 +- .../hdfs/server/namenode/TestDeleteRace.java | 10 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 18 + .../namenode/TestINodeAttributeProvider.java | 43 +- .../TestNameNodeMetadataConsistency.java | 2 +- .../hdfs/server/namenode/TestSaveNamespace.java | 10 +- .../server/namenode/TestSecureNameNode.java | 52 ++- .../hdfs/server/namenode/ha/TestDNFencing.java | 6 +- .../namenode/ha/TestPipelinesFailover.java | 10 +- .../namenode/ha/TestStandbyCheckpoints.java | 6 +- .../security/TestRefreshUserMappings.java | 51 ++- .../test/resources/testErasureCodingConf.xml | 6 +- hadoop-mapreduce-project/pom.xml | 2 +- hadoop-ozone/docs/content/GettingStarted.md | 2 +- hadoop-ozone/docs/pom.xml | 17 + hadoop-ozone/docs/static/OzoneOverview.svg | 13 + .../hdds/scm/pipeline/TestNode2PipelineMap.java | 8 +- .../hdds/scm/pipeline/TestPipelineClose.java | 10 +- .../hadoop/ozone/TestMiniOzoneCluster.java | 3 +- .../ozone/container/ContainerTestHelper.java | 5 +- .../commandhandler/TestBlockDeletion.java | 57 ++- .../TestCloseContainerByPipeline.java | 41 +- .../TestCloseContainerHandler.java | 7 +- .../server/TestContainerStateMachine.java | 201 ++++++++ .../hadoop/ozone/om/TestOzoneManager.java | 11 +- .../apache/hadoop/ozone/om/package-info.java | 22 + .../apache/hadoop/ozone/om/KeyManagerImpl.java | 37 +- .../web/ozShell/volume/UpdateVolumeHandler.java | 5 + .../ITestOzoneContractGetFileStatus.java | 2 +- hadoop-ozone/pom.xml | 10 - .../genesis/BenchMarkContainerStateMap.java | 13 +- .../genesis/BenchMarkDatanodeDispatcher.java | 3 +- hadoop-project/pom.xml | 4 +- .../fs/aliyun/oss/AliyunOSSInputStream.java | 6 +- .../apache/hadoop/tools/HadoopArchiveLogs.java | 6 +- .../hadoop/tools/HadoopArchiveLogsRunner.java | 8 +- .../org/apache/hadoop/tools/HadoopArchives.java | 6 +- .../hadoop/fs/s3a/s3guard/DDBPathMetadata.java | 77 ++++ .../fs/s3a/s3guard/DynamoDBMetadataStore.java | 130 ++++-- .../PathMetadataDynamoDBTranslation.java | 71 ++- .../apache/hadoop/fs/s3a/s3guard/S3Guard.java | 4 + .../site/markdown/tools/hadoop-aws/s3guard.md | 5 +- .../site/markdown/tools/hadoop-aws/testing.md | 157 +++++++ .../s3a/ITestS3AContractGetFileStatus.java | 2 +- .../ITestS3AContractGetFileStatusV1List.java | 2 +- .../s3guard/AbstractS3GuardToolTestBase.java | 13 +- .../s3a/s3guard/ITestS3GuardToolDynamoDB.java | 7 +- .../fs/s3a/s3guard/ITestS3GuardToolLocal.java | 11 + .../fs/s3a/s3guard/MetadataStoreTestBase.java | 49 ++ .../TestPathMetadataDynamoDBTranslation.java | 47 +- .../hadoop/contrib/utils/join/JobBase.java | 6 +- .../org/apache/hadoop/tools/CopyListing.java | 6 +- .../java/org/apache/hadoop/tools/DistCp.java | 6 +- .../apache/hadoop/tools/GlobbedCopyListing.java | 6 +- .../org/apache/hadoop/tools/OptionsParser.java | 6 +- .../apache/hadoop/tools/RegexCopyFilter.java | 8 +- .../apache/hadoop/tools/SimpleCopyListing.java | 10 +- .../apache/hadoop/tools/mapred/CopyMapper.java | 6 +- .../tools/mapred/RetriableFileCopyCommand.java | 8 +- .../tools/mapred/UniformSizeInputFormat.java | 8 +- .../tools/mapred/lib/DynamicInputChunk.java | 8 +- .../mapred/lib/DynamicInputChunkContext.java | 6 +- .../tools/mapred/lib/DynamicInputFormat.java | 6 +- .../tools/mapred/lib/DynamicRecordReader.java | 6 +- .../apache/hadoop/tools/util/DistCpUtils.java | 6 +- .../hadoop/tools/util/ProducerConsumer.java | 6 +- .../hadoop/tools/util/RetriableCommand.java | 6 +- .../apache/hadoop/tools/TestCopyListing.java | 6 +- .../apache/hadoop/tools/TestDistCpSystem.java | 8 +- .../apache/hadoop/tools/TestDistCpViewFs.java | 6 +- .../apache/hadoop/tools/TestExternalCall.java | 6 +- .../hadoop/tools/TestFileBasedCopyListing.java | 6 +- .../apache/hadoop/tools/TestIntegration.java | 6 +- .../contract/AbstractContractDistCpTest.java | 2 +- .../hadoop/tools/mapred/TestCopyCommitter.java | 6 +- .../hadoop/tools/mapred/TestCopyMapper.java | 6 +- .../tools/mapred/TestCopyOutputFormat.java | 6 +- .../mapred/lib/TestDynamicInputFormat.java | 6 +- .../hadoop/tools/util/TestDistCpUtils.java | 6 +- .../tools/util/TestThrottledInputStream.java | 8 +- .../java/org/apache/hadoop/tools/DistTool.java | 6 +- .../mapred/gridmix/ClusterSummarizer.java | 6 +- .../gridmix/CompressionEmulationUtil.java | 6 +- .../gridmix/DistributedCacheEmulator.java | 8 +- .../hadoop/mapred/gridmix/EchoUserResolver.java | 6 +- .../mapred/gridmix/ExecutionSummarizer.java | 6 +- .../apache/hadoop/mapred/gridmix/FilePool.java | 6 +- .../hadoop/mapred/gridmix/GridmixJob.java | 6 +- .../hadoop/mapred/gridmix/InputStriper.java | 6 +- .../hadoop/mapred/gridmix/JobFactory.java | 6 +- .../hadoop/mapred/gridmix/JobMonitor.java | 6 +- .../hadoop/mapred/gridmix/JobSubmitter.java | 6 +- .../apache/hadoop/mapred/gridmix/LoadJob.java | 6 +- .../mapred/gridmix/RandomTextDataGenerator.java | 6 +- .../hadoop/mapred/gridmix/ReplayJobFactory.java | 6 +- .../mapred/gridmix/RoundRobinUserResolver.java | 6 +- .../hadoop/mapred/gridmix/SerialJobFactory.java | 6 +- .../apache/hadoop/mapred/gridmix/SleepJob.java | 6 +- .../hadoop/mapred/gridmix/Statistics.java | 6 +- .../hadoop/mapred/gridmix/StressJobFactory.java | 6 +- .../mapred/gridmix/SubmitterUserResolver.java | 6 +- .../hadoop/mapred/gridmix/CommonJobTest.java | 6 +- .../hadoop/mapred/gridmix/DebugJobProducer.java | 6 +- .../hadoop/mapred/gridmix/GridmixTestUtils.java | 6 +- .../hadoop/mapred/gridmix/TestFilePool.java | 6 +- .../hadoop/mapred/gridmix/TestFileQueue.java | 6 +- .../mapred/gridmix/TestGridMixClasses.java | 6 +- .../mapred/gridmix/TestGridmixRecord.java | 6 +- .../mapred/gridmix/TestRecordFactory.java | 6 +- .../tools/rumen/DeskewedJobTraceReader.java | 8 +- .../org/apache/hadoop/tools/rumen/Folder.java | 6 +- .../hadoop/tools/rumen/HadoopLogsAnalyzer.java | 6 +- .../hadoop/tools/rumen/HistoryEventEmitter.java | 6 +- .../apache/hadoop/tools/rumen/JobBuilder.java | 6 +- .../apache/hadoop/tools/rumen/ParsedJob.java | 6 +- .../apache/hadoop/tools/rumen/ParsedTask.java | 6 +- .../hadoop/tools/rumen/ParsedTaskAttempt.java | 6 +- .../hadoop/tools/rumen/RandomSeedGenerator.java | 6 +- .../apache/hadoop/tools/rumen/TraceBuilder.java | 8 +- .../apache/hadoop/tools/rumen/ZombieJob.java | 6 +- .../sls/resourcemanager/MockAMLauncher.java | 6 +- .../hadoop/yarn/sls/synthetic/SynthJob.java | 6 +- .../sls/synthetic/SynthTraceJobProducer.java | 6 +- .../org/apache/hadoop/streaming/PipeMapRed.java | 17 +- .../streaming/StreamBaseRecordReader.java | 5 +- .../org/apache/hadoop/streaming/StreamJob.java | 6 +- .../mapreduce/StreamBaseRecordReader.java | 8 +- .../streaming/TestMultipleArchiveFiles.java | 6 +- .../streaming/TestStreamXmlMultipleRecords.java | 6 +- .../streaming/TestStreamingBadRecords.java | 8 +- .../org/apache/hadoop/streaming/UtilTest.java | 6 +- .../hadoop/yarn/conf/YarnConfiguration.java | 6 +- .../hadoop-yarn-services-api/pom.xml | 57 +++ .../yarn/service/client/ApiServiceClient.java | 83 +++- .../client/TestSecureApiServiceClient.java | 83 ++++ .../yarn/service/component/Component.java | 3 +- .../component/instance/ComponentInstance.java | 2 +- .../containerlaunch/AbstractLauncher.java | 5 + .../containerlaunch/ContainerLaunchService.java | 11 + .../provider/AbstractProviderService.java | 5 +- .../defaultImpl/DefaultProviderService.java | 4 +- .../provider/docker/DockerProviderService.java | 30 +- .../tarball/TarballProviderService.java | 7 +- .../instance/TestComponentInstance.java | 4 +- .../provider/TestAbstractProviderService.java | 138 ++++++ .../hadoop-yarn-submarine/README.md | 53 +++ .../hadoop-yarn-submarine/pom.xml | 213 +++++++++ .../yarn/submarine/client/cli/AbstractCli.java | 47 ++ .../hadoop/yarn/submarine/client/cli/Cli.java | 106 +++++ .../yarn/submarine/client/cli/CliConstants.java | 48 ++ .../yarn/submarine/client/cli/CliUtils.java | 174 +++++++ .../yarn/submarine/client/cli/RunJobCli.java | 204 +++++++++ .../yarn/submarine/client/cli/ShowJobCli.java | 125 +++++ .../client/cli/param/BaseParameters.java | 56 +++ .../client/cli/param/RunJobParameters.java | 222 +++++++++ .../client/cli/param/RunParameters.java | 103 +++++ .../client/cli/param/ShowJobParameters.java | 18 + .../yarn/submarine/common/ClientContext.java | 77 ++++ .../hadoop/yarn/submarine/common/Envs.java | 27 ++ .../common/api/JobComponentStatus.java | 73 +++ .../yarn/submarine/common/api/JobState.java | 52 +++ .../yarn/submarine/common/api/JobStatus.java | 87 ++++ .../yarn/submarine/common/api/TaskType.java | 32 ++ .../api/builder/JobComponentStatusBuilder.java | 44 ++ .../common/api/builder/JobStatusBuilder.java | 64 +++ .../common/conf/SubmarineConfiguration.java | 51 +++ .../submarine/common/conf/SubmarineLogs.java | 31 ++ .../common/exception/SubmarineException.java | 21 + .../exception/SubmarineRuntimeException.java | 25 + .../fs/DefaultRemoteDirectoryManager.java | 84 ++++ .../common/fs/RemoteDirectoryManager.java | 30 ++ .../yarn/submarine/runtimes/RuntimeFactory.java | 106 +++++ .../common/FSBasedSubmarineStorageImpl.java | 106 +++++ .../submarine/runtimes/common/JobMonitor.java | 84 ++++ .../submarine/runtimes/common/JobSubmitter.java | 36 ++ .../runtimes/common/StorageKeyConstants.java | 24 + .../runtimes/common/SubmarineStorage.java | 57 +++ .../yarnservice/YarnServiceJobMonitor.java | 46 ++ .../yarnservice/YarnServiceJobSubmitter.java | 458 +++++++++++++++++++ .../yarnservice/YarnServiceRuntimeFactory.java | 44 ++ .../runtimes/yarnservice/YarnServiceUtils.java | 78 ++++ .../src/site/DeveloperGuide.md | 26 ++ .../src/site/QuickStart.md | 134 ++++++ .../client/cli/TestRunJobCliParsing.java | 229 ++++++++++ .../client/cli/TestShowJobCliParsing.java | 104 +++++ .../yarnservice/TestYarnServiceRunJobCli.java | 167 +++++++ .../yarnservice/YarnServiceCliTestUtils.java | 35 ++ .../submarine/common/MockClientContext.java | 56 +++ .../common/fs/MockRemoteDirectoryManager.java | 83 ++++ .../runtimes/common/MemorySubmarineStorage.java | 74 +++ .../common/TestFSBasedSubmarineStorage.java | 73 +++ .../yarnservice/TestTFConfigGenerator.java | 42 ++ .../src/test/resources/core-site.xml | 21 + .../src/test/resources/hdfs-site.xml | 21 + .../hadoop-yarn-applications/pom.xml | 1 + .../logaggregation/AggregatedLogFormat.java | 3 +- .../filecontroller/LogAggregationHtmlBlock.java | 61 +++ .../ifile/IndexedFileAggregatedLogsBlock.java | 136 ++---- .../tfile/TFileAggregatedLogsBlock.java | 56 +-- .../server/nodemanager/ContainerExecutor.java | 62 ++- .../yarn/server/nodemanager/NodeManager.java | 20 +- .../server/nodemanager/NodeManagerMXBean.java | 38 ++ .../launcher/ContainerLaunch.java | 51 ++- .../runtime/DockerLinuxContainerRuntime.java | 4 +- .../impl/container-executor.c | 92 ++-- .../impl/container-executor.h | 3 +- .../main/native/container-executor/impl/main.c | 26 +- .../nodemanager/TestContainerExecutor.java | 31 +- .../nodemanager/TestNodeManagerMXBean.java | 56 +++ .../runtime/TestDockerContainerRuntime.java | 8 +- .../server/resourcemanager/ResourceManager.java | 18 +- .../resourcemanager/ResourceManagerMXBean.java | 38 ++ .../TestResourceManagerMXBean.java | 56 +++ .../TestAMLaunchFailure.java | 2 +- .../TestSchedulerNegotiator.java | 2 +- .../TestTimelineServiceClientIntegration.java | 3 +- .../security/TestTimelineAuthFilterForV2.java | 2 +- .../PerNodeTimelineCollectorsAuxService.java | 13 +- 323 files changed, 8429 insertions(+), 1311 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d53c5997/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d53c5997/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org