Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53201734 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53201734 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53201734 Branch: refs/heads/HDFS-12943 Commit: 53201734f5d888b892118a3f8d873ac01c209de4 Parents: 191faeb 8aa6c4f Author: Erik Krogen <xkro...@apache.org> Authored: Fri Aug 31 09:06:54 2018 -0700 Committer: Erik Krogen <xkro...@apache.org> Committed: Fri Aug 31 09:06:54 2018 -0700 ---------------------------------------------------------------------- dev-support/bin/ozone-dist-layout-stitching | 6 +- dev-support/bin/ozone-dist-tar-stitching | 9 +- .../hadoop/fs/FileSystemMultipartUploader.java | 6 +- .../org/apache/hadoop/fs/MultipartUploader.java | 11 + .../apache/hadoop/fs/TrashPolicyDefault.java | 14 + .../src/site/markdown/Compatibility.md | 2 +- .../site/markdown/InterfaceClassification.md | 2 +- .../java/org/apache/hadoop/fs/TestTrash.java | 54 + .../AbstractContractMultipartUploaderTest.java | 43 + .../crypto/key/kms/server/KMSConfiguration.java | 31 + .../hadoop/crypto/key/kms/server/KMSWebApp.java | 38 +- .../crypto/key/kms/server/KMSWebServer.java | 1 + .../apache/hadoop/hdds/scm/XceiverClient.java | 6 +- .../hadoop/hdds/scm/XceiverClientGrpc.java | 8 +- .../hadoop/hdds/scm/XceiverClientManager.java | 8 +- .../scm/client/ContainerOperationClient.java | 9 + .../hdds/scm/storage/ChunkInputStream.java | 7 +- .../hdds/scm/storage/ChunkOutputStream.java | 44 +- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 6 + .../org/apache/hadoop/hdds/client/BlockID.java | 5 +- .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 - .../hadoop/hdds/scm/XceiverClientSpi.java | 2 - .../hadoop/hdds/scm/client/ScmClient.java | 9 +- .../common/helpers/AllocatedBlock.java | 4 +- .../container/common/helpers/ContainerInfo.java | 12 +- .../common/helpers/ContainerWithPipeline.java | 7 +- .../scm/container/common/helpers/Pipeline.java | 11 +- .../StorageContainerLocationProtocol.java | 6 +- ...rLocationProtocolClientSideTranslatorPB.java | 21 +- .../scm/storage/ContainerProtocolCalls.java | 6 +- .../apache/hadoop/ozone/OzoneConfigKeys.java | 7 - .../org/apache/hadoop/ozone/OzoneConsts.java | 8 +- .../apache/hadoop/ozone/audit/AuditLogger.java | 66 +- .../apache/hadoop/ozone/audit/AuditMessage.java | 64 + .../apache/hadoop/ozone/audit/package-info.java | 19 +- .../ozone/container/common/helpers/KeyData.java | 8 +- .../apache/hadoop/utils/HddsVersionInfo.java | 6 +- .../hadoop/utils/db/DBConfigFromFile.java | 134 +++ .../org/apache/hadoop/utils/db/DBProfile.java | 120 ++ .../apache/hadoop/utils/db/DBStoreBuilder.java | 201 ++++ .../org/apache/hadoop/utils/db/RDBStore.java | 32 +- .../org/apache/hadoop/utils/db/TableConfig.java | 93 ++ .../common/src/main/resources/ozone-default.xml | 40 +- .../ozone/audit/TestOzoneAuditLogger.java | 124 +- .../apache/hadoop/utils/TestMetadataStore.java | 1 - .../hadoop/utils/db/TestDBConfigFromFile.java | 116 ++ .../hadoop/utils/db/TestDBStoreBuilder.java | 174 +++ .../apache/hadoop/utils/db/TestRDBStore.java | 17 +- .../hadoop/utils/db/TestRDBTableStore.java | 11 +- .../common/src/test/resources/test.db.ini | 145 +++ .../hadoop/ozone/HddsDatanodeService.java | 3 +- .../common/helpers/ContainerUtils.java | 22 +- .../container/common/impl/ContainerData.java | 24 +- .../common/impl/ContainerDataYaml.java | 5 +- .../container/common/impl/ContainerSet.java | 2 +- .../container/common/impl/HddsDispatcher.java | 6 +- .../common/impl/OpenContainerBlockMap.java | 19 +- .../transport/server/GrpcXceiverService.java | 8 +- .../transport/server/XceiverServerGrpc.java | 2 +- .../transport/server/ratis/CSMMetrics.java | 115 ++ .../server/ratis/ContainerStateMachine.java | 33 + .../server/ratis/XceiverServerRatis.java | 6 +- .../container/keyvalue/KeyValueContainer.java | 2 +- .../keyvalue/KeyValueContainerData.java | 10 +- .../container/keyvalue/KeyValueHandler.java | 15 +- .../keyvalue/interfaces/KeyManager.java | 4 +- .../container/ozoneimpl/OzoneContainer.java | 11 +- .../ozone/protocol/commands/CommandStatus.java | 16 +- .../ozone/container/common/ScmTestMock.java | 6 +- .../common/TestKeyValueContainerData.java | 5 +- .../common/impl/TestContainerDataYaml.java | 7 +- .../container/common/impl/TestContainerSet.java | 7 +- .../common/impl/TestHddsDispatcher.java | 3 +- .../common/interfaces/TestHandler.java | 7 - .../endpoint/TestHeartbeatEndpointTask.java | 2 - .../TestRoundRobinVolumeChoosingPolicy.java | 5 +- .../keyvalue/TestChunkManagerImpl.java | 4 +- .../container/keyvalue/TestKeyManagerImpl.java | 4 +- .../keyvalue/TestKeyValueBlockIterator.java | 4 +- .../keyvalue/TestKeyValueContainer.java | 13 +- .../container/keyvalue/TestKeyValueHandler.java | 4 +- .../container/ozoneimpl/TestOzoneContainer.java | 8 +- .../test/resources/additionalfields.container | 4 +- .../test/resources/incorrect.checksum.container | 2 +- .../src/test/resources/incorrect.container | 2 +- .../hadoop/hdds/server/events/EventWatcher.java | 6 +- .../hdds/server/events/TestEventQueue.java | 3 - hadoop-hdds/pom.xml | 1 + .../hadoop/hdds/scm/block/BlockManagerImpl.java | 18 +- .../hdds/scm/block/DeletedBlockLogImpl.java | 3 +- .../hdds/scm/block/SCMBlockDeletingService.java | 4 +- .../container/CloseContainerEventHandler.java | 4 +- .../hdds/scm/container/ContainerMapping.java | 4 +- .../scm/container/ContainerReportHandler.java | 4 + .../scm/container/ContainerStateManager.java | 7 +- .../replication/ReplicationManager.java | 2 +- .../scm/container/states/ContainerStateMap.java | 2 +- .../hdds/scm/node/states/Node2ContainerMap.java | 4 +- .../scm/node/states/NodeNotFoundException.java | 2 - .../hdds/scm/node/states/ReportResult.java | 3 +- .../hdds/scm/pipelines/Node2PipelineMap.java | 50 +- .../hdds/scm/pipelines/PipelineManager.java | 6 +- .../hdds/scm/pipelines/PipelineSelector.java | 7 +- .../scm/server/SCMClientProtocolServer.java | 3 +- .../org/apache/hadoop/hdds/scm/TestUtils.java | 8 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 1 - .../hdds/scm/block/TestDeletedBlockLog.java | 7 +- .../command/TestCommandStatusReportHandler.java | 22 +- .../TestCloseContainerEventHandler.java | 1 - .../scm/container/TestContainerMapping.java | 7 +- .../container/TestContainerReportHandler.java | 42 +- .../TestSCMContainerPlacementCapacity.java | 8 +- .../TestSCMContainerPlacementRandom.java | 4 +- .../replication/TestReplicationManager.java | 11 +- .../replication/TestReplicationQueue.java | 4 +- .../hdds/scm/node/TestContainerPlacement.java | 5 +- .../hadoop/hdds/scm/node/TestNodeManager.java | 3 +- .../hdds/scm/node/TestNodeReportHandler.java | 3 +- .../ozone/container/common/TestEndPoint.java | 9 +- .../placement/TestContainerPlacement.java | 6 +- .../main/native/fuse-dfs/test/TestFuseDFS.java | 6 +- .../hadoop/hdfs/nfs/mount/RpcProgramMountd.java | 7 +- .../hadoop/hdfs/nfs/nfs3/AsyncDataService.java | 6 +- .../hadoop/hdfs/nfs/nfs3/OpenFileCtx.java | 4 +- .../hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java | 13 +- .../nfs/nfs3/PrivilegedNfsGatewayStarter.java | 7 +- .../apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java | 6 +- .../hadoop/hdfs/nfs/nfs3/WriteManager.java | 6 +- .../org/apache/hadoop/hdfs/nfs/TestMountd.java | 6 +- .../hadoop/hdfs/nfs/TestOutOfOrderWrite.java | 9 +- .../metrics/FederationRPCMetrics.java | 13 +- .../FederationRPCPerformanceMonitor.java | 15 +- .../federation/router/ConnectionManager.java | 18 +- .../server/federation/router/RBFConfigKeys.java | 5 + .../router/RouterPermissionChecker.java | 7 +- .../server/federation/store/RecordStore.java | 6 +- .../hdfs/tools/federation/RouterAdmin.java | 98 +- .../src/main/resources/hdfs-rbf-default.xml | 8 + .../federation/router/TestRouterAdminCLI.java | 82 ++ .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 + .../hdfs/server/blockmanagement/BlockInfo.java | 2 +- .../server/blockmanagement/BlockManager.java | 4 + .../datanode/fsdataset/impl/FsDatasetImpl.java | 3 + .../hdfs/server/namenode/FSNamesystem.java | 13 +- .../hdfs/server/namenode/NamenodeFsck.java | 5 +- .../server/namenode/ReencryptionHandler.java | 6 +- .../server/namenode/ReencryptionUpdater.java | 52 +- .../src/main/resources/hdfs-default.xml | 10 + .../hadoop/hdfs/TestDistributedFileSystem.java | 5 +- .../namenode/TestLargeDirectoryDelete.java | 2 +- .../lib/output/PathOutputCommitter.java | 12 +- .../hadoop/mapreduce/task/reduce/Shuffle.java | 24 +- .../task/reduce/ShuffleClientMetrics.java | 43 +- .../task/reduce/TestShuffleClientMetrics.java | 75 ++ .../dev-support/bin/robot-all.sh | 2 +- .../dev-support/bin/robot-dnd-all.sh | 10 +- .../acceptance-test/dev-support/bin/robot.sh | 7 +- hadoop-ozone/acceptance-test/pom.xml | 7 +- .../src/test/acceptance/basic/ozone-shell.robot | 1 - .../src/test/acceptance/commonlib.robot | 2 +- .../apache/hadoop/ozone/client/ObjectStore.java | 7 +- .../ozone/client/io/ChunkGroupOutputStream.java | 207 +++- hadoop-ozone/common/pom.xml | 5 +- .../hadoop/ozone/om/helpers/OmKeyInfo.java | 23 +- hadoop-ozone/docs/content/GettingStarted.md | 12 +- .../hdds/scm/pipeline/TestPipelineClose.java | 4 - .../apache/hadoop/ozone/MiniOzoneCluster.java | 3 +- .../hadoop/ozone/MiniOzoneClusterImpl.java | 13 +- .../apache/hadoop/ozone/RatisTestHelper.java | 16 +- .../TestStorageContainerManagerHelper.java | 2 - .../ozone/client/rest/TestOzoneRestClient.java | 7 +- .../rpc/TestCloseContainerHandlingByClient.java | 405 +++++++ .../ozone/client/rpc/TestOzoneRpcClient.java | 9 +- .../ozone/container/ContainerTestHelper.java | 37 +- .../common/TestBlockDeletingService.java | 2 +- .../TestContainerDeletionChoosingPolicy.java | 16 +- .../common/impl/TestContainerPersistence.java | 120 +- .../commandhandler/TestBlockDeletion.java | 17 +- .../TestCloseContainerByPipeline.java | 35 +- .../transport/server/ratis/TestCSMMetrics.java | 202 ++++ .../container/ozoneimpl/TestOzoneContainer.java | 53 +- .../ozoneimpl/TestOzoneContainerRatis.java | 5 +- .../container/ozoneimpl/TestRatisManager.java | 6 +- .../hadoop/ozone/freon/TestDataValidate.java | 7 +- .../apache/hadoop/ozone/freon/TestFreon.java | 3 +- .../ozone/om/TestContainerReportWithKeys.java | 12 +- .../om/TestMultipleContainerReadWrite.java | 5 +- .../hadoop/ozone/om/TestOmBlockVersioning.java | 23 +- .../apache/hadoop/ozone/om/TestOmMetrics.java | 7 +- .../apache/hadoop/ozone/om/TestOmSQLCli.java | 6 +- .../hadoop/ozone/om/TestOzoneManager.java | 5 +- .../hadoop/ozone/ozShell/TestOzoneShell.java | 20 +- .../hadoop/ozone/scm/TestAllocateContainer.java | 2 - .../ozone/scm/TestXceiverClientManager.java | 25 - .../ozone/web/TestDistributedOzoneVolumes.java | 188 --- .../hadoop/ozone/web/TestLocalOzoneVolumes.java | 187 --- .../hadoop/ozone/web/TestOzoneVolumes.java | 183 +++ .../hadoop/ozone/web/TestOzoneWebAccess.java | 11 +- .../hadoop/ozone/web/client/TestBuckets.java | 14 +- .../ozone/web/client/TestBucketsRatis.java | 2 +- .../hadoop/ozone/web/client/TestKeys.java | 27 +- .../hadoop/ozone/web/client/TestKeysRatis.java | 6 +- .../ozone/web/client/TestOzoneClient.java | 3 - .../hadoop/ozone/web/client/TestVolume.java | 11 +- .../ozone/web/client/TestVolumeRatis.java | 3 - .../server/datanode/ObjectStoreHandler.java | 182 ++- .../web/handlers/StorageHandlerBuilder.java | 18 +- .../web/localstorage/LocalStorageHandler.java | 385 ------ .../web/localstorage/OzoneMetadataManager.java | 1138 ------------------ .../apache/hadoop/ozone/om/KeyManagerImpl.java | 13 - .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 2 - .../hadoop/fs/ozone/TestOzoneFSInputStream.java | 6 +- hadoop-ozone/pom.xml | 5 + .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 4 - hadoop-project/pom.xml | 2 +- .../java/org/apache/hadoop/fs/s3a/Invoker.java | 15 +- .../hadoop/fs/s3a/S3AMultipartUploader.java | 10 +- .../fs/s3a/commit/AbstractS3ACommitter.java | 16 +- .../fs/s3a/commit/S3ACommitterFactory.java | 18 +- .../s3a/commit/magic/MagicS3GuardCommitter.java | 7 + .../staging/DirectoryStagingCommitter.java | 8 +- .../staging/PartitionedStagingCommitter.java | 9 +- .../hadoop/fs/s3a/commit/staging/Paths.java | 14 +- .../fs/s3a/commit/staging/StagingCommitter.java | 50 +- .../tools/hadoop-aws/committer_architecture.md | 94 +- .../markdown/tools/hadoop-aws/committers.md | 2 +- .../fs/s3a/commit/AbstractCommitITest.java | 19 + .../fs/s3a/commit/AbstractITCommitMRJob.java | 5 +- .../fs/s3a/commit/AbstractITCommitProtocol.java | 63 +- .../fs/s3a/commit/ITestS3ACommitterFactory.java | 200 +++ .../fs/s3a/commit/magic/ITMagicCommitMRJob.java | 6 +- .../commit/magic/ITestMagicCommitProtocol.java | 25 +- .../ITStagingCommitMRJobBadDest.java | 62 + .../integration/ITestStagingCommitProtocol.java | 13 + ...estNativeAzureFileSystemConcurrencyLive.java | 2 +- .../hadoop/yarn/conf/YarnConfiguration.java | 5 + .../hadoop/yarn/service/ServiceScheduler.java | 100 +- .../service/api/records/ComponentState.java | 2 +- .../service/api/records/ContainerState.java | 3 +- .../yarn/service/api/records/ServiceState.java | 2 +- .../component/instance/ComponentInstance.java | 144 ++- .../timelineservice/ServiceTimelineEvent.java | 5 +- .../ServiceTimelinePublisher.java | 33 +- .../yarn/service/MockRunningServiceContext.java | 18 +- .../hadoop/yarn/service/ServiceTestUtils.java | 9 +- .../yarn/service/component/TestComponent.java | 55 +- .../component/TestComponentRestartPolicy.java | 1 - .../instance/TestComponentInstance.java | 35 +- .../TestServiceTimelinePublisher.java | 4 +- .../src/main/resources/yarn-default.xml | 7 + .../yarn/server/AMHeartbeatRequestHandler.java | 227 ++++ .../LocalityMulticastAMRMProxyPolicy.java | 105 +- .../server/uam/UnmanagedApplicationManager.java | 170 +-- .../TestLocalityMulticastAMRMProxyPolicy.java | 53 +- .../amrmproxy/FederationInterceptor.java | 245 ++-- .../runtime/DockerLinuxContainerRuntime.java | 87 +- .../linux/runtime/docker/DockerRunCommand.java | 5 + .../container-executor/impl/utils/docker-util.c | 42 + .../container-executor/impl/utils/docker-util.h | 3 +- .../test/utils/test_docker_util.cc | 64 + .../containermanager/TestContainerManager.java | 18 +- .../runtime/TestDockerContainerRuntime.java | 149 +++ .../runtime/docker/TestDockerRunCommand.java | 5 +- .../scheduler/capacity/CapacityScheduler.java | 2 +- .../src/site/markdown/DockerContainers.md | 1 + .../site/markdown/OpportunisticContainers.md.vm | 2 +- 266 files changed, 5288 insertions(+), 3716 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/53201734/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/53201734/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/53201734/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org