Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb75861e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb75861e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb75861e Branch: refs/heads/HDFS-12943 Commit: eb75861eab63e1a56419b954ea12bf1b6f3bf8f7 Parents: 967aab6 cdf5d58 Author: Konstantin V Shvachko <s...@apache.org> Authored: Fri Oct 5 15:01:08 2018 -0700 Committer: Konstantin V Shvachko <s...@apache.org> Committed: Fri Oct 5 15:01:08 2018 -0700 ---------------------------------------------------------------------- hadoop-assemblies/pom.xml | 4 +- hadoop-build-tools/pom.xml | 2 +- hadoop-client-modules/hadoop-client-api/pom.xml | 4 +- .../hadoop-client-check-invariants/pom.xml | 4 +- .../hadoop-client-check-test-invariants/pom.xml | 4 +- .../hadoop-client-integration-tests/pom.xml | 4 +- .../hadoop-client-minicluster/pom.xml | 4 +- .../hadoop-client-runtime/pom.xml | 4 +- hadoop-client-modules/hadoop-client/pom.xml | 4 +- hadoop-client-modules/pom.xml | 2 +- .../hadoop-cloud-storage/pom.xml | 4 +- hadoop-cloud-storage-project/pom.xml | 4 +- .../hadoop-annotations/pom.xml | 4 +- .../hadoop-auth-examples/pom.xml | 4 +- hadoop-common-project/hadoop-auth/pom.xml | 4 +- hadoop-common-project/hadoop-common/pom.xml | 83 +- .../hadoop/fs/FileSystemMultipartUploader.java | 25 +- .../apache/hadoop/security/SecurityUtil.java | 10 +- .../io/compress/zstd/ZStandardDecompressor.c | 2 +- .../src/main/resources/core-default.xml | 10 + .../src/site/markdown/SecureMode.md | 2 +- .../AbstractContractMultipartUploaderTest.java | 22 + .../hadoop/fs/sftp/TestSFTPFileSystem.java | 3 + hadoop-common-project/hadoop-kms/pom.xml | 4 +- .../crypto/key/kms/server/KMSJSONReader.java | 4 +- hadoop-common-project/hadoop-minikdc/pom.xml | 4 +- hadoop-common-project/hadoop-nfs/pom.xml | 4 +- hadoop-common-project/pom.xml | 4 +- hadoop-dist/pom.xml | 4 +- .../hadoop/hdds/scm/XceiverClientGrpc.java | 23 - .../hadoop/hdds/scm/XceiverClientRatis.java | 32 +- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 + .../org/apache/hadoop/hdds/cli/GenericCli.java | 2 +- .../hadoop/hdds/scm/XceiverClientSpi.java | 10 +- .../scm/container/common/helpers/Pipeline.java | 4 + .../StorageContainerLocationProtocol.java | 16 + ...rLocationProtocolClientSideTranslatorPB.java | 42 + .../apache/hadoop/ozone/OzoneConfigKeys.java | 11 - .../org/apache/hadoop/ozone/OzoneConsts.java | 1 + .../apache/hadoop/ozone/audit/AuditLogger.java | 44 +- .../apache/hadoop/ozone/audit/AuditMessage.java | 77 +- .../org/apache/hadoop/ozone/audit/Auditor.java | 33 + .../apache/hadoop/ozone/audit/package-info.java | 16 +- ...rLocationProtocolServerSideTranslatorPB.java | 32 + .../StorageContainerLocationProtocol.proto | 27 + .../common/src/main/resources/ozone-default.xml | 87 +- .../ozone/audit/TestOzoneAuditLogger.java | 89 +- .../common/src/test/resources/log4j2.properties | 4 +- .../report/CommandStatusReportPublisher.java | 2 +- .../common/statemachine/StateContext.java | 58 +- .../CloseContainerCommandHandler.java | 14 +- .../commandhandler/CommandHandler.java | 7 +- .../DeleteBlocksCommandHandler.java | 43 +- .../ReplicateContainerCommandHandler.java | 3 +- .../states/endpoint/HeartbeatEndpointTask.java | 33 +- .../states/endpoint/VersionEndpointTask.java | 44 +- .../container/common/utils/HddsVolumeUtil.java | 2 +- .../container/common/volume/VolumeSet.java | 178 ++-- .../container/keyvalue/KeyValueContainer.java | 6 +- .../container/keyvalue/KeyValueHandler.java | 4 +- .../container/ozoneimpl/ContainerReader.java | 20 + .../container/ozoneimpl/OzoneContainer.java | 9 - .../StorageContainerDatanodeProtocol.java | 13 - .../protocol/commands/CommandForDatanode.java | 8 +- .../ozone/protocol/commands/CommandStatus.java | 41 +- .../commands/DeleteBlockCommandStatus.java | 92 ++ ...rDatanodeProtocolClientSideTranslatorPB.java | 17 - ...rDatanodeProtocolServerSideTranslatorPB.java | 16 - .../StorageContainerDatanodeProtocol.proto | 11 +- .../ozone/container/common/ScmTestMock.java | 24 +- .../common/report/TestReportPublisher.java | 4 +- .../endpoint/TestHeartbeatEndpointTask.java | 12 +- hadoop-hdds/pom.xml | 2 +- .../hadoop/hdds/scm/block/BlockManagerImpl.java | 13 +- .../block/DatanodeDeletedBlockTransactions.java | 30 +- .../hdds/scm/block/DeletedBlockLogImpl.java | 32 +- .../hdds/scm/block/SCMBlockDeletingService.java | 17 +- .../scm/command/CommandStatusReportHandler.java | 10 +- .../container/CloseContainerEventHandler.java | 4 +- .../scm/container/CloseContainerWatcher.java | 4 +- .../hdds/scm/container/ContainerManager.java | 142 +++ .../hdds/scm/container/ContainerMapping.java | 698 ------------- .../scm/container/ContainerReportHandler.java | 28 +- .../scm/container/ContainerStateManager.java | 51 +- .../hadoop/hdds/scm/container/Mapping.java | 141 --- .../hdds/scm/container/SCMContainerManager.java | 731 ++++++++++++++ .../placement/algorithms/SCMCommonPolicy.java | 4 +- .../placement/metrics/SCMNodeMetric.java | 1 + .../hadoop/hdds/scm/events/SCMEvents.java | 19 +- .../hadoop/hdds/scm/node/DeadNodeHandler.java | 30 +- .../hadoop/hdds/scm/node/NewNodeHandler.java | 9 +- .../hadoop/hdds/scm/node/NodeManager.java | 91 +- .../hadoop/hdds/scm/node/NodeManagerMXBean.java | 19 - .../hadoop/hdds/scm/node/NodeStateManager.java | 142 ++- .../hadoop/hdds/scm/node/SCMNodeManager.java | 243 +++-- .../hdds/scm/node/SCMNodeStorageStatMap.java | 2 +- .../hadoop/hdds/scm/node/StaleNodeHandler.java | 9 +- .../hdds/scm/node/states/Node2ContainerMap.java | 2 +- .../hdds/scm/node/states/Node2PipelineMap.java | 75 ++ .../hdds/scm/node/states/NodeStateMap.java | 68 +- .../hdds/scm/pipelines/Node2PipelineMap.java | 75 -- .../hdds/scm/pipelines/PipelineSelector.java | 18 +- .../hdds/scm/server/SCMChillModeManager.java | 71 +- .../scm/server/SCMClientProtocolServer.java | 44 +- .../server/SCMDatanodeHeartbeatDispatcher.java | 11 +- .../scm/server/SCMDatanodeProtocolServer.java | 24 +- .../hadoop/hdds/scm/server/SCMMXBean.java | 12 + .../scm/server/StorageContainerManager.java | 79 +- .../commands/RetriableDatanodeEventWatcher.java | 58 ++ .../ozone/protocol/commands/package-info.java | 18 + .../hadoop/hdds/scm/block/TestBlockManager.java | 6 +- .../hdds/scm/block/TestDeletedBlockLog.java | 8 +- .../command/TestCommandStatusReportHandler.java | 12 +- .../hdds/scm/container/MockNodeManager.java | 169 ++-- .../TestCloseContainerEventHandler.java | 4 +- .../scm/container/TestContainerMapping.java | 380 ------- .../container/TestContainerReportHandler.java | 22 +- .../container/TestContainerStateManager.java | 2 +- .../scm/container/TestSCMContainerManager.java | 378 +++++++ .../hdds/scm/node/TestContainerPlacement.java | 15 +- .../hdds/scm/node/TestDeadNodeHandler.java | 145 ++- .../hadoop/hdds/scm/node/TestNodeManager.java | 123 +-- .../hdds/scm/node/TestNodeReportHandler.java | 5 +- .../scm/server/TestSCMChillModeManager.java | 41 +- .../TestSCMDatanodeHeartbeatDispatcher.java | 2 +- .../container/TestCloseContainerWatcher.java | 12 +- .../testutils/ReplicationNodeManagerMock.java | 150 +-- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 4 +- .../main/java/org/apache/hadoop/fs/XAttr.java | 2 +- .../org/apache/hadoop/hdfs/AddBlockFlag.java | 8 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 6 +- .../hadoop/hdfs/DFSInotifyEventInputStream.java | 2 +- .../java/org/apache/hadoop/hdfs/DFSPacket.java | 16 +- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 4 +- .../hadoop/hdfs/DistributedFileSystem.java | 13 +- .../hadoop/hdfs/NameNodeProxiesClient.java | 4 +- .../apache/hadoop/hdfs/client/HdfsAdmin.java | 12 +- .../hadoop/hdfs/client/impl/LeaseRenewer.java | 2 +- .../hdfs/protocol/ClientDatanodeProtocol.java | 2 +- .../hadoop/hdfs/protocol/ClientProtocol.java | 30 +- .../hdfs/protocol/ReconfigurationProtocol.java | 2 +- .../token/block/BlockTokenIdentifier.java | 4 +- .../hdfs/shortcircuit/DfsClientShmManager.java | 5 +- .../hadoop/hdfs/util/ByteArrayManager.java | 4 +- .../hadoop/hdfs/util/StripedBlockUtil.java | 59 +- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 5 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 1 - hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 4 +- .../hadoop/fs/http/client/HttpFSFileSystem.java | 17 +- .../hadoop/fs/http/server/FSOperations.java | 50 + .../http/server/HttpFSParametersProvider.java | 3 + .../hadoop/fs/http/server/HttpFSServer.java | 13 + .../fs/http/client/BaseTestHttpFSWith.java | 107 +- .../hadoop/fs/http/server/TestHttpFSServer.java | 96 ++ .../hadoop-hdfs-native-client/pom.xml | 4 +- hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 +- hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 4 +- ...uterAdminProtocolServerSideTranslatorPB.java | 2 +- .../resolver/ActiveNamenodeResolver.java | 11 +- .../resolver/FileSubclusterResolver.java | 1 + .../federation/resolver/MountTableResolver.java | 2 + .../MultipleDestinationMountTableResolver.java | 4 +- .../resolver/NamenodeStatusReport.java | 5 +- .../resolver/order/AvailableSpaceResolver.java | 2 +- .../resolver/order/LocalResolver.java | 2 +- .../federation/resolver/package-info.java | 5 +- .../federation/router/ConnectionPool.java | 3 +- .../federation/router/NameserviceManager.java | 9 + .../hdfs/server/federation/router/Quota.java | 4 +- .../hdfs/server/federation/router/Router.java | 5 +- .../federation/router/RouterAdminServer.java | 4 +- .../federation/router/RouterClientProtocol.java | 6 +- .../federation/router/RouterQuotaManager.java | 6 +- .../federation/router/RouterQuotaUsage.java | 4 +- .../federation/router/RouterRpcClient.java | 23 +- .../federation/router/RouterRpcServer.java | 14 +- .../router/RouterSafemodeService.java | 26 +- .../federation/router/RouterStateManager.java | 9 + .../federation/store/CachedRecordStore.java | 4 +- .../federation/store/MembershipStore.java | 16 +- .../server/federation/store/RouterStore.java | 10 +- .../federation/store/StateStoreService.java | 19 +- .../federation/store/StateStoreUtils.java | 4 + .../driver/StateStoreRecordOperations.java | 15 +- .../store/driver/impl/StateStoreBaseImpl.java | 1 - .../server/federation/store/package-info.java | 12 +- .../federation/store/records/BaseRecord.java | 4 +- .../store/records/MembershipState.java | 3 - .../federation/store/records/MountTable.java | 26 +- .../server/federation/store/records/Query.java | 2 +- .../federation/utils/ConsistentHashRing.java | 2 +- .../hdfs/tools/federation/RouterAdmin.java | 5 + hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +- .../server/blockmanagement/BlockManager.java | 19 +- .../hadoop/hdfs/server/datanode/DataNode.java | 4 +- .../hdfs/server/datanode/DirectoryScanner.java | 617 ++++++------ .../server/datanode/fsdataset/FsVolumeSpi.java | 36 +- .../datanode/fsdataset/impl/BlockPoolSlice.java | 177 +++- .../datanode/fsdataset/impl/FsDatasetImpl.java | 14 + .../datanode/fsdataset/impl/FsDatasetUtil.java | 30 +- .../datanode/fsdataset/impl/FsVolumeImpl.java | 203 ++-- .../datanode/fsdataset/impl/FsVolumeList.java | 5 +- .../fsdataset/impl/ProvidedVolumeImpl.java | 50 +- .../datanode/fsdataset/impl/ReplicaMap.java | 25 + .../namenode/ErasureCodingPolicyManager.java | 6 + .../hdfs/server/namenode/FSNamesystem.java | 9 + .../namenode/metrics/ECBlockGroupsMBean.java | 5 + .../src/main/resources/hdfs-default.xml | 9 + .../src/main/webapps/hdfs/dfshealth.html | 4 +- .../src/site/markdown/HdfsPermissionsGuide.md | 2 + .../src/site/markdown/HdfsProvidedStorage.md | 2 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 5 + .../TestRBWBlockInvalidation.java | 38 + .../server/datanode/SimulatedFSDataset.java | 222 ++--- .../TestDataNodeMultipleRegistrations.java | 2 +- .../server/datanode/TestDirectoryScanner.java | 249 +++-- .../datanode/extdataset/ExternalVolumeImpl.java | 10 +- .../fsdataset/impl/TestFsVolumeList.java | 64 +- .../fsdataset/impl/TestProvidedImpl.java | 13 +- .../server/namenode/TestAddStripedBlocks.java | 4 +- .../namenode/TestListCorruptFileBlocks.java | 50 +- .../server/namenode/TestNameNodeMXBean.java | 49 + .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 8 +- hadoop-hdfs-project/pom.xml | 4 +- .../hadoop-mapreduce-client-app/pom.xml | 4 +- .../hadoop-mapreduce-client-common/pom.xml | 4 +- .../hadoop-mapreduce-client-core/pom.xml | 4 +- .../hadoop/mapreduce/JobResourceUploader.java | 2 +- .../hadoop-mapreduce-client-hs-plugins/pom.xml | 4 +- .../hadoop-mapreduce-client-hs/pom.xml | 4 +- .../hadoop-mapreduce-client-jobclient/pom.xml | 4 +- .../hadoop-mapreduce-client-nativetask/pom.xml | 24 +- .../hadoop-mapreduce-client-shuffle/pom.xml | 4 +- .../hadoop-mapreduce-client-uploader/pom.xml | 4 +- .../hadoop-mapreduce-client/pom.xml | 4 +- .../hadoop-mapreduce-examples/pom.xml | 4 +- hadoop-mapreduce-project/pom.xml | 4 +- hadoop-maven-plugins/pom.xml | 2 +- hadoop-minicluster/pom.xml | 4 +- .../apache/hadoop/ozone/client/ObjectStore.java | 12 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 24 +- .../apache/hadoop/ozone/client/OzoneClient.java | 7 + .../apache/hadoop/ozone/client/OzoneVolume.java | 32 +- hadoop-ozone/common/src/main/bin/ozone | 7 +- .../dist/dev-support/bin/dist-layout-stitching | 2 +- hadoop-ozone/dist/pom.xml | 11 + .../main/compose/ozones3/docker-compose.yaml | 58 ++ .../dist/src/main/compose/ozones3/docker-config | 73 ++ .../src/main/smoketest/ozonefs/ozonefs.robot | 75 +- .../dist/src/main/smoketest/s3/awscli.robot | 63 ++ hadoop-ozone/dist/src/main/smoketest/test.sh | 8 +- hadoop-ozone/docs/content/Settings.md | 2 +- hadoop-ozone/integration-test/pom.xml | 4 + .../TestContainerStateManagerIntegration.java | 53 +- .../hdds/scm/pipeline/TestNode2PipelineMap.java | 23 +- .../hdds/scm/pipeline/TestNodeFailure.java | 18 +- .../hdds/scm/pipeline/TestPipelineClose.java | 30 +- .../hdds/scm/pipeline/TestSCMRestart.java | 33 +- .../hadoop/ozone/MiniOzoneClusterImpl.java | 4 +- .../org/apache/hadoop/ozone/OzoneTestUtils.java | 6 +- .../ozone/TestOzoneConfigurationFields.java | 4 +- .../ozone/TestStorageContainerManager.java | 154 +-- .../ozone/client/rest/TestOzoneRestClient.java | 2 +- .../rpc/TestCloseContainerHandlingByClient.java | 6 +- .../ozone/client/rpc/TestOzoneRpcClient.java | 43 +- .../common/impl/TestContainerPersistence.java | 3 +- .../commandhandler/TestBlockDeletion.java | 84 +- .../TestCloseContainerByPipeline.java | 6 +- .../TestCloseContainerHandler.java | 2 +- .../hadoop/ozone/om/TestScmChillMode.java | 330 ++++-- .../hadoop/ozone/ozShell/TestOzoneShell.java | 55 +- .../apache/hadoop/ozone/scm/TestSCMMXBean.java | 38 +- .../hadoop/ozone/web/client/TestBuckets.java | 4 +- .../hadoop/ozone/om/OMMetadataManager.java | 6 + .../hadoop/ozone/om/OmMetadataManagerImpl.java | 14 + .../apache/hadoop/ozone/om/OzoneManager.java | 90 +- .../hadoop/ozone/om/OzoneManagerLock.java | 58 +- .../apache/hadoop/ozone/om/S3BucketManager.java | 59 ++ .../hadoop/ozone/om/S3BucketManagerImpl.java | 194 ++++ .../hadoop/ozone/om/exceptions/OMException.java | 5 +- .../web/ozShell/bucket/CreateBucketHandler.java | 13 +- .../web/ozShell/bucket/InfoBucketHandler.java | 14 +- .../web/ozShell/bucket/ListBucketHandler.java | 15 +- .../ozone/web/ozShell/keys/ListKeyHandler.java | 16 +- .../web/ozShell/volume/CreateVolumeHandler.java | 21 +- .../web/ozShell/volume/InfoVolumeHandler.java | 16 +- .../web/ozShell/volume/ListVolumeHandler.java | 2 +- .../hadoop/ozone/om/TestOzoneManagerLock.java | 3 +- .../hadoop/ozone/om/TestS3BucketManager.java | 115 +++ .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 2 +- hadoop-ozone/pom.xml | 14 +- hadoop-ozone/s3gateway/pom.xml | 178 ++++ .../CommonHeadersContainerResponseFilter.java | 39 + .../apache/hadoop/ozone/s3/EndpointBase.java | 102 ++ .../org/apache/hadoop/ozone/s3/Gateway.java | 66 ++ .../hadoop/ozone/s3/GatewayApplication.java | 29 + .../hadoop/ozone/s3/OzoneClientProducer.java | 48 + .../ozone/s3/OzoneConfigurationHolder.java | 43 + .../hadoop/ozone/s3/S3GatewayConfigKeys.java | 55 + .../hadoop/ozone/s3/S3GatewayHttpServer.java | 85 ++ .../hadoop/ozone/s3/VirtualHostStyleFilter.java | 143 +++ .../hadoop/ozone/s3/bucket/DeleteBucket.java | 44 + .../hadoop/ozone/s3/bucket/HeadBucket.java | 67 ++ .../hadoop/ozone/s3/bucket/PutBucket.java | 44 + .../hadoop/ozone/s3/bucket/package-info.java | 30 + .../ozone/s3/commontypes/CommonPrefix.java | 47 + .../ozone/s3/commontypes/IsoDateAdapter.java | 47 + .../ozone/s3/commontypes/KeyMetadata.java | 87 ++ .../ozone/s3/commontypes/package-info.java | 29 + .../hadoop/ozone/s3/exception/OS3Exception.java | 161 +++ .../ozone/s3/exception/OS3ExceptionMapper.java | 41 + .../hadoop/ozone/s3/exception/S3ErrorTable.java | 81 ++ .../hadoop/ozone/s3/exception/package-info.java | 21 + .../hadoop/ozone/s3/object/DeleteObject.java | 51 + .../hadoop/ozone/s3/object/HeadObject.java | 58 ++ .../hadoop/ozone/s3/object/ListObject.java | 119 +++ .../ozone/s3/object/ListObjectResponse.java | 147 +++ .../hadoop/ozone/s3/object/PutObject.java | 61 ++ .../hadoop/ozone/s3/object/package-info.java | 29 + .../apache/hadoop/ozone/s3/package-info.java | 22 + .../src/main/resources/META-INF/beans.xml | 20 + .../webapps/s3gateway/WEB-INF/beans.xml | 20 + .../resources/webapps/s3gateway/WEB-INF/web.xml | 36 + .../hadoop/ozone/client/ObjectStoreStub.java | 110 ++ .../hadoop/ozone/client/OzoneBucketStub.java | 143 +++ .../hadoop/ozone/client/OzoneClientStub.java | 37 + .../hadoop/ozone/client/OzoneVolumeStub.java | 95 ++ .../hadoop/ozone/client/package-info.java | 21 + .../ozone/s3/TestVirtualHostStyleFilter.java | 163 +++ .../ozone/s3/bucket/TestBucketResponse.java | 40 + .../hadoop/ozone/s3/bucket/TestGetBucket.java | 114 +++ .../hadoop/ozone/s3/bucket/TestHeadBucket.java | 95 ++ .../hadoop/ozone/s3/bucket/package-info.java | 21 + .../ozone/s3/exception/TestOS3Exception.java | 49 + .../hadoop/ozone/s3/exception/package-info.java | 22 + .../ozone/s3/object/TestDeleteObject.java | 56 ++ .../hadoop/ozone/s3/object/package-info.java | 21 + .../apache/hadoop/ozone/s3/package-info.java | 21 + .../GenerateOzoneRequiredConfigurations.java | 130 +-- ...TestGenerateOzoneRequiredConfigurations.java | 167 +++- .../hadoop/ozone/scm/TestContainerSQLCli.java | 12 +- hadoop-project-dist/pom.xml | 4 +- hadoop-project/pom.xml | 44 +- hadoop-project/src/site/site.xml | 6 + hadoop-tools/hadoop-aliyun/pom.xml | 2 +- .../aliyun/oss/AliyunOSSBlockOutputStream.java | 5 +- .../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 5 +- .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 8 +- .../oss/AssumedRoleCredentialProvider.java | 115 +++ .../apache/hadoop/fs/aliyun/oss/Constants.java | 22 + .../site/markdown/tools/hadoop-aliyun/index.md | 50 + .../fs/aliyun/oss/TestAliyunCredentials.java | 55 +- hadoop-tools/hadoop-archive-logs/pom.xml | 4 +- hadoop-tools/hadoop-archives/pom.xml | 4 +- hadoop-tools/hadoop-aws/pom.xml | 4 +- .../org/apache/hadoop/fs/s3a/Constants.java | 10 + .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 30 +- .../hadoop/fs/s3a/s3guard/DDBPathMetadata.java | 16 +- .../fs/s3a/s3guard/DirListingMetadata.java | 15 +- .../fs/s3a/s3guard/DynamoDBMetadataStore.java | 7 +- .../fs/s3a/s3guard/ExpirableMetadata.java | 39 + .../hadoop/fs/s3a/s3guard/PathMetadata.java | 2 +- .../PathMetadataDynamoDBTranslation.java | 61 +- .../apache/hadoop/fs/s3a/s3guard/S3Guard.java | 61 +- .../site/markdown/tools/hadoop-aws/s3guard.md | 11 + .../apache/hadoop/fs/s3a/ITestS3GuardTtl.java | 105 ++ .../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 13 + .../hadoop/fs/s3a/auth/ITestAssumeRole.java | 47 +- .../hadoop/fs/s3a/auth/RoleTestUtils.java | 6 - .../fs/s3a/s3guard/MetadataStoreTestBase.java | 16 +- .../TestPathMetadataDynamoDBTranslation.java | 48 +- .../hadoop/fs/s3a/s3guard/TestS3Guard.java | 6 +- hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +- hadoop-tools/hadoop-azure/pom.xml | 42 +- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 44 + .../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 138 ++- .../fs/azurebfs/AzureBlobFileSystemStore.java | 39 +- .../fs/azurebfs/SecureAzureBlobFileSystem.java | 2 +- .../azurebfs/constants/ConfigurationKeys.java | 3 + .../constants/FileSystemConfigurations.java | 2 + .../exceptions/AbfsRestOperationException.java | 9 + .../extensions/AbfsAuthorizationException.java | 41 + .../fs/azurebfs/extensions/AbfsAuthorizer.java | 57 ++ .../azurebfs/oauth2/AzureADAuthenticator.java | 35 +- .../fs/azurebfs/services/AbfsRestOperation.java | 9 + .../azurebfs/AbstractAbfsIntegrationTest.java | 8 +- .../ITestAzureBlobFileSystemAuthorization.java | 362 +++++++ .../ITestAzureBlobFileSystemBackCompat.java | 2 +- .../ITestAzureBlobFileSystemFileStatus.java | 2 +- .../ITestAzureBlobFileSystemPermission.java | 4 +- .../ITestAzureBlobFileSystemRandomRead.java | 2 +- .../azurebfs/ITestAzureBlobFilesystemAcl.java | 116 +-- .../fs/azurebfs/ITestClientUrlScheme.java | 101 ++ .../fs/azurebfs/ITestOauthOverAbfsScheme.java | 63 ++ .../fs/azurebfs/ITestWasbAbfsCompatibility.java | 8 +- .../contract/AbfsFileSystemContract.java | 2 +- .../ITestAbfsFileSystemContractDistCp.java | 3 + .../azurebfs/extensions/MockAbfsAuthorizer.java | 87 ++ .../fs/azurebfs/extensions/package-info.java | 22 + .../services/TestOauthFailOverHttp.java | 55 - hadoop-tools/hadoop-datajoin/pom.xml | 4 +- hadoop-tools/hadoop-distcp/pom.xml | 4 +- hadoop-tools/hadoop-extras/pom.xml | 4 +- hadoop-tools/hadoop-fs2img/pom.xml | 4 +- hadoop-tools/hadoop-gridmix/pom.xml | 4 +- hadoop-tools/hadoop-kafka/pom.xml | 4 +- hadoop-tools/hadoop-openstack/pom.xml | 4 +- hadoop-tools/hadoop-pipes/pom.xml | 4 +- hadoop-tools/hadoop-resourceestimator/pom.xml | 2 +- hadoop-tools/hadoop-rumen/pom.xml | 4 +- hadoop-tools/hadoop-sls/pom.xml | 4 +- hadoop-tools/hadoop-streaming/pom.xml | 4 +- hadoop-tools/hadoop-tools-dist/pom.xml | 4 +- hadoop-tools/pom.xml | 4 +- .../hadoop-yarn/hadoop-yarn-api/pom.xml | 4 +- .../hadoop/yarn/conf/YarnConfiguration.java | 22 +- .../yarn/conf/TestYarnConfigurationFields.java | 2 + .../pom.xml | 4 +- .../pom.xml | 4 +- .../hadoop-yarn-services-api/pom.xml | 2 +- .../yarn/service/client/ApiServiceClient.java | 20 + .../hadoop/yarn/service/webapp/ApiServer.java | 34 +- ...RN-Simplified-V1-API-Layer-For-Services.yaml | 5 + .../hadoop-yarn-services-core/pom.xml | 2 +- .../hadoop/yarn/service/ClientAMProtocol.java | 5 + .../hadoop/yarn/service/ClientAMService.java | 12 + .../hadoop/yarn/service/ServiceEvent.java | 14 +- .../hadoop/yarn/service/ServiceEventType.java | 3 +- .../hadoop/yarn/service/ServiceManager.java | 332 +++++-- .../hadoop/yarn/service/ServiceScheduler.java | 2 + .../service/api/records/ContainerState.java | 2 +- .../yarn/service/api/records/Service.java | 15 + .../yarn/service/api/records/ServiceState.java | 7 +- .../yarn/service/client/ServiceClient.java | 21 + .../yarn/service/component/Component.java | 275 +++-- .../yarn/service/component/ComponentEvent.java | 10 - .../service/component/ComponentEventType.java | 1 + .../yarn/service/component/ComponentState.java | 3 +- .../component/instance/ComponentInstance.java | 269 ++++- .../instance/ComponentInstanceEventType.java | 3 +- .../instance/ComponentInstanceState.java | 3 +- .../containerlaunch/ContainerLaunchService.java | 4 +- .../pb/client/ClientAMProtocolPBClientImpl.java | 13 + .../service/ClientAMProtocolPBServiceImpl.java | 13 + .../yarn/service/provider/ProviderUtils.java | 9 +- .../yarn/service/utils/ServiceApiUtil.java | 57 +- .../yarn/service/utils/SliderFileSystem.java | 49 + .../src/main/proto/ClientAMProtocol.proto | 8 + .../yarn/service/MockRunningServiceContext.java | 20 +- .../hadoop/yarn/service/ServiceTestUtils.java | 2 +- .../hadoop/yarn/service/TestServiceManager.java | 136 ++- .../yarn/service/TestYarnNativeServices.java | 46 + .../yarn/service/client/TestServiceCLI.java | 17 + .../yarn/service/component/TestComponent.java | 239 ++++- .../instance/TestComponentInstance.java | 177 +++- .../yarn/service/utils/TestServiceApiUtil.java | 39 + .../src/test/resources/log4j.properties | 19 + .../hadoop-yarn-services/pom.xml | 2 +- .../hadoop-yarn-submarine/README.md | 6 +- .../hadoop-yarn-submarine/pom.xml | 10 +- .../base/ubuntu-16.04/Dockerfile.cpu.tf_1.8.0 | 69 ++ .../base/ubuntu-16.04/Dockerfile.gpu.tf_1.8.0 | 67 ++ .../src/main/docker/build-all.sh | 32 + .../ubuntu-16.04/Dockerfile.cpu.tf_1.8.0 | 22 + .../ubuntu-16.04/Dockerfile.gpu.tf_1.8.0 | 22 + .../cifar10_estimator_tf_1.8.0/README.md | 542 ++++++++++ .../cifar10_estimator_tf_1.8.0/cifar10.py | 113 +++ .../cifar10_estimator_tf_1.8.0/cifar10_main.py | 521 ++++++++++ .../cifar10_estimator_tf_1.8.0/cifar10_model.py | 80 ++ .../cifar10_estimator_tf_1.8.0/cifar10_utils.py | 154 +++ .../generate_cifar10_tfrecords.py | 114 +++ .../cifar10_estimator_tf_1.8.0/model_base.py | 219 ++++ .../zeppelin-notebook-example/Dockerfile.gpu | 75 ++ .../zeppelin-notebook-example/run_container.sh | 22 + .../docker/zeppelin-notebook-example/shiro.ini | 120 +++ .../zeppelin-notebook-example/zeppelin-site.xml | 569 +++++++++++ .../yarnservice/YarnServiceJobSubmitter.java | 16 +- .../src/site/DeveloperGuide.md | 26 - .../src/site/QuickStart.md | 134 --- .../src/site/markdown/DeveloperGuide.md | 24 + .../src/site/markdown/Examples.md | 21 + .../src/site/markdown/Index.md | 42 + .../src/site/markdown/QuickStart.md | 174 ++++ .../markdown/RunningDistributedCifar10TFJobs.md | 162 +++ .../src/site/markdown/RunningZeppelinOnYARN.md | 37 + .../src/site/markdown/WriteDockerfile.md | 117 +++ .../src/site/resources/css/site.css | 29 + .../src/site/resources/images/job-logs-ui.png | Bin 0 -> 229944 bytes .../images/multiple-tensorboard-jobs.png | Bin 0 -> 184717 bytes .../resources/images/tensorboard-service.png | Bin 0 -> 107567 bytes .../hadoop-yarn-submarine/src/site/site.xml | 28 + .../hadoop-yarn-applications/pom.xml | 4 +- .../hadoop-yarn/hadoop-yarn-client/pom.xml | 4 +- .../yarn/client/api/async/AMRMClientAsync.java | 11 + .../api/async/impl/AMRMClientAsyncImpl.java | 9 + .../hadoop/yarn/client/cli/ApplicationCLI.java | 12 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 2 + .../hadoop-yarn/hadoop-yarn-common/pom.xml | 4 +- .../hadoop/yarn/client/AMRMClientUtils.java | 7 + .../hadoop/yarn/client/api/AppAdminClient.java | 13 + .../src/main/resources/yarn-default.xml | 18 + .../client/api/impl/TestTimelineClient.java | 25 +- .../hadoop-yarn/hadoop-yarn-registry/pom.xml | 4 +- .../pom.xml | 4 +- .../hadoop-yarn-server-common/pom.xml | 4 +- .../yarn/server/AMHeartbeatRequestHandler.java | 23 +- .../hadoop/yarn/server/AMRMClientRelayer.java | 25 +- .../utils/FederationRegistryClient.java | 10 +- .../yarn/server/uam/UnmanagedAMPoolManager.java | 15 + .../server/uam/UnmanagedApplicationManager.java | 14 + .../yarn/server/MockResourceManagerFacade.java | 72 +- .../yarn/server/TestAMRMClientRelayer.java | 25 + .../hadoop-yarn-server-nodemanager/pom.xml | 4 +- .../amrmproxy/FederationInterceptor.java | 388 +++++--- .../containermanager/ContainerManagerImpl.java | 1 + .../container/ContainerImpl.java | 3 + .../launcher/ContainerLaunch.java | 1 + .../resources/gpu/GpuResourceHandlerImpl.java | 3 + .../runtime/DefaultLinuxContainerRuntime.java | 6 +- .../runtime/DockerLinuxContainerRuntime.java | 23 +- .../container-executor/impl/utils/docker-util.c | 4 +- .../amrmproxy/TestFederationInterceptor.java | 298 +++--- .../TestableFederationInterceptor.java | 107 ++ .../containermanager/TestNMProxy.java | 2 +- .../resources/gpu/TestGpuResourceHandler.java | 9 +- .../runtime/TestDockerContainerRuntime.java | 91 +- .../hadoop-yarn-server-resourcemanager/pom.xml | 4 +- .../ApplicationMasterService.java | 15 +- .../server/resourcemanager/RMAppManager.java | 11 +- .../server/resourcemanager/RMServerUtils.java | 9 +- .../resourcemanager/ResourceTrackerService.java | 23 + .../resourcemanager/recovery/RMStateStore.java | 2 + .../server/resourcemanager/rmapp/RMAppImpl.java | 63 +- .../scheduler/AbstractYarnScheduler.java | 21 +- .../scheduler/ResourceLimits.java | 24 + .../scheduler/SchedulerUtils.java | 62 +- .../scheduler/YarnScheduler.java | 9 +- .../scheduler/capacity/CapacityScheduler.java | 7 +- .../scheduler/capacity/ParentQueue.java | 16 +- .../allocator/RegularContainerAllocator.java | 18 +- .../processor/PlacementConstraintProcessor.java | 12 +- .../scheduler/fair/AllocationConfiguration.java | 13 + .../scheduler/fair/FSLeafQueue.java | 10 + .../scheduler/fair/FSParentQueue.java | 15 +- .../resourcemanager/scheduler/fair/FSQueue.java | 7 + .../scheduler/fair/FairScheduler.java | 52 +- .../allocation/AllocationFileQueueParser.java | 7 + .../fair/allocation/QueueProperties.java | 16 +- .../webapp/FairSchedulerPage.java | 4 + .../webapp/dao/FairSchedulerQueueInfo.java | 9 +- .../resourcemanager/AppManagerTestBase.java | 107 ++ .../ApplicationMasterServiceTestBase.java | 582 +++++++++++ .../resourcemanager/MockMemoryRMStateStore.java | 1 + .../yarn/server/resourcemanager/MockRM.java | 48 + .../server/resourcemanager/TestAppManager.java | 89 +- .../TestAppManagerWithFairScheduler.java | 177 ++++ .../TestApplicationMasterService.java | 993 ------------------- .../TestApplicationMasterServiceCapacity.java | 207 ++++ .../TestApplicationMasterServiceFair.java | 118 +++ ...TestApplicationMasterServiceInterceptor.java | 217 ++++ .../resourcemanager/TestClientRMService.java | 104 +- .../server/resourcemanager/TestRMRestart.java | 68 ++ .../resourcemanager/TestRMServerUtils.java | 98 +- .../TestResourceTrackerService.java | 45 + .../rmapp/TestRMAppTransitions.java | 171 ++-- .../scheduler/AppMetricsChecker.java | 122 +++ .../scheduler/ResourceMetricsChecker.java | 170 ++++ .../scheduler/TestQueueMetrics.java | 550 +++++++--- .../scheduler/TestSchedulerUtils.java | 144 +-- .../capacity/TestCapacityScheduler.java | 39 +- .../capacity/TestContainerAllocation.java | 78 ++ .../TestNodeLabelContainerAllocation.java | 62 ++ .../scheduler/fair/FairSchedulerTestBase.java | 4 +- .../fair/TestAllocationFileLoaderService.java | 37 +- .../TestApplicationMasterServiceWithFS.java | 174 ++++ .../scheduler/fair/TestFairScheduler.java | 140 ++- .../allocationfile/AllocationFileQueue.java | 6 +- .../AllocationFileQueueBuilder.java | 6 + .../AllocationFileQueueProperties.java | 12 + .../webapp/TestRMWebServicesAppAttempts.java | 397 ++++++++ ...TestRMWebServicesAppCustomResourceTypes.java | 170 ++++ .../webapp/TestRMWebServicesApps.java | 287 ------ ...estRMWebServicesAppsCustomResourceTypes.java | 81 +- ...RMWebServicesCustomResourceTypesCommons.java | 99 ++ .../webapp/TestRMWebServicesFairScheduler.java | 160 --- ...ustomResourceTypesConfigurationProvider.java | 4 +- .../FairSchedulerJsonVerifications.java | 4 +- .../FairSchedulerXmlVerifications.java | 4 +- .../TestRMWebServicesFairScheduler.java | 160 +++ ...ervicesFairSchedulerCustomResourceTypes.java | 4 +- .../webapp/helper/AppInfoJsonVerifications.java | 4 +- .../webapp/helper/AppInfoXmlVerifications.java | 4 +- .../webapp/helper/BufferedClientResponse.java | 11 +- .../helper/JsonCustomResourceTypeTestcase.java | 9 + .../ResourceRequestsJsonVerifications.java | 4 +- .../ResourceRequestsXmlVerifications.java | 4 +- .../helper/XmlCustomResourceTypeTestCase.java | 9 + .../hadoop-yarn-server-router/pom.xml | 4 +- .../clientrm/FederationClientInterceptor.java | 2 +- .../TestFederationClientInterceptor.java | 9 + .../pom.xml | 4 +- .../hadoop-yarn-server-tests/pom.xml | 4 +- .../pom.xml | 4 +- .../pom.xml | 4 +- .../pom.xml | 2 +- .../pom.xml | 4 +- .../pom.xml | 4 +- .../pom.xml | 4 +- .../pom.xml | 4 +- .../pom.xml | 4 +- .../hadoop-yarn-server-timelineservice/pom.xml | 4 +- .../collector/TimelineCollectorWebService.java | 16 +- .../PerNodeAggTimelineCollectorMetrics.java | 117 +++ .../metrics/TimelineReaderMetrics.java | 113 +++ .../timelineservice/metrics/package-info.java | 28 + .../reader/TimelineReaderWebServices.java | 166 +++- .../TestPerNodeAggTimelineCollectorMetrics.java | 56 ++ .../reader/TestTimelineReaderMetrics.java | 56 ++ .../hadoop-yarn-server-web-proxy/pom.xml | 4 +- .../hadoop-yarn/hadoop-yarn-server/pom.xml | 4 +- .../hadoop-yarn/hadoop-yarn-site/pom.xml | 4 +- .../src/site/markdown/DockerContainers.md | 73 +- .../src/site/markdown/FairScheduler.md | 2 + .../src/site/markdown/registry/yarn-registry.md | 2 +- .../markdown/yarn-service/YarnServiceAPI.md | 2 +- .../hadoop-yarn/hadoop-yarn-ui/pom.xml | 4 +- .../main/webapp/app/adapters/yarn-service.js | 32 + .../webapp/app/controllers/app-table-columns.js | 14 +- .../main/webapp/app/controllers/application.js | 7 +- .../src/main/webapp/app/controllers/yarn-app.js | 20 +- .../src/main/webapp/app/models/yarn-app.js | 15 +- .../src/main/webapp/app/models/yarn-service.js | 30 + .../src/main/webapp/app/router.js | 7 +- .../src/main/webapp/app/routes/application.js | 8 +- .../src/main/webapp/app/routes/yarn-app.js | 12 + .../src/main/webapp/app/serializers/yarn-app.js | 4 +- .../main/webapp/app/serializers/yarn-service.js | 44 + .../src/main/webapp/app/templates/error.hbs | 2 +- .../src/main/webapp/app/templates/notfound.hbs | 2 +- .../src/main/webapp/app/templates/yarn-app.hbs | 28 +- .../main/webapp/app/templates/yarn-services.hbs | 4 +- hadoop-yarn-project/hadoop-yarn/pom.xml | 4 +- hadoop-yarn-project/pom.xml | 4 +- pom.xml | 4 +- 645 files changed, 22630 insertions(+), 7630 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb75861e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb75861e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb75861e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb75861e/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb75861e/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb75861e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb75861e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb75861e/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb75861e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
