Merge remote-tracking branch 'apache-commit/trunk' into HDDS-48
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1045251 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1045251 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1045251 Branch: refs/heads/trunk Commit: c10452516804eed793e575dedcd1ee7758ec1f4c Parents: 52d1d96 d3fa83a Author: Bharat Viswanadham <bha...@apache.org> Authored: Thu Jun 28 14:28:23 2018 -0700 Committer: Bharat Viswanadham <bha...@apache.org> Committed: Thu Jun 28 14:28:23 2018 -0700 ---------------------------------------------------------------------- NOTICE.txt | 8 + dev-support/bin/ozone-dist-layout-stitching | 4 +- .../hadoop-client-minicluster/pom.xml | 8 - hadoop-common-project/hadoop-common/pom.xml | 5 - .../org/apache/hadoop/conf/Configuration.java | 8 +- .../hadoop/conf/ReconfigurationServlet.java | 42 +- .../apache/hadoop/crypto/key/KeyProvider.java | 4 +- .../org/apache/hadoop/crypto/key/KeyShell.java | 2 +- .../crypto/key/kms/KMSClientProvider.java | 16 +- .../java/org/apache/hadoop/fs/BBPartHandle.java | 58 ++ .../org/apache/hadoop/fs/BBUploadHandle.java | 57 ++ .../fs/CommonConfigurationKeysPublic.java | 4 +- .../hadoop/fs/FSDataOutputStreamBuilder.java | 22 + .../java/org/apache/hadoop/fs/FileContext.java | 66 ++ .../hadoop/fs/FileSystemMultipartUploader.java | 132 ++++ .../main/java/org/apache/hadoop/fs/FsShell.java | 3 +- .../hadoop/fs/LocalFileSystemPathHandle.java | 100 +++ .../org/apache/hadoop/fs/MultipartUploader.java | 90 +++ .../hadoop/fs/MultipartUploaderFactory.java | 65 ++ .../main/java/org/apache/hadoop/fs/Options.java | 3 + .../java/org/apache/hadoop/fs/PartHandle.java | 45 ++ .../main/java/org/apache/hadoop/fs/Path.java | 2 +- .../apache/hadoop/fs/RawLocalFileSystem.java | 61 +- .../UnsupportedMultipartUploaderException.java | 41 ++ .../java/org/apache/hadoop/fs/UploadHandle.java | 47 ++ .../java/org/apache/hadoop/fs/shell/Count.java | 2 +- .../apache/hadoop/io/ElasticByteBufferPool.java | 2 +- .../apache/hadoop/io/FastByteComparisons.java | 44 +- .../apache/hadoop/io/erasurecode/ECSchema.java | 4 +- .../io/erasurecode/ErasureCodeNative.java | 2 +- .../nativeio/SharedFileDescriptorFactory.java | 2 +- .../org/apache/hadoop/ipc/CallerContext.java | 4 +- .../apache/hadoop/ipc/DecayRpcScheduler.java | 4 +- .../org/apache/hadoop/ipc/FairCallQueue.java | 4 +- .../hadoop/metrics2/MetricsJsonBuilder.java | 2 +- .../hadoop/metrics2/impl/MetricsConfig.java | 15 +- .../hadoop/metrics2/lib/MethodMetric.java | 2 +- .../metrics2/lib/MutableMetricsFactory.java | 2 +- .../hadoop/metrics2/lib/MutableQuantiles.java | 2 +- .../metrics2/lib/MutableRollingAverages.java | 2 +- .../apache/hadoop/metrics2/lib/MutableStat.java | 2 +- .../metrics2/sink/RollingFileSystemSink.java | 2 +- .../org/apache/hadoop/net/TableMapping.java | 2 +- .../apache/hadoop/net/unix/DomainSocket.java | 2 +- .../hadoop/net/unix/DomainSocketWatcher.java | 2 +- .../security/ShellBasedUnixGroupsMapping.java | 12 +- .../hadoop/security/UserGroupInformation.java | 4 +- .../hadoop/security/alias/CredentialShell.java | 2 +- .../hadoop/security/http/CrossOriginFilter.java | 2 +- .../hadoop/security/token/DtFileOperations.java | 2 +- .../web/DelegationTokenAuthenticator.java | 8 +- .../org/apache/hadoop/tools/TableListing.java | 6 +- .../apache/hadoop/util/HttpExceptionUtils.java | 12 +- .../apache/hadoop/util/JsonSerialization.java | 24 + .../main/java/org/apache/hadoop/util/Shell.java | 2 +- .../org/apache/hadoop/util/StringUtils.java | 64 +- .../org/apache/hadoop/util/SysInfoLinux.java | 19 +- .../hadoop-common/src/main/proto/FSProtos.proto | 8 + ...rg.apache.hadoop.fs.MultipartUploaderFactory | 16 + .../apache/hadoop/conf/TestConfiguration.java | 2 +- .../conf/TestConfigurationFieldsBase.java | 2 +- .../crypto/random/TestOsSecureRandom.java | 2 +- .../fs/AbstractSystemMultipartUploaderTest.java | 143 ++++ .../org/apache/hadoop/fs/FSTestWrapper.java | 2 +- .../fs/FileContextMainOperationsBaseTest.java | 44 +- .../apache/hadoop/fs/FileContextTestHelper.java | 1 - .../hadoop/fs/TestDFCachingGetSpaceUsed.java | 2 +- .../fs/TestFileSystemStorageStatistics.java | 18 +- .../apache/hadoop/fs/TestLocalFileSystem.java | 23 +- .../TestLocalFileSystemMultipartUploader.java | 65 ++ .../AbstractContractPathHandleTest.java | 6 + .../TestRawlocalContractPathHandle.java | 40 ++ .../hadoop/fs/shell/TestCopyFromLocal.java | 10 +- .../TestSharedFileDescriptorFactory.java | 2 +- .../org/apache/hadoop/ipc/TestProtoBufRpc.java | 2 +- .../java/org/apache/hadoop/ipc/TestSaslRPC.java | 2 +- .../org/apache/hadoop/net/TestNetUtils.java | 2 +- .../hadoop/net/unix/TestDomainSocket.java | 2 +- .../TestShellBasedUnixGroupsMapping.java | 39 +- ...tionTokenAuthenticationHandlerWithMocks.java | 2 +- .../AbstractServiceLauncherTestBase.java | 2 +- .../apache/hadoop/test/GenericTestUtils.java | 4 +- .../hadoop/util/TestShutdownHookManager.java | 2 +- .../apache/hadoop/util/TestSignalLogger.java | 2 +- .../apache/hadoop/util/TestSysInfoLinux.java | 60 ++ .../src/test/resources/contract/rawlocal.xml | 5 + .../test/scripts/process_with_sigterm_trap.sh | 24 + .../crypto/key/kms/server/KMSJSONWriter.java | 6 +- hadoop-dist/pom.xml | 5 + .../src/main/compose/ozone/docker-compose.yaml | 12 - .../src/main/compose/ozone/docker-config | 5 - .../main/compose/ozoneperf/docker-compose.yaml | 13 - .../src/main/compose/ozoneperf/docker-config | 5 - .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 26 - .../container/common/helpers/ContainerInfo.java | 32 + .../scm/container/common/helpers/Pipeline.java | 142 ++-- .../common/helpers/PipelineChannel.java | 124 ---- .../org/apache/hadoop/ozone/OzoneConsts.java | 1 - .../apache/hadoop/utils/BackgroundService.java | 2 +- hadoop-hdds/common/src/main/proto/hdds.proto | 8 +- .../common/src/main/resources/ozone-default.xml | 47 -- .../hadoop/ozone/HddsDatanodeService.java | 13 +- .../statemachine/DatanodeStateMachine.java | 3 + .../background/BlockDeletingService.java | 8 +- .../ReplicateContainerCommandHandler.java | 67 ++ .../states/endpoint/HeartbeatEndpointTask.java | 12 + .../container/common/utils/HddsVolumeUtil.java | 2 +- .../container/replication/ReplicationQueue.java | 76 ++ .../replication/ReplicationRequest.java | 106 +++ .../container/replication/package-info.java | 23 + .../protocol/commands/CommandForDatanode.java | 45 ++ .../commands/ReplicateContainerCommand.java | 94 +++ .../StorageContainerDatanodeProtocol.proto | 12 +- .../replication/TestReplicationQueue.java | 134 ++++ .../container/replication/package-info.java | 23 + .../testutils/BlockDeletingServiceTestImpl.java | 4 +- .../hadoop/hdds/server/events/EventWatcher.java | 157 +++++ .../server/events/IdentifiableEventPayload.java | 30 + .../hdds/server/events/EventHandlerStub.java | 38 + .../hdds/server/events/TestEventWatcher.java | 220 ++++++ .../hdds/scm/container/ContainerMapping.java | 10 +- .../scm/container/closer/ContainerCloser.java | 6 +- .../replication/ContainerSupervisor.java | 340 --------- .../container/replication/InProgressPool.java | 255 ------- .../scm/container/replication/PeriodicPool.java | 119 ---- .../scm/container/replication/package-info.java | 23 - .../hadoop/hdds/scm/node/NodeManager.java | 6 - .../hadoop/hdds/scm/node/NodePoolManager.java | 71 -- .../hadoop/hdds/scm/node/SCMNodeManager.java | 41 +- .../hdds/scm/node/SCMNodePoolManager.java | 269 -------- .../hdds/scm/pipelines/PipelineManager.java | 67 +- .../hdds/scm/pipelines/PipelineSelector.java | 11 +- .../scm/pipelines/ratis/RatisManagerImpl.java | 13 +- .../standalone/StandaloneManagerImpl.java | 8 +- .../server/SCMDatanodeHeartbeatDispatcher.java | 126 ++++ .../scm/server/SCMDatanodeProtocolServer.java | 25 +- .../scm/server/StorageContainerManager.java | 10 +- .../SCMDatanodeContainerReportHandler.java | 76 -- .../report/SCMDatanodeHeartbeatDispatcher.java | 189 ----- .../report/SCMDatanodeNodeReportHandler.java | 43 -- .../server/report/SCMDatanodeReportHandler.java | 83 --- .../report/SCMDatanodeReportHandlerFactory.java | 82 --- .../hdds/scm/server/report/package-info.java | 57 -- .../hdds/scm/block/TestDeletedBlockLog.java | 8 +- .../hdds/scm/container/MockNodeManager.java | 6 - .../hadoop/hdds/scm/node/TestNodeManager.java | 39 ++ .../hdds/scm/node/TestSCMNodePoolManager.java | 160 ----- .../TestSCMDatanodeHeartbeatDispatcher.java | 119 ++++ .../TestSCMDatanodeContainerReportHandler.java | 34 - .../TestSCMDatanodeHeartbeatDispatcher.java | 138 ---- .../TestSCMDatanodeNodeReportHandler.java | 36 - .../TestSCMDatanodeReportHandlerFactory.java | 51 -- .../hdds/scm/server/report/package-info.java | 21 - .../testutils/ReplicationNodeManagerMock.java | 5 - .../ReplicationNodePoolManagerMock.java | 133 ---- .../main/java/org/apache/hadoop/fs/XAttr.java | 4 +- .../hdfs/DFSMultipartUploaderFactory.java | 40 ++ .../org/apache/hadoop/hdfs/ExtendedBlockId.java | 4 +- .../hdfs/client/impl/BlockReaderFactory.java | 2 +- .../AddErasureCodingPolicyResponse.java | 4 +- .../hdfs/protocol/CacheDirectiveInfo.java | 4 +- .../hadoop/hdfs/protocol/CachePoolInfo.java | 4 +- .../hadoop/hdfs/protocol/EncryptionZone.java | 4 +- .../hdfs/protocol/ErasureCodingPolicy.java | 4 +- .../hdfs/protocol/ErasureCodingPolicyInfo.java | 4 +- .../hadoop/hdfs/protocol/ExtendedBlock.java | 2 +- .../hdfs/shortcircuit/DfsClientShmManager.java | 2 +- .../hdfs/shortcircuit/ShortCircuitCache.java | 2 +- .../hdfs/shortcircuit/ShortCircuitShm.java | 4 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 7 +- ...onfRefreshTokenBasedAccessTokenProvider.java | 8 +- .../CredentialBasedAccessTokenProvider.java | 8 +- ...rg.apache.hadoop.fs.MultipartUploaderFactory | 16 + .../ha/TestRequestHedgingProxyProvider.java | 2 +- hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 5 - .../federation/router/ConnectionPoolId.java | 2 +- .../router/RemoteLocationContext.java | 2 +- .../store/driver/impl/StateStoreFileImpl.java | 2 +- .../federation/store/records/MountTable.java | 2 +- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 5 - .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +- .../hadoop/hdfs/protocol/CacheDirective.java | 2 +- .../qjournal/server/GetJournalEditServlet.java | 4 +- .../hadoop/hdfs/qjournal/server/Journal.java | 12 +- .../hadoop/hdfs/server/balancer/Balancer.java | 6 +- .../hadoop/hdfs/server/balancer/Dispatcher.java | 30 +- .../server/blockmanagement/BlockManager.java | 2 +- .../hdfs/server/datanode/DirectoryScanner.java | 4 +- .../datanode/fsdataset/impl/FsDatasetCache.java | 2 +- .../server/diskbalancer/command/Command.java | 4 +- .../diskbalancer/command/PlanCommand.java | 4 +- .../diskbalancer/command/ReportCommand.java | 4 +- .../server/namenode/EncryptionZoneManager.java | 4 +- .../server/namenode/FSDirErasureCodingOp.java | 2 +- .../hdfs/server/namenode/FSNamesystem.java | 6 +- .../hdfs/server/namenode/FSNamesystemLock.java | 17 +- .../server/namenode/INodeAttributeProvider.java | 2 +- .../server/namenode/startupprogress/Step.java | 6 +- .../server/namenode/top/metrics/TopMetrics.java | 2 +- .../apache/hadoop/hdfs/tools/CacheAdmin.java | 2 +- .../apache/hadoop/hdfs/util/EnumCounters.java | 2 +- .../src/main/resources/hdfs-default.xml | 10 + .../hadoop/fs/TestEnhancedByteBufferAccess.java | 4 +- .../org/apache/hadoop/fs/TestGlobPaths.java | 2 +- .../hadoop/fs/TestHDFSMultipartUploader.java | 76 ++ .../TestWebHdfsFileContextMainOperations.java | 2 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 4 +- .../org/apache/hadoop/hdfs/TestDFSShell.java | 2 +- .../apache/hadoop/hdfs/TestDecommission.java | 2 +- .../hadoop/hdfs/TestFsShellPermission.java | 2 +- .../hadoop/hdfs/TestHDFSPolicyProvider.java | 2 +- .../hadoop/hdfs/TestLeaseRecoveryStriped.java | 2 +- .../hdfs/TestSecureEncryptionZoneWithKMS.java | 34 +- .../sasl/SaslDataTransferTestCase.java | 2 +- .../hdfs/server/balancer/TestBalancer.java | 81 ++- .../server/datanode/SimulatedFSDataset.java | 2 +- .../command/TestDiskBalancerCommand.java | 2 +- .../hadoop/hdfs/server/mover/TestMover.java | 46 ++ .../hdfs/server/namenode/FSImageTestUtil.java | 2 +- .../hdfs/server/namenode/NameNodeAdapter.java | 2 +- .../hdfs/server/namenode/TestAuditLogger.java | 2 +- .../server/namenode/TestCacheDirectives.java | 2 +- .../namenode/TestEditLogJournalFailures.java | 2 +- .../hdfs/server/namenode/TestEditLogRace.java | 6 +- .../server/namenode/TestFSNamesystemLock.java | 10 +- .../hdfs/server/namenode/TestStartup.java | 9 +- .../namenode/metrics/TestNameNodeMetrics.java | 4 +- .../shortcircuit/TestShortCircuitCache.java | 2 +- .../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 2 +- .../hadoop/hdfs/tools/TestDFSAdminWithHA.java | 12 + .../org/apache/hadoop/tracing/TestTracing.java | 2 +- .../v2/app/job/impl/TestTaskAttempt.java | 4 +- .../apache/hadoop/mapreduce/JobSubmitter.java | 8 +- .../lib/input/CombineFileInputFormat.java | 4 +- .../mapred/uploader/FrameworkUploader.java | 2 +- .../mapred/uploader/TestFrameworkUploader.java | 13 + hadoop-maven-plugins/pom.xml | 38 +- .../maven/plugin/resourcegz/ResourceGzMojo.java | 2 +- .../dev-support/bin/robot-all.sh | 2 +- .../dev-support/bin/robot-dnd-all.sh | 63 ++ .../acceptance-test/dev-support/bin/robot.sh | 9 +- .../dev-support/docker/Dockerfile | 21 + .../dev-support/docker/docker-compose.yaml | 23 + hadoop-ozone/acceptance-test/pom.xml | 1 + .../src/test/acceptance/basic/.env | 17 + .../src/test/acceptance/basic/basic.robot | 50 ++ .../test/acceptance/basic/docker-compose.yaml | 50 ++ .../src/test/acceptance/basic/docker-config | 33 + .../src/test/acceptance/basic/ozone-shell.robot | 85 +++ .../src/test/acceptance/commonlib.robot | 78 +++ .../src/test/acceptance/ozonefs/.env | 17 + .../test/acceptance/ozonefs/docker-compose.yaml | 59 ++ .../src/test/acceptance/ozonefs/docker-config | 34 + .../src/test/acceptance/ozonefs/ozonefs.robot | 39 ++ .../acceptance-test/src/test/compose/.env | 17 - .../src/test/compose/docker-compose.yaml | 62 -- .../src/test/compose/docker-config | 36 - .../robotframework/acceptance/ozone-shell.robot | 256 ------- .../test/robotframework/acceptance/ozone.robot | 104 --- hadoop-ozone/common/src/main/bin/ozone | 9 +- .../hadoop/ozone/TestMiniOzoneCluster.java | 8 +- .../ozone/container/ContainerTestHelper.java | 19 +- .../common/TestBlockDeletingService.java | 11 +- .../TestReplicateContainerHandler.java | 71 ++ ...TestGenerateOzoneRequiredConfigurations.java | 2 +- .../hadoop/ozone/scm/TestContainerSQLCli.java | 31 - .../apache/hadoop/ozone/scm/TestSCMMetrics.java | 253 ------- .../ozone-manager/src/main/webapps/ksm/ksm.js | 4 +- hadoop-ozone/ozonefs/pom.xml | 211 ++++++ .../org/apache/hadoop/fs/ozone/Constants.java | 42 ++ .../java/org/apache/hadoop/fs/ozone/OzFs.java | 44 ++ .../hadoop/fs/ozone/OzoneFSInputStream.java | 79 +++ .../hadoop/fs/ozone/OzoneFSOutputStream.java | 59 ++ .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 689 +++++++++++++++++++ .../apache/hadoop/fs/ozone/package-info.java | 30 + .../hadoop/fs/ozone/TestOzoneFSInputStream.java | 157 +++++ .../fs/ozone/TestOzoneFileInterfaces.java | 231 +++++++ .../contract/ITestOzoneContractCreate.java | 48 ++ .../contract/ITestOzoneContractDelete.java | 48 ++ .../contract/ITestOzoneContractDistCp.java | 50 ++ .../ITestOzoneContractGetFileStatus.java | 61 ++ .../ozone/contract/ITestOzoneContractMkdir.java | 48 ++ .../ozone/contract/ITestOzoneContractOpen.java | 47 ++ .../contract/ITestOzoneContractRename.java | 49 ++ .../contract/ITestOzoneContractRootDir.java | 51 ++ .../ozone/contract/ITestOzoneContractSeek.java | 47 ++ .../hadoop/fs/ozone/contract/OzoneContract.java | 123 ++++ .../src/test/resources/contract/ozone.xml | 113 +++ .../ozonefs/src/test/resources/log4j.properties | 23 + hadoop-ozone/pom.xml | 1 + .../genesis/BenchMarkContainerStateMap.java | 11 +- .../genesis/BenchMarkDatanodeDispatcher.java | 6 +- .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 78 +-- hadoop-project/pom.xml | 16 +- .../aliyun/oss/AliyunCredentialsProvider.java | 2 +- .../fs/aliyun/oss/AliyunOSSFileSystem.java | 2 +- .../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 2 +- .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 2 +- .../fs/aliyun/oss/AliyunOSSTestUtils.java | 2 +- hadoop-tools/hadoop-aws/pom.xml | 42 +- .../fs/s3a/AWSCredentialProviderList.java | 2 +- .../fs/s3a/BasicAWSCredentialsProvider.java | 2 +- .../org/apache/hadoop/fs/s3a/Constants.java | 11 + .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 4 +- .../apache/hadoop/fs/s3a/S3AInputStream.java | 2 +- .../hadoop/fs/s3a/S3AMultipartUploader.java | 150 ++++ .../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 2 +- .../fs/s3a/SimpleAWSCredentialsProvider.java | 2 +- .../fs/s3a/TemporaryAWSCredentialsProvider.java | 2 +- .../s3a/auth/AssumedRoleCredentialProvider.java | 2 +- .../s3a/commit/files/SinglePendingCommit.java | 2 +- .../hadoop/fs/s3a/commit/files/SuccessData.java | 2 +- .../fs/s3a/commit/magic/MagicCommitTracker.java | 2 +- .../hadoop/fs/s3a/commit/staging/Paths.java | 2 +- .../fs/s3a/s3guard/DynamoDBClientFactory.java | 2 +- .../fs/s3a/s3guard/DynamoDBMetadataStore.java | 3 +- .../fs/s3a/s3guard/LocalMetadataEntry.java | 81 +++ .../fs/s3a/s3guard/LocalMetadataStore.java | 249 ++++--- .../hadoop/fs/s3a/s3guard/S3GuardTool.java | 6 +- .../hadoop/fs/s3native/S3xLoginHelper.java | 4 +- ...rg.apache.hadoop.fs.MultipartUploaderFactory | 15 + .../org.apache.hadoop.fs.MultipartUploader | 16 + .../hadoop/fs/s3a/AbstractS3ATestBase.java | 21 +- .../hadoop/fs/s3a/ITestS3AConfiguration.java | 4 +- .../ITestS3AEncryptionSSEKMSUserDefinedKey.java | 2 +- ...onSSEKMSUserDefinedKeyBlockOutputStream.java | 2 +- .../apache/hadoop/fs/s3a/S3ATestConstants.java | 1 - .../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 36 +- .../fs/s3a/commit/AbstractCommitITest.java | 2 +- .../fs/s3a/commit/staging/StagingTestBase.java | 30 +- .../s3a/s3guard/DynamoDBLocalClientFactory.java | 160 ----- .../fs/s3a/s3guard/MetadataStoreTestBase.java | 6 +- .../s3a/s3guard/TestDynamoDBMetadataStore.java | 589 ---------------- .../fs/s3a/s3guard/TestLocalMetadataStore.java | 33 +- .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 2 +- .../hadoop/fs/adl/TestAzureADTokenProvider.java | 2 +- .../fs/azure/AzureNativeFileSystemStore.java | 2 +- .../hadoop/fs/azure/BlockBlobAppendStream.java | 2 +- .../fs/azure/ClientThrottlingAnalyzer.java | 2 +- .../hadoop/fs/azure/NativeAzureFileSystem.java | 4 +- .../hadoop/fs/azure/PageBlobOutputStream.java | 2 +- .../fs/azure/RemoteWasbAuthorizerImpl.java | 2 +- .../fs/azure/SecureWasbRemoteCallHelper.java | 2 +- .../hadoop/fs/azure/security/JsonUtils.java | 4 +- .../fs/azure/ITestBlobDataValidation.java | 2 +- .../azure/ITestNativeAzureFileSystemAppend.java | 2 +- .../hadoop/fs/azure/MockStorageInterface.java | 12 +- ...estNativeAzureFileSystemBlockCompaction.java | 2 +- .../fs/azure/integration/AzureTestUtils.java | 2 +- .../org/apache/hadoop/tools/DistCpOptions.java | 2 +- .../org/apache/hadoop/tools/OptionsParser.java | 2 +- .../apache/hadoop/tools/mapred/CopyMapper.java | 2 +- .../mapred/gridmix/ClusterSummarizer.java | 2 +- .../mapred/gridmix/ExecutionSummarizer.java | 4 +- .../hadoop/mapred/gridmix/JobFactory.java | 2 +- .../mapred/gridmix/RandomTextDataGenerator.java | 2 +- hadoop-tools/hadoop-ozone/pom.xml | 174 ----- .../org/apache/hadoop/fs/ozone/Constants.java | 42 -- .../java/org/apache/hadoop/fs/ozone/OzFs.java | 44 -- .../hadoop/fs/ozone/OzoneFSInputStream.java | 79 --- .../hadoop/fs/ozone/OzoneFSOutputStream.java | 59 -- .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 689 ------------------- .../apache/hadoop/fs/ozone/package-info.java | 30 - .../hadoop/fs/ozone/TestOzoneFSInputStream.java | 157 ----- .../fs/ozone/TestOzoneFileInterfaces.java | 231 ------- .../contract/ITestOzoneContractCreate.java | 48 -- .../contract/ITestOzoneContractDelete.java | 48 -- .../contract/ITestOzoneContractDistCp.java | 50 -- .../ITestOzoneContractGetFileStatus.java | 61 -- .../ozone/contract/ITestOzoneContractMkdir.java | 48 -- .../ozone/contract/ITestOzoneContractOpen.java | 47 -- .../contract/ITestOzoneContractRename.java | 49 -- .../contract/ITestOzoneContractRootDir.java | 51 -- .../ozone/contract/ITestOzoneContractSeek.java | 47 -- .../hadoop/fs/ozone/contract/OzoneContract.java | 123 ---- .../src/test/resources/contract/ozone.xml | 113 --- .../src/test/resources/log4j.properties | 23 - .../translator/impl/BaseLogParser.java | 1 - .../WordListAnonymizerUtility.java | 2 +- .../util/MapReduceJobPropertiesParser.java | 2 +- .../hadoop/streaming/TestUnconsumedInput.java | 2 +- hadoop-tools/hadoop-tools-dist/pom.xml | 15 - hadoop-tools/pom.xml | 11 - .../yarn/api/records/ResourceRequest.java | 20 + .../hadoop/yarn/conf/YarnConfiguration.java | 2 +- .../exceptions/RestApiErrorMessages.java | 2 + .../yarn/service/utils/ServiceApiUtil.java | 5 + .../hadoop/yarn/service/TestServiceApiUtil.java | 18 + .../yarn/client/api/impl/AMRMClientImpl.java | 11 +- .../api/impl/TestAMRMClientOnRMRestart.java | 6 +- .../yarn/client/api/impl/TestYarnClient.java | 337 ++------- .../client/api/impl/TestYarnClientImpl.java | 254 +++++++ .../hadoop-yarn/hadoop-yarn-common/pom.xml | 2 +- .../org/apache/hadoop/yarn/webapp/WebApps.java | 14 +- .../hadoop/yarn/webapp/view/JQueryUI.java | 2 +- .../jquery/jquery-ui-1.12.1.custom.min.js | 13 + .../static/jquery/jquery-ui-1.9.1.custom.min.js | 6 - .../registry/client/api/RegistryConstants.java | 2 +- .../hadoop/registry/server/dns/RegistryDNS.java | 29 +- .../registry/server/dns/TestRegistryDNS.java | 23 +- .../hadoop/yarn/server/AMRMClientRelayer.java | 8 +- .../LocalityMulticastAMRMProxyPolicy.java | 10 +- .../server/scheduler/ResourceRequestSet.java | 14 +- .../hadoop/yarn/server/utils/BuilderUtils.java | 12 - .../src/CMakeLists.txt | 4 +- .../server/nodemanager/ContainerExecutor.java | 4 +- .../containermanager/container/Container.java | 6 + .../container/ContainerImpl.java | 11 + .../launcher/ContainerLaunch.java | 46 +- .../launcher/ContainerRelaunch.java | 1 - .../CGroupsMemoryResourceHandlerImpl.java | 24 + .../linux/resources/MemoryResourceHandler.java | 10 + .../resources/gpu/GpuResourceAllocator.java | 68 +- .../resources/gpu/GpuResourceHandlerImpl.java | 1 - .../localizer/ContainerLocalizer.java | 7 +- .../localizer/ResourceLocalizationService.java | 5 +- .../monitor/ContainersMonitorImpl.java | 108 +-- .../oom-listener/test/oom_listener_test_main.cc | 33 +- .../container/TestContainer.java | 8 +- .../launcher/TestContainerLaunch.java | 2 - .../TestCGroupsMemoryResourceHandlerImpl.java | 43 ++ .../nodemanager/webapp/MockContainer.java | 3 + .../server/resourcemanager/ResourceManager.java | 2 +- .../scheduler/capacity/CSQueueUtils.java | 18 +- .../scheduler/capacity/CapacityScheduler.java | 18 +- .../fair/AllocationFileLoaderService.java | 4 +- .../LocalityAppPlacementAllocator.java | 10 +- .../webapp/dao/ClusterMetricsInfo.java | 2 +- .../server/resourcemanager/Application.java | 10 +- .../server/resourcemanager/TestAppManager.java | 18 +- .../src/site/markdown/FairScheduler.md | 2 +- .../site/markdown/yarn-service/RegistryDNS.md | 10 +- .../markdown/yarn-service/ServiceDiscovery.md | 6 +- .../hadoop-yarn-ui/src/main/webapp/.bowerrc | 3 +- .../src/main/webapp/WEB-INF/web.xml | 6 +- pom.xml | 2 +- start-build-env.sh | 8 +- 437 files changed, 7828 insertions(+), 7627 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/common/src/main/resources/ozone-default.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java ---------------------------------------------------------------------- diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java index b6a9bb9,b073d7b..245d76f --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java @@@ -93,8 -95,9 +95,9 @@@ public class DatanodeStateMachine imple // trick. commandDispatcher = CommandDispatcher.newBuilder() .addHandler(new CloseContainerCommandHandler()) - .addHandler(new DeleteBlocksCommandHandler( - container.getContainerManager(), conf)) + .addHandler(new DeleteBlocksCommandHandler(container.getContainerSet(), + conf)) + .addHandler(new ReplicateContainerCommandHandler()) .setConnectionManager(connectionManager) .setContainer(container) .setContext(context) http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java ---------------------------------------------------------------------- diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java index 6809d57,0000000..5d6fc0a mode 100644,000000..100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java @@@ -1,163 -1,0 +1,163 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.container.common.utils; + +import com.google.common.annotations.VisibleForTesting; - import org.apache.commons.lang.StringUtils; ++import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.common.InconsistentStorageStateException; +import org.apache.hadoop.ozone.container.common.DataNodeLayoutVersion; +import org.apache.hadoop.ozone.container.common.volume.HddsVolume; +import org.apache.hadoop.util.Time; + +import java.io.File; +import java.util.Properties; +import java.util.UUID; + +/** + * A util class for {@link HddsVolume}. + */ +public final class HddsVolumeUtil { + + // Private constructor for Utility class. Unused. + private HddsVolumeUtil() { + } + + private static final String VERSION_FILE = "VERSION"; + private static final String STORAGE_ID_PREFIX = "DS-"; + + public static File getVersionFile(File rootDir) { + return new File(rootDir, VERSION_FILE); + } + + public static String generateUuid() { + return STORAGE_ID_PREFIX + UUID.randomUUID(); + } + + /** + * Get hddsRoot from volume root. If volumeRoot points to hddsRoot, it is + * returned as is. + * For a volumeRoot /data/disk1, the hddsRoot is /data/disk1/hdds. + * @param volumeRoot root of the volume. + * @return hddsRoot of the volume. + */ + public static String getHddsRoot(String volumeRoot) { + if (volumeRoot.endsWith(HddsVolume.HDDS_VOLUME_DIR)) { + return volumeRoot; + } else { + File hddsRoot = new File(volumeRoot, HddsVolume.HDDS_VOLUME_DIR); + return hddsRoot.getPath(); + } + } + + /** + * Returns storageID if it is valid. Throws an exception otherwise. + */ + @VisibleForTesting + public static String getStorageID(Properties props, File versionFile) + throws InconsistentStorageStateException { + return getProperty(props, OzoneConsts.STORAGE_ID, versionFile); + } + + /** + * Returns clusterID if it is valid. It should match the clusterID from the + * Datanode. Throws an exception otherwise. + */ + @VisibleForTesting + public static String getClusterID(Properties props, File versionFile, + String clusterID) throws InconsistentStorageStateException { + String cid = getProperty(props, OzoneConsts.CLUSTER_ID, versionFile); + + if (clusterID == null) { + return cid; + } + if (!clusterID.equals(cid)) { + throw new InconsistentStorageStateException("Mismatched " + + "ClusterIDs. Version File : " + versionFile + " has clusterID: " + + cid + " and Datanode has clusterID: " + clusterID); + } + return cid; + } + + /** + * Returns datanodeUuid if it is valid. It should match the UUID of the + * Datanode. Throws an exception otherwise. + */ + @VisibleForTesting + public static String getDatanodeUUID(Properties props, File versionFile, + String datanodeUuid) + throws InconsistentStorageStateException { + String datanodeID = getProperty(props, OzoneConsts.DATANODE_UUID, + versionFile); + + if (datanodeUuid != null && !datanodeUuid.equals(datanodeID)) { + throw new InconsistentStorageStateException("Mismatched " + + "DatanodeUUIDs. Version File : " + versionFile + " has datanodeUuid: " + + datanodeID + " and Datanode has datanodeUuid: " + datanodeUuid); + } + return datanodeID; + } + + /** + * Returns creationTime if it is valid. Throws an exception otherwise. + */ + @VisibleForTesting + public static long getCreationTime(Properties props, File versionFile) + throws InconsistentStorageStateException { + String cTimeStr = getProperty(props, OzoneConsts.CTIME, versionFile); + + long cTime = Long.parseLong(cTimeStr); + long currentTime = Time.now(); + if (cTime > currentTime || cTime < 0) { + throw new InconsistentStorageStateException("Invalid Creation time in " + + "Version File : " + versionFile + " - " + cTime + ". Current system" + + " time is " + currentTime); + } + return cTime; + } + + /** + * Returns layOutVersion if it is valid. Throws an exception otherwise. + */ + @VisibleForTesting + public static int getLayOutVersion(Properties props, File versionFile) throws + InconsistentStorageStateException { + String lvStr = getProperty(props, OzoneConsts.LAYOUTVERSION, versionFile); + + int lv = Integer.parseInt(lvStr); + if(DataNodeLayoutVersion.getLatestVersion().getVersion() != lv) { + throw new InconsistentStorageStateException("Invalid layOutVersion. " + + "Version file has layOutVersion as " + lv + " and latest Datanode " + + "layOutVersion is " + + DataNodeLayoutVersion.getLatestVersion().getVersion()); + } + return lv; + } + + private static String getProperty(Properties props, String propName, File + versionFile) + throws InconsistentStorageStateException { + String value = props.getProperty(propName); + if (StringUtils.isBlank(value)) { + throw new InconsistentStorageStateException("Invalid " + propName + + ". Version File : " + versionFile + " has null or empty " + propName); + } + return value; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java ---------------------------------------------------------------------- diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index 9ac9930,adca8ea..ff5b9f1 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@@ -40,9 -44,9 +44,10 @@@ import org.apache.hadoop.hdds.protocol. .StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.metrics2.util.MBeans; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.protocol.StorageContainerNodeProtocol; import org.apache.hadoop.ozone.protocol.VersionResponse; + import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand; import org.apache.hadoop.ozone.protocol.commands.ReregisterCommand; import org.apache.hadoop.ozone.protocol.commands.SCMCommand; http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1045251/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
