Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ac9cc8a8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ac9cc8a8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ac9cc8a8 Branch: refs/heads/HDFS-7240 Commit: ac9cc8a8c32a459c6a178bdadfd767ab080e76be Parents: b72799a 30941d9 Author: Anu Engineer <aengin...@apache.org> Authored: Tue Nov 28 13:17:33 2017 -0800 Committer: Anu Engineer <aengin...@apache.org> Committed: Tue Nov 28 13:17:33 2017 -0800 ---------------------------------------------------------------------- .../dev-support/findbugsExcludeFile.xml | 7 + .../apache/hadoop/fs/FSDataOutputStream.java | 9 + .../apache/hadoop/fs/PathExistsException.java | 4 +- .../org/apache/hadoop/fs/StorageStatistics.java | 5 + .../hadoop/metrics2/source/JvmMetrics.java | 2 +- .../apache/hadoop/util/JsonSerialization.java | 299 +++ .../src/main/resources/core-default.xml | 117 +- .../hadoop/fs/contract/ContractTestUtils.java | 51 +- .../hadoop/metrics2/source/TestJvmMetrics.java | 2 +- .../apache/hadoop/test/GenericTestUtils.java | 29 +- .../org/apache/hadoop/test/HadoopTestBase.java | 51 +- .../org/apache/hadoop/test/LambdaTestUtils.java | 144 +- .../hadoop/util/TestJsonSerialization.java | 185 ++ .../java/org/apache/hadoop/hdfs/DFSClient.java | 15 +- .../hadoop/hdfs/DFSClientFaultInjector.java | 2 + .../hadoop/hdfs/client/impl/LeaseRenewer.java | 55 +- .../hdfs/client/impl/TestLeaseRenewer.java | 23 +- .../hadoop/hdfs/tools/StoragePolicyAdmin.java | 6 +- .../src/site/markdown/HDFSCommands.md | 23 + .../hadoop/hdfs/TestDFSClientRetries.java | 80 + .../hadoop/hdfs/TestDistributedFileSystem.java | 5 +- .../tools/TestViewFSStoragePolicyCommands.java | 37 +- .../hadoop-hdfs/src/test/resources/editsStored | Bin 6293 -> 6753 bytes .../src/test/resources/editsStored.xml | 750 ++++--- .../mapreduce/TestMapreduceConfigFields.java | 27 +- .../lib/output/BindingPathOutputCommitter.java | 184 ++ .../lib/output/FileOutputCommitter.java | 12 +- .../lib/output/FileOutputCommitterFactory.java | 38 + .../mapreduce/lib/output/FileOutputFormat.java | 10 +- .../lib/output/NamedCommitterFactory.java | 79 + .../lib/output/PathOutputCommitter.java | 17 + .../lib/output/PathOutputCommitterFactory.java | 204 ++ .../src/main/resources/mapred-default.xml | 22 + .../TestClientDistributedCacheManager.java | 28 +- .../lib/output/TestPathOutputCommitter.java | 24 +- .../output/TestPathOutputCommitterFactory.java | 495 +++++ hadoop-tools/hadoop-aws/pom.xml | 46 +- .../hadoop/fs/s3a/AWSBadRequestException.java | 42 + .../hadoop/fs/s3a/AWSClientIOException.java | 3 +- .../hadoop/fs/s3a/AWSNoResponseException.java | 31 + .../hadoop/fs/s3a/AWSRedirectException.java | 38 + .../fs/s3a/AWSServiceThrottledException.java | 42 + .../hadoop/fs/s3a/AWSStatus500Exception.java | 37 + .../s3a/BlockingThreadPoolExecutorService.java | 2 +- .../org/apache/hadoop/fs/s3a/Constants.java | 72 +- .../fs/s3a/InconsistentAmazonS3Client.java | 232 ++- .../java/org/apache/hadoop/fs/s3a/Invoker.java | 487 +++++ .../java/org/apache/hadoop/fs/s3a/Listing.java | 26 +- .../java/org/apache/hadoop/fs/s3a/Retries.java | 92 + .../hadoop/fs/s3a/S3ABlockOutputStream.java | 307 +-- .../org/apache/hadoop/fs/s3a/S3ADataBlocks.java | 2 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 940 +++++---- .../apache/hadoop/fs/s3a/S3AInputStream.java | 56 +- .../hadoop/fs/s3a/S3AInstrumentation.java | 231 ++- .../apache/hadoop/fs/s3a/S3ARetryPolicy.java | 246 +++ .../hadoop/fs/s3a/S3AStorageStatistics.java | 12 +- .../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 328 ++- .../org/apache/hadoop/fs/s3a/S3ListRequest.java | 11 + .../hadoop/fs/s3a/S3ObjectAttributes.java | 10 +- .../org/apache/hadoop/fs/s3a/Statistic.java | 56 +- .../hadoop/fs/s3a/WriteOperationHelper.java | 475 +++++ .../fs/s3a/commit/AbstractS3ACommitter.java | 757 +++++++ .../s3a/commit/AbstractS3ACommitterFactory.java | 90 + .../hadoop/fs/s3a/commit/CommitConstants.java | 240 +++ .../hadoop/fs/s3a/commit/CommitOperations.java | 596 ++++++ .../hadoop/fs/s3a/commit/CommitUtils.java | 129 ++ .../hadoop/fs/s3a/commit/CommitUtilsWithMR.java | 192 ++ .../apache/hadoop/fs/s3a/commit/Duration.java | 60 + .../hadoop/fs/s3a/commit/DurationInfo.java | 59 + .../s3a/commit/InternalCommitterConstants.java | 100 + .../hadoop/fs/s3a/commit/LocalTempDir.java | 80 + .../fs/s3a/commit/MagicCommitIntegration.java | 182 ++ .../hadoop/fs/s3a/commit/MagicCommitPaths.java | 229 ++ .../fs/s3a/commit/PathCommitException.java | 43 + .../apache/hadoop/fs/s3a/commit/PutTracker.java | 100 + .../fs/s3a/commit/S3ACommitterFactory.java | 129 ++ .../org/apache/hadoop/fs/s3a/commit/Tasks.java | 410 ++++ .../hadoop/fs/s3a/commit/ValidationFailure.java | 53 + .../hadoop/fs/s3a/commit/files/PendingSet.java | 192 ++ .../s3a/commit/files/PersistentCommitData.java | 69 + .../s3a/commit/files/SinglePendingCommit.java | 432 ++++ .../hadoop/fs/s3a/commit/files/SuccessData.java | 322 +++ .../fs/s3a/commit/files/package-info.java | 45 + .../fs/s3a/commit/magic/MagicCommitTracker.java | 161 ++ .../s3a/commit/magic/MagicS3GuardCommitter.java | 288 +++ .../magic/MagicS3GuardCommitterFactory.java | 47 + .../fs/s3a/commit/magic/package-info.java | 27 + .../hadoop/fs/s3a/commit/package-info.java | 28 + .../s3a/commit/staging/ConflictResolution.java | 33 + .../staging/DirectoryStagingCommitter.java | 116 ++ .../DirectoryStagingCommitterFactory.java | 48 + .../staging/PartitionedStagingCommitter.java | 159 ++ .../PartitionedStagingCommitterFactory.java | 48 + .../hadoop/fs/s3a/commit/staging/Paths.java | 300 +++ .../fs/s3a/commit/staging/StagingCommitter.java | 852 ++++++++ .../staging/StagingCommitterConstants.java | 64 + .../commit/staging/StagingCommitterFactory.java | 49 + .../fs/s3a/commit/staging/package-info.java | 27 + .../fs/s3a/s3guard/DynamoDBMetadataStore.java | 368 ++-- .../apache/hadoop/fs/s3a/s3guard/S3Guard.java | 2 + .../hadoop/fs/s3a/s3guard/S3GuardTool.java | 16 +- .../tools/hadoop-aws/committer_architecture.md | 1951 ++++++++++++++++++ .../markdown/tools/hadoop-aws/committers.md | 819 ++++++++ .../src/site/markdown/tools/hadoop-aws/index.md | 188 +- .../site/markdown/tools/hadoop-aws/s3guard.md | 7 +- .../site/markdown/tools/hadoop-aws/testing.md | 60 + .../tools/hadoop-aws/troubleshooting_s3a.md | 124 ++ .../s3a/ITestS3AContractGetFileStatus.java | 11 + .../hadoop/fs/s3a/AbstractS3AMockTest.java | 5 +- .../hadoop/fs/s3a/AbstractS3ATestBase.java | 23 +- .../hadoop/fs/s3a/ITestS3AConfiguration.java | 8 +- .../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 6 +- .../hadoop/fs/s3a/ITestS3AFailureHandling.java | 49 +- .../fs/s3a/ITestS3AFileOperationCost.java | 2 + .../fs/s3a/ITestS3ATemporaryCredentials.java | 15 +- .../fs/s3a/ITestS3GuardListConsistency.java | 45 +- .../apache/hadoop/fs/s3a/MockS3AFileSystem.java | 322 +++ .../hadoop/fs/s3a/MockS3ClientFactory.java | 8 + .../apache/hadoop/fs/s3a/S3ATestConstants.java | 6 + .../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 132 +- .../hadoop/fs/s3a/StorageStatisticsTracker.java | 99 + .../org/apache/hadoop/fs/s3a/TestInvoker.java | 460 +++++ .../org/apache/hadoop/fs/s3a/TestListing.java | 2 +- .../fs/s3a/TestS3AExceptionTranslation.java | 84 +- .../fs/s3a/commit/AbstractCommitITest.java | 412 ++++ .../fs/s3a/commit/AbstractITCommitMRJob.java | 324 +++ .../fs/s3a/commit/AbstractITCommitProtocol.java | 1371 ++++++++++++ .../fs/s3a/commit/CommitterFaultInjection.java | 44 + .../s3a/commit/CommitterFaultInjectionImpl.java | 131 ++ .../fs/s3a/commit/ITestCommitOperations.java | 545 +++++ .../fs/s3a/commit/LoggingTextOutputFormat.java | 133 ++ .../fs/s3a/commit/MiniDFSClusterService.java | 79 + .../fs/s3a/commit/TestMagicCommitPaths.java | 246 +++ .../apache/hadoop/fs/s3a/commit/TestTasks.java | 550 +++++ .../fs/s3a/commit/magic/ITMagicCommitMRJob.java | 70 + .../commit/magic/ITestMagicCommitProtocol.java | 190 ++ .../commit/magic/ITestS3AHugeMagicCommits.java | 195 ++ .../commit/staging/MockedStagingCommitter.java | 98 + .../staging/PartitionedCommitterForTesting.java | 58 + .../fs/s3a/commit/staging/StagingTestBase.java | 724 +++++++ .../hadoop/fs/s3a/commit/staging/TestPaths.java | 127 ++ .../commit/staging/TestStagingCommitter.java | 696 +++++++ .../TestStagingDirectoryOutputCommitter.java | 138 ++ .../TestStagingPartitionedFileListing.java | 186 ++ .../TestStagingPartitionedJobCommit.java | 236 +++ .../TestStagingPartitionedTaskCommit.java | 237 +++ .../integration/ITDirectoryCommitMRJob.java | 33 + .../integration/ITPartitionCommitMRJob.java | 33 + .../integration/ITStagingCommitMRJob.java | 66 + .../ITestDirectoryCommitProtocol.java | 131 ++ .../ITestPartitionedCommitProtocol.java | 139 ++ .../integration/ITestStagingCommitProtocol.java | 190 ++ .../s3guard/AbstractS3GuardToolTestBase.java | 23 + .../s3a/s3guard/TestDynamoDBMetadataStore.java | 13 +- .../fs/s3a/scale/AbstractSTestS3AHugeFiles.java | 191 +- .../hadoop-aws/src/test/resources/core-site.xml | 9 +- .../src/test/resources/log4j.properties | 42 +- .../hadoop/fs/azure/PageBlobInputStream.java | 4 +- .../dev-support/findbugs-exclude.xml | 8 + .../api/protocolrecords/AllocateResponse.java | 54 + .../hadoop/yarn/api/records/NodeReport.java | 47 +- .../hadoop/yarn/api/records/NodeUpdateType.java | 29 + .../hadoop/yarn/api/records/Resource.java | 21 + .../hadoop/yarn/conf/YarnConfiguration.java | 6 + .../src/main/proto/yarn_protos.proto | 8 + .../src/main/proto/yarn_service_protos.proto | 1 + .../hadoop/yarn/client/ProtocolHATestBase.java | 14 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 2 +- .../impl/pb/AllocateResponsePBImpl.java | 37 + .../api/records/impl/pb/NodeReportPBImpl.java | 50 +- .../yarn/api/records/impl/pb/ProtoUtils.java | 12 + .../src/main/resources/yarn-default.xml | 8 + .../registry/client/binding/JsonSerDeser.java | 224 +- .../hadoop/yarn/server/utils/BuilderUtils.java | 14 +- .../nodemanager/LinuxContainerExecutor.java | 28 +- .../WindowsSecureContainerExecutor.java | 2 +- .../resources/gpu/GpuResourceAllocator.java | 19 +- .../resources/gpu/GpuResourceHandlerImpl.java | 1 - .../runtime/DockerLinuxContainerRuntime.java | 42 + .../linux/runtime/docker/DockerRunCommand.java | 12 + .../localizer/ContainerLocalizer.java | 46 +- .../resourceplugin/ResourcePlugin.java | 11 + .../resourceplugin/gpu/AssignedGpuDevice.java | 79 + .../resourceplugin/gpu/GpuDevice.java | 4 +- .../resourceplugin/gpu/GpuResourcePlugin.java | 24 +- .../nodemanager/webapp/NMWebServices.java | 28 + .../nodemanager/webapp/dao/NMResourceInfo.java | 28 + .../webapp/dao/gpu/GpuDeviceInformation.java | 2 +- .../webapp/dao/gpu/NMGpuResourceInfo.java | 71 + .../webapp/dao/gpu/PerGpuDeviceInformation.java | 2 +- .../webapp/dao/gpu/PerGpuMemoryUsage.java | 2 +- .../TestLinuxContainerExecutorWithMocks.java | 19 +- .../resources/gpu/TestGpuResourceHandler.java | 6 +- .../runtime/TestDockerContainerRuntime.java | 109 + .../nodemanager/webapp/TestNMWebServices.java | 185 +- .../dao/gpu/TestGpuDeviceInformationParser.java | 2 +- .../server/resourcemanager/ClientRMService.java | 5 +- .../DecommissioningNodesWatcher.java | 38 +- .../resourcemanager/DefaultAMSProcessor.java | 15 +- .../resourcemanager/NodesListManager.java | 78 +- .../NodesListManagerEventType.java | 3 +- .../server/resourcemanager/rmapp/RMApp.java | 10 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 11 +- .../rmapp/RMAppNodeUpdateEvent.java | 9 +- .../server/resourcemanager/rmnode/RMNode.java | 2 +- .../resourcemanager/rmnode/RMNodeImpl.java | 5 + .../scheduler/AbstractYarnScheduler.java | 4 +- .../resourcemanager/scheduler/Allocation.java | 13 +- .../scheduler/SchedulerApplicationAttempt.java | 48 + .../scheduler/capacity/CapacityScheduler.java | 16 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 5 +- .../scheduler/fair/FSAppAttempt.java | 37 +- .../scheduler/fair/FSPreemptionThread.java | 68 +- .../scheduler/fair/FairScheduler.java | 11 +- .../fair/FairSchedulerConfiguration.java | 16 - .../scheduler/fair/FairSchedulerEventLog.java | 152 -- .../RMDelegationTokenSecretManager.java | 42 +- .../yarn/server/resourcemanager/MockRM.java | 15 + .../resourcemanager/TestClientRMService.java | 50 + .../TestDecommissioningNodesWatcher.java | 4 +- .../TestRMHAForAsyncScheduler.java | 155 ++ .../resourcemanager/TestRMNodeTransitions.java | 13 +- .../TestResourceTrackerService.java | 116 +- .../applicationsmanager/MockAsm.java | 4 +- .../TestAMRMRPCNodeUpdates.java | 51 + .../applicationsmanager/TestAMRestart.java | 149 ++ .../server/resourcemanager/rmapp/MockRMApp.java | 4 +- .../fair/TestFairSchedulerEventLog.java | 83 - .../fair/TestFairSchedulerPreemption.java | 37 +- .../src/site/markdown/DockerContainers.md | 48 + .../src/main/webapp/app/adapters/yarn-nm-gpu.js | 33 + .../main/webapp/app/components/donut-chart.js | 18 +- .../app/components/em-table-tooltip-text.js | 33 + .../webapp/app/components/gpu-donut-chart.js | 66 + .../src/main/webapp/app/constants.js | 13 + .../webapp/app/controllers/app-table-columns.js | 8 +- .../src/main/webapp/app/controllers/yarn-app.js | 69 +- .../webapp/app/controllers/yarn-flowrun/info.js | 2 +- .../webapp/app/controllers/yarn-nodes/table.js | 2 +- .../main/webapp/app/models/cluster-metric.js | 69 + .../src/main/webapp/app/models/yarn-app.js | 6 +- .../src/main/webapp/app/models/yarn-nm-gpu.js | 27 + .../app/models/yarn-queue/capacity-queue.js | 3 +- .../src/main/webapp/app/models/yarn-rm-node.js | 35 + .../src/main/webapp/app/router.js | 17 +- .../main/webapp/app/routes/cluster-overview.js | 2 +- .../src/main/webapp/app/routes/yarn-app.js | 23 +- .../main/webapp/app/routes/yarn-app/attempts.js | 15 +- .../main/webapp/app/routes/yarn-app/charts.js | 18 +- .../webapp/app/routes/yarn-app/components.js | 16 +- .../main/webapp/app/routes/yarn-app/configs.js | 16 +- .../src/main/webapp/app/routes/yarn-app/info.js | 17 +- .../src/main/webapp/app/routes/yarn-node.js | 2 + .../webapp/app/routes/yarn-node/yarn-nm-gpu.js | 22 + .../src/main/webapp/app/serializers/yarn-app.js | 2 +- .../main/webapp/app/serializers/yarn-nm-gpu.js | 43 + .../serializers/yarn-queue/capacity-queue.js | 1 + .../main/webapp/app/serializers/yarn-rm-node.js | 4 +- .../src/main/webapp/app/styles/app.scss | 24 + .../src/main/webapp/app/styles/colors.scss | 2 + .../src/main/webapp/app/styles/layout.scss | 42 + .../src/main/webapp/app/styles/variables.scss | 4 + .../src/main/webapp/app/styles/yarn-app.scss | 35 + .../webapp/app/templates/cluster-overview.hbs | 88 +- .../components/em-table-tooltip-text.hbs | 26 + .../templates/components/node-menu-panel.hbs | 10 +- .../app/templates/components/timeline-view.hbs | 2 +- .../templates/components/yarn-nm-gpu-info.hbs | 69 + .../src/main/webapp/app/templates/yarn-app.hbs | 149 +- .../webapp/app/templates/yarn-app/attempts.hbs | 2 +- .../webapp/app/templates/yarn-app/charts.hbs | 46 +- .../app/templates/yarn-app/components.hbs | 6 +- .../webapp/app/templates/yarn-app/configs.hbs | 58 +- .../main/webapp/app/templates/yarn-app/info.hbs | 281 +-- .../webapp/app/templates/yarn-app/loading.hbs | 2 +- .../src/main/webapp/app/templates/yarn-node.hbs | 125 -- .../webapp/app/templates/yarn-node/info.hbs | 154 ++ .../app/templates/yarn-node/yarn-nm-gpu.hbs | 53 + .../main/webapp/app/templates/yarn-services.hbs | 2 +- .../src/main/webapp/app/utils/converter.js | 51 + .../components/em-table-tooltip-text-test.js | 43 + 281 files changed, 27954 insertions(+), 2573 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac9cc8a8/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
