Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9e45ed3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9e45ed3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9e45ed3 Branch: refs/heads/HDFS-7240 Commit: a9e45ed3ec73d4574eba70a3a51566cdb2eea19c Parents: 9176dd2 f63cd78 Author: Anu Engineer <aengin...@apache.org> Authored: Fri Oct 21 09:43:53 2016 -0700 Committer: Anu Engineer <aengin...@apache.org> Committed: Fri Oct 21 09:43:53 2016 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-auth/pom.xml | 42 + .../client/KerberosAuthenticator.java | 8 +- .../server/AuthenticationFilter.java | 47 +- .../server/AuthenticationHandler.java | 2 +- .../server/AuthenticationHandlerUtil.java | 105 +++ .../server/CompositeAuthenticationHandler.java | 30 + .../authentication/server/HttpConstants.java | 55 ++ .../server/LdapAuthenticationHandler.java | 339 ++++++++ .../MultiSchemeAuthenticationHandler.java | 209 +++++ .../authentication/server/package-info.java | 27 + .../src/site/markdown/Configuration.md | 137 +++ .../client/AuthenticatorTestCase.java | 49 +- .../client/TestKerberosAuthenticator.java | 71 +- .../authentication/server/LdapConstants.java | 31 + .../server/TestLdapAuthenticationHandler.java | 159 ++++ .../TestMultiSchemeAuthenticationHandler.java | 189 ++++ .../hadoop/fs/CommonConfigurationKeys.java | 26 - .../java/org/apache/hadoop/fs/FileSystem.java | 14 +- .../apache/hadoop/fs/RawLocalFileSystem.java | 5 +- .../apache/hadoop/fs/TrashPolicyDefault.java | 11 +- .../hadoop/fs/permission/FsPermission.java | 12 + .../java/org/apache/hadoop/fs/shell/Stat.java | 11 +- .../hadoop/fs/viewfs/ChRootedFileSystem.java | 5 + .../org/apache/hadoop/fs/viewfs/InodeTree.java | 206 ++--- .../apache/hadoop/fs/viewfs/ViewFileSystem.java | 99 +-- .../apache/hadoop/io/erasurecode/CodecUtil.java | 168 +++- .../io/erasurecode/ErasureCodeConstants.java | 3 +- .../io/erasurecode/ErasureCodecOptions.java | 37 + .../erasurecode/codec/AbstractErasureCodec.java | 53 -- .../io/erasurecode/codec/DummyErasureCodec.java | 45 + .../io/erasurecode/codec/ErasureCodec.java | 76 +- .../io/erasurecode/codec/HHXORErasureCodec.java | 20 +- .../io/erasurecode/codec/RSErasureCodec.java | 20 +- .../io/erasurecode/codec/XORErasureCodec.java | 22 +- .../io/erasurecode/codec/package-info.java | 28 + .../erasurecode/coder/AbstractErasureCoder.java | 64 -- .../coder/AbstractErasureCodingStep.java | 61 -- .../coder/AbstractErasureDecoder.java | 170 ---- .../coder/AbstractErasureEncoder.java | 62 -- .../coder/AbstractHHErasureCodingStep.java | 49 -- .../erasurecode/coder/DummyErasureDecoder.java | 46 + .../erasurecode/coder/DummyErasureEncoder.java | 45 + .../io/erasurecode/coder/ErasureCoder.java | 25 +- .../io/erasurecode/coder/ErasureCodingStep.java | 8 +- .../io/erasurecode/coder/ErasureDecoder.java | 198 +++++ .../erasurecode/coder/ErasureDecodingStep.java | 21 +- .../io/erasurecode/coder/ErasureEncoder.java | 91 ++ .../erasurecode/coder/ErasureEncodingStep.java | 22 +- .../erasurecode/coder/HHErasureCodingStep.java | 68 ++ .../erasurecode/coder/HHXORErasureDecoder.java | 24 +- .../coder/HHXORErasureDecodingStep.java | 2 +- .../erasurecode/coder/HHXORErasureEncoder.java | 19 +- .../coder/HHXORErasureEncodingStep.java | 2 +- .../io/erasurecode/coder/RSErasureDecoder.java | 16 +- .../io/erasurecode/coder/RSErasureEncoder.java | 20 +- .../io/erasurecode/coder/XORErasureDecoder.java | 15 +- .../io/erasurecode/coder/XORErasureEncoder.java | 16 +- .../io/erasurecode/coder/package-info.java | 28 + .../io/erasurecode/rawcoder/CoderUtil.java | 2 +- .../DelegationTokenAuthenticationFilter.java | 9 +- .../DelegationTokenAuthenticationHandler.java | 25 +- ...emeDelegationTokenAuthenticationHandler.java | 182 ++++ .../org/apache/hadoop/util/DiskChecker.java | 178 ++-- .../src/org/apache/hadoop/io/erasurecode/dump.c | 8 +- .../apache/hadoop/io/erasurecode/isal_load.h | 2 +- .../src/main/resources/core-default.xml | 74 +- .../src/site/markdown/ClusterSetup.md | 2 +- .../src/site/markdown/Compatibility.md | 16 +- .../src/site/markdown/FileSystemShell.md | 4 +- .../site/markdown/InterfaceClassification.md | 28 +- .../hadoop-common/src/site/markdown/Tracing.md | 2 +- .../src/site/markdown/filesystem/filesystem.md | 20 +- .../markdown/filesystem/fsdatainputstream.md | 16 +- .../site/markdown/filesystem/introduction.md | 12 +- .../src/site/markdown/filesystem/model.md | 7 +- .../src/site/markdown/filesystem/notation.md | 2 +- .../src/site/markdown/filesystem/testing.md | 4 +- .../conf/TestCommonConfigurationFields.java | 5 +- .../hadoop/fs/FSMainOperationsBaseTest.java | 24 +- .../java/org/apache/hadoop/fs/TestTrash.java | 352 +++++++- .../AbstractContractRootDirectoryTest.java | 48 +- .../hadoop/fs/contract/ContractTestUtils.java | 22 +- .../apache/hadoop/fs/shell/TestPathData.java | 19 + .../hadoop/fs/viewfs/TestViewFsConfig.java | 42 +- .../erasurecode/TestCodecRawCoderMapping.java | 3 +- .../codec/TestHHXORErasureCodec.java | 6 +- .../erasurecode/coder/TestErasureCoderBase.java | 13 +- .../coder/TestHHXORErasureCoder.java | 4 +- .../erasurecode/coder/TestRSErasureCoder.java | 4 +- .../org/apache/hadoop/test/LambdaTestUtils.java | 521 +++++++++++ .../apache/hadoop/test/TestLambdaTestUtils.java | 395 +++++++++ .../org/apache/hadoop/util/TestDiskChecker.java | 22 - .../src/test/resources/testConf.xml | 6 +- .../dev-support/findbugsExcludeFile.xml | 7 + .../key/kms/server/KMSAuthenticationFilter.java | 10 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 5 +- .../hadoop/hdfs/DistributedFileSystem.java | 5 +- .../hadoop/hdfs/client/impl/DfsClientConf.java | 4 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 8 + .../apache/hadoop/hdfs/web/TestTokenAspect.java | 6 +- .../hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java | 16 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 + .../java/org/apache/hadoop/hdfs/DFSUtil.java | 53 +- .../hadoop/hdfs/server/balancer/Dispatcher.java | 11 +- .../server/blockmanagement/BlockManager.java | 324 ++++--- .../BlockPlacementPolicyDefault.java | 4 +- .../CacheReplicationMonitor.java | 2 +- .../blockmanagement/CorruptReplicasMap.java | 4 +- .../blockmanagement/DatanodeDescriptor.java | 83 +- .../server/blockmanagement/DatanodeManager.java | 47 +- .../blockmanagement/DatanodeStorageInfo.java | 11 - .../blockmanagement/DecommissionManager.java | 142 ++- .../blockmanagement/ErasureCodingWork.java | 16 +- .../blockmanagement/HeartbeatManager.java | 23 +- .../blockmanagement/LowRedundancyBlocks.java | 47 +- .../server/blockmanagement/NumberReplicas.java | 30 +- .../blockmanagement/StorageTypeStats.java | 24 +- .../hdfs/server/datanode/BlockSender.java | 4 +- .../hadoop/hdfs/server/datanode/DataNode.java | 5 +- .../hdfs/server/datanode/DataNodeMXBean.java | 6 + .../datanode/fsdataset/impl/FsDatasetImpl.java | 33 +- .../datanode/fsdataset/impl/FsDatasetUtil.java | 32 + .../server/diskbalancer/command/Command.java | 21 +- .../diskbalancer/command/HelpCommand.java | 2 +- .../diskbalancer/command/PlanCommand.java | 55 +- .../diskbalancer/command/ReportCommand.java | 11 +- .../hdfs/server/namenode/FSNamesystem.java | 11 +- .../hadoop/hdfs/server/namenode/NameNode.java | 10 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 4 +- .../hdfs/server/namenode/NamenodeFsck.java | 14 +- .../apache/hadoop/hdfs/tools/DebugAdmin.java | 107 ++- .../hadoop/hdfs/tools/DiskBalancerCLI.java | 27 +- .../src/main/resources/hdfs-default.xml | 7 + .../src/site/markdown/HDFSCommands.md | 22 +- .../src/site/markdown/TransparentEncryption.md | 4 +- .../viewfs/TestViewFileSystemWithTruncate.java | 123 +++ .../apache/hadoop/hdfs/AdminStatesBaseTest.java | 20 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 72 +- .../apache/hadoop/hdfs/TestDFSPermission.java | 30 +- .../org/apache/hadoop/hdfs/TestDFSShell.java | 20 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 3 +- .../hadoop/hdfs/TestDFSStripedOutputStream.java | 6 +- .../TestDFSStripedOutputStreamWithFailure.java | 4 +- .../apache/hadoop/hdfs/TestDecommission.java | 2 +- .../hdfs/TestDecommissionWithStriped.java | 2 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 10 +- .../org/apache/hadoop/hdfs/TestFetchImage.java | 105 ++- .../apache/hadoop/hdfs/TestFileChecksum.java | 2 +- .../org/apache/hadoop/hdfs/TestHDFSTrash.java | 145 +++- .../hadoop/hdfs/TestMaintenanceState.java | 775 +++++++++++++++-- .../hadoop/hdfs/TestReconstructStripedFile.java | 4 +- .../hdfs/TestTrashWithEncryptionZones.java | 188 ++++ .../TestTrashWithSecureEncryptionZones.java | 443 ++++++++++ .../hdfs/server/balancer/TestBalancer.java | 61 +- .../blockmanagement/TestBlockManager.java | 27 +- .../TestNameNodePrunesMissingStorages.java | 70 +- .../TestPendingInvalidateBlock.java | 13 +- .../server/datanode/BlockReportTestBase.java | 50 ++ .../datanode/TestDataNodeHotSwapVolumes.java | 19 +- .../server/datanode/TestDataNodeMXBean.java | 6 +- .../diskbalancer/DiskBalancerTestUtil.java | 72 ++ .../server/diskbalancer/TestDiskBalancer.java | 4 +- .../command/TestDiskBalancerCommand.java | 61 +- .../hdfs/server/mover/TestStorageMover.java | 17 +- .../TestAddOverReplicatedStripedBlocks.java | 4 + .../namenode/TestDecommissioningStatus.java | 57 +- .../namenode/TestDiskspaceQuotaUpdate.java | 2 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 14 +- .../server/namenode/TestNameNodeMXBean.java | 105 +++ .../namenode/TestNamenodeCapacityReport.java | 78 +- .../namenode/TestNestedEncryptionZones.java | 175 ++-- .../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 2 +- .../hadoop/hdfs/tools/TestDebugAdmin.java | 56 +- .../hadoop/hdfs/util/HostsFileWriter.java | 1 + .../hadoop-mapreduce-client-jobclient/pom.xml | 4 - hadoop-project/pom.xml | 6 +- .../fs/aliyun/oss/AliyunOSSInputStream.java | 2 +- .../src/site/markdown/HadoopArchives.md.vm | 2 +- hadoop-tools/hadoop-aws/pom.xml | 58 +- .../s3a/BlockingThreadPoolExecutorService.java | 168 +--- .../org/apache/hadoop/fs/s3a/Constants.java | 71 +- .../hadoop/fs/s3a/S3ABlockOutputStream.java | 703 +++++++++++++++ .../org/apache/hadoop/fs/s3a/S3ADataBlocks.java | 821 ++++++++++++++++++ .../hadoop/fs/s3a/S3AFastOutputStream.java | 410 --------- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 408 ++++++++- .../hadoop/fs/s3a/S3AInstrumentation.java | 255 +++++- .../apache/hadoop/fs/s3a/S3AOutputStream.java | 57 +- .../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 39 + .../fs/s3a/SemaphoredDelegatingExecutor.java | 230 +++++ .../org/apache/hadoop/fs/s3a/Statistic.java | 32 +- .../src/site/markdown/tools/hadoop-aws/index.md | 668 +++++++++++++- .../fs/contract/s3a/ITestS3AContractDistCp.java | 10 +- .../hadoop/fs/s3a/AbstractS3ATestBase.java | 1 + .../ITestBlockingThreadPoolExecutorService.java | 48 +- .../hadoop/fs/s3a/ITestS3ABlockOutputArray.java | 90 ++ .../fs/s3a/ITestS3ABlockOutputByteBuffer.java | 30 + .../hadoop/fs/s3a/ITestS3ABlockOutputDisk.java | 30 + .../fs/s3a/ITestS3ABlockingThreadPool.java | 2 + .../hadoop/fs/s3a/ITestS3AConfiguration.java | 29 + .../ITestS3AEncryptionBlockOutputStream.java | 36 + .../s3a/ITestS3AEncryptionFastOutputStream.java | 35 - .../hadoop/fs/s3a/ITestS3AFailureHandling.java | 20 +- .../hadoop/fs/s3a/ITestS3AFastOutputStream.java | 74 -- .../apache/hadoop/fs/s3a/ITestS3ATestUtils.java | 98 +++ .../apache/hadoop/fs/s3a/S3ATestConstants.java | 75 +- .../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 182 +++- .../apache/hadoop/fs/s3a/TestDataBlocks.java | 124 +++ .../ITestS3AFileContextStatistics.java | 1 + .../fs/s3a/scale/AbstractSTestS3AHugeFiles.java | 412 +++++++++ .../fs/s3a/scale/ITestS3ADeleteManyFiles.java | 19 +- .../s3a/scale/ITestS3AHugeFilesArrayBlocks.java | 31 + .../ITestS3AHugeFilesByteBufferBlocks.java | 34 + .../scale/ITestS3AHugeFilesClassicOutput.java | 41 + .../s3a/scale/ITestS3AHugeFilesDiskBlocks.java | 31 + .../hadoop/fs/s3a/scale/S3AScaleTestBase.java | 151 ++-- .../java/org/apache/hadoop/tools/DiffInfo.java | 47 +- .../java/org/apache/hadoop/tools/DistCp.java | 37 +- .../apache/hadoop/tools/DistCpConstants.java | 1 + .../apache/hadoop/tools/DistCpOptionSwitch.java | 5 + .../org/apache/hadoop/tools/DistCpOptions.java | 79 +- .../org/apache/hadoop/tools/DistCpSync.java | 256 ++++-- .../org/apache/hadoop/tools/OptionsParser.java | 27 +- .../apache/hadoop/tools/SimpleCopyListing.java | 17 +- .../org/apache/hadoop/tools/TestDistCpSync.java | 4 +- .../hadoop/tools/TestDistCpSyncReverseBase.java | 868 +++++++++++++++++++ .../tools/TestDistCpSyncReverseFromSource.java | 36 + .../tools/TestDistCpSyncReverseFromTarget.java | 36 + .../apache/hadoop/tools/TestDistCpWithAcls.java | 2 +- .../hadoop/tools/TestDistCpWithXAttrs.java | 2 +- .../apache/hadoop/tools/TestOptionsParser.java | 85 +- .../hadoop/yarn/conf/YarnConfiguration.java | 17 +- .../api/impl/FileSystemTimelineWriter.java | 7 - .../nodelabels/FileSystemNodeLabelsStore.java | 7 +- .../src/main/resources/yarn-default.xml | 20 - ...pplicationHistoryManagerOnTimelineStore.java | 69 +- ...pplicationHistoryManagerOnTimelineStore.java | 38 +- .../metrics/AppAttemptMetricsConstants.java | 16 +- .../metrics/ApplicationMetricsConstants.java | 16 +- .../metrics/ContainerMetricsConstants.java | 21 +- .../nodemanager/DefaultContainerExecutor.java | 272 +++++- .../WindowsSecureContainerExecutor.java | 2 +- .../timelineservice/NMTimelinePublisher.java | 34 +- .../impl/container-executor.c | 11 +- .../impl/container-executor.h | 4 + .../main/native/container-executor/impl/main.c | 42 +- .../test/test-container-executor.c | 51 ++ .../metrics/TimelineServiceV1Publisher.java | 44 +- .../metrics/TimelineServiceV2Publisher.java | 148 ++-- .../recovery/FileSystemRMStateStore.java | 5 - .../scheduler/AbstractYarnScheduler.java | 186 +++- .../scheduler/capacity/CapacityScheduler.java | 122 +-- .../scheduler/fair/FairScheduler.java | 86 +- .../scheduler/fifo/FifoScheduler.java | 94 +- .../metrics/TestSystemMetricsPublisher.java | 40 +- .../TestSystemMetricsPublisherForV2.java | 18 +- ...estProportionalCapacityPreemptionPolicy.java | 4 +- .../recovery/TestFSRMStateStore.java | 4 - .../scheduler/fair/TestFairScheduler.java | 28 + .../fair/TestQueuePlacementPolicy.java | 9 +- .../reader/TimelineReaderWebServices.java | 739 ++++++++++++++++ .../reader/TestTimelineReaderWebServices.java | 185 ++++ .../TestFileSystemTimelineReaderImpl.java | 44 +- 262 files changed, 15432 insertions(+), 3613 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9e45ed3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
