Merge branch 'trunk' into HDFS-1312
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9af2370 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9af2370 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9af2370 Branch: refs/heads/HDFS-1312 Commit: a9af23708827848d841d90c5d3c8bacefc50a3d2 Parents: 1bffbf6 89022f8 Author: Arpit Agarwal <[email protected]> Authored: Thu Jan 7 14:40:23 2016 -0800 Committer: Arpit Agarwal <[email protected]> Committed: Thu Jan 7 14:40:23 2016 -0800 ---------------------------------------------------------------------- .../authentication/util/KerberosUtil.java | 87 +- .../authentication/util/TestKerberosUtil.java | 31 +- hadoop-common-project/hadoop-common/CHANGES.txt | 117 +- hadoop-common-project/hadoop-common/pom.xml | 17 +- .../hadoop-common/src/CMakeLists.txt | 3 +- .../crypto/key/kms/KMSClientProvider.java | 14 +- .../hadoop/fs/CommonConfigurationKeys.java | 5 + .../fs/CommonConfigurationKeysPublic.java | 6 + .../java/org/apache/hadoop/fs/FileSystem.java | 104 +- .../org/apache/hadoop/fs/FilterFileSystem.java | 11 + .../main/java/org/apache/hadoop/fs/FsShell.java | 12 +- .../main/java/org/apache/hadoop/fs/Trash.java | 14 +- .../java/org/apache/hadoop/fs/TrashPolicy.java | 56 +- .../apache/hadoop/fs/TrashPolicyDefault.java | 193 ++- .../apache/hadoop/fs/shell/CopyCommands.java | 40 +- .../org/apache/hadoop/http/HttpServer2.java | 8 +- .../main/java/org/apache/hadoop/io/IOUtils.java | 5 +- .../rawcoder/AbstractRawErasureDecoder.java | 55 +- .../rawcoder/AbstractRawErasureEncoder.java | 56 +- .../erasurecode/rawcoder/RawErasureDecoder.java | 5 +- .../erasurecode/rawcoder/RawErasureEncoder.java | 5 +- .../main/java/org/apache/hadoop/ipc/Client.java | 4 +- .../apache/hadoop/ipc/DecayRpcScheduler.java | 2 +- .../main/java/org/apache/hadoop/ipc/RPC.java | 19 +- .../apache/hadoop/net/unix/DomainSocket.java | 59 +- .../hadoop/security/NullGroupsMapping.java | 50 + .../apache/hadoop/security/SecurityUtil.java | 6 + .../hadoop/security/UserGroupInformation.java | 36 + .../org/apache/hadoop/util/JvmPauseMonitor.java | 50 +- .../hadoop/util/NativeLibraryChecker.java | 10 +- .../org/apache/hadoop/util/StringUtils.java | 19 +- .../java/org/apache/hadoop/util/ThreadUtil.java | 30 + .../org/apache/hadoop/util/VersionInfo.java | 6 +- .../src/main/resources/core-default.xml | 39 + .../src/site/markdown/FileSystemShell.md | 31 +- .../src/site/markdown/HttpAuthentication.md | 52 +- .../src/site/markdown/SecureMode.md | 303 ++-- .../src/site/markdown/SingleCluster.md.vm | 4 +- .../conf/TestConfigurationFieldsBase.java | 14 +- .../hadoop/fs/TestFileSystemInitialization.java | 16 + .../org/apache/hadoop/fs/TestFsShellCopy.java | 24 +- .../org/apache/hadoop/fs/TestHarFileSystem.java | 4 + .../java/org/apache/hadoop/fs/TestTrash.java | 9 + .../apache/hadoop/fs/shell/find/TestFind.java | 53 +- .../org/apache/hadoop/http/TestHttpServer.java | 3 - .../apache/hadoop/http/TestHttpServerLogs.java | 80 + .../org/apache/hadoop/io/TestSequenceFile.java | 17 +- .../io/erasurecode/rawcoder/TestRSRawCoder.java | 7 + .../erasurecode/rawcoder/TestRawCoderBase.java | 34 + .../ipc/TestProtoBufRPCCompatibility.java | 178 ++ .../java/org/apache/hadoop/ipc/TestRPC.java | 30 + .../metrics2/impl/TestGangliaMetrics.java | 10 +- .../metrics2/impl/TestMetricsSystemImpl.java | 1 + .../hadoop/metrics2/source/TestJvmMetrics.java | 70 +- .../hadoop/security/TestNullGroupsMapping.java | 60 + .../apache/hadoop/test/GenericTestUtils.java | 54 +- .../hadoop-common/src/test/proto/test.proto | 8 + .../src/test/proto/test_rpc_service.proto | 15 + .../src/test/resources/testConf.xml | 7 +- .../hadoop-kms/src/main/conf/kms-site.xml | 6 +- .../hadoop/crypto/key/kms/server/MiniKMS.java | 28 +- .../hadoop/crypto/key/kms/server/TestKMS.java | 21 +- .../crypto/key/kms/server/TestKMSAudit.java | 14 +- .../java/org/apache/hadoop/minikdc/MiniKdc.java | 54 +- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 1 + .../apache/hadoop/hdfs/BlockReaderFactory.java | 2 +- .../org/apache/hadoop/hdfs/DFSOutputStream.java | 28 +- .../hadoop/hdfs/DFSStripedOutputStream.java | 47 +- .../org/apache/hadoop/hdfs/DataStreamer.java | 1 + .../hadoop/hdfs/DistributedFileSystem.java | 62 + .../org/apache/hadoop/hdfs/protocol/Block.java | 7 +- .../hdfs/protocol/LocatedStripedBlock.java | 10 +- .../hdfs/protocol/ReconfigurationProtocol.java | 54 + .../ClientDatanodeProtocolTranslatorPB.java | 46 +- .../hadoop/hdfs/protocolPB/PBHelperClient.java | 43 +- .../protocolPB/ReconfigurationProtocolPB.java | 43 + .../ReconfigurationProtocolTranslatorPB.java | 146 ++ .../ReconfigurationProtocolUtils.java | 64 + .../hadoop/hdfs/protocolPB/package-info.java | 18 + .../hdfs/shortcircuit/DfsClientShmManager.java | 2 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 365 ++++- .../src/main/proto/ClientDatanodeProtocol.proto | 32 +- .../main/proto/ReconfigurationProtocol.proto | 74 + .../src/main/proto/erasurecoding.proto | 2 +- .../src/main/proto/hdfs.proto | 2 +- .../hadoop-hdfs-native-client/pom.xml | 5 +- .../src/CMakeLists.txt | 5 - .../src/contrib/libwebhdfs/CMakeLists.txt | 88 - .../libwebhdfs/resources/FindJansson.cmake | 43 - .../contrib/libwebhdfs/src/hdfs_http_client.c | 490 ------ .../contrib/libwebhdfs/src/hdfs_http_client.h | 294 ---- .../contrib/libwebhdfs/src/hdfs_http_query.c | 402 ----- .../contrib/libwebhdfs/src/hdfs_http_query.h | 240 --- .../contrib/libwebhdfs/src/hdfs_json_parser.c | 654 -------- .../contrib/libwebhdfs/src/hdfs_json_parser.h | 178 -- .../src/contrib/libwebhdfs/src/hdfs_web.c | 1538 ------------------ .../libwebhdfs/src/test_libwebhdfs_ops.c | 552 ------- .../libwebhdfs/src/test_libwebhdfs_read.c | 78 - .../libwebhdfs/src/test_libwebhdfs_threaded.c | 247 --- .../libwebhdfs/src/test_libwebhdfs_write.c | 111 -- .../src/main/native/fuse-dfs/doc/README | 26 +- .../main/native/fuse-dfs/fuse_dfs_wrapper.sh | 34 +- .../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 3 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 213 ++- .../bkjournal/TestBootstrapStandbyWithBKJM.java | 1 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +- ...tDatanodeProtocolServerSideTranslatorPB.java | 58 +- .../DatanodeProtocolClientSideTranslatorPB.java | 5 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 38 +- ...igurationProtocolServerSideTranslatorPB.java | 89 + .../ReconfigurationProtocolServerSideUtils.java | 78 + .../server/blockmanagement/BlockIdManager.java | 4 +- .../blockmanagement/BlockInfoStriped.java | 2 +- .../server/blockmanagement/BlockManager.java | 397 +++-- .../blockmanagement/BlockManagerSafeMode.java | 638 ++++++++ .../BlockPlacementPolicyDefault.java | 15 +- .../BlockUnderConstructionFeature.java | 4 +- .../blockmanagement/DatanodeDescriptor.java | 24 +- .../server/blockmanagement/DatanodeManager.java | 432 ++--- .../server/blockmanagement/DatanodeStats.java | 19 +- .../blockmanagement/DatanodeStorageInfo.java | 2 +- .../blockmanagement/DecommissionManager.java | 39 +- .../blockmanagement/ErasureCodingWork.java | 6 +- .../blockmanagement/HeartbeatManager.java | 8 +- .../hdfs/server/datanode/BPServiceActor.java | 4 + .../datanode/BPServiceActorActionException.java | 5 + .../server/datanode/BlockRecoveryWorker.java | 238 ++- .../hadoop/hdfs/server/datanode/DataNode.java | 64 +- .../hdfs/server/datanode/DataXceiver.java | 32 +- .../hdfs/server/datanode/DataXceiverServer.java | 58 +- .../hdfs/server/datanode/ErrorReportAction.java | 2 +- .../hdfs/server/datanode/ReplicaInfo.java | 79 + .../server/datanode/ReportBadBlockAction.java | 2 +- .../server/datanode/ShortCircuitRegistry.java | 14 +- .../erasurecode/ErasureCodingWorker.java | 2 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 121 +- .../hdfs/server/namenode/CacheManager.java | 2 +- .../hdfs/server/namenode/Checkpointer.java | 3 +- .../hdfs/server/namenode/FSDirConcatOp.java | 24 +- .../hdfs/server/namenode/FSDirDeleteOp.java | 2 +- .../hdfs/server/namenode/FSDirRenameOp.java | 3 +- .../hdfs/server/namenode/FSDirTruncateOp.java | 2 +- .../hdfs/server/namenode/FSDirWriteFileOp.java | 2 +- .../hdfs/server/namenode/FSDirectory.java | 49 +- .../hdfs/server/namenode/FSEditLogLoader.java | 3 +- .../server/namenode/FSImageFormatPBINode.java | 1 + .../hdfs/server/namenode/FSNamesystem.java | 1135 ++----------- .../hadoop/hdfs/server/namenode/NNStorage.java | 18 +- .../hadoop/hdfs/server/namenode/NameNode.java | 29 +- .../server/namenode/NameNodeHttpServer.java | 9 + .../hdfs/server/namenode/NameNodeRpcServer.java | 43 +- .../hdfs/server/namenode/NamenodeFsck.java | 16 +- .../hadoop/hdfs/server/namenode/Namesystem.java | 18 +- .../hadoop/hdfs/server/namenode/SafeMode.java | 18 - .../hdfs/server/namenode/SecondaryNameNode.java | 14 +- .../server/namenode/ha/BootstrapStandby.java | 6 +- .../namenode/metrics/NameNodeMetrics.java | 12 + .../server/protocol/BlockECRecoveryCommand.java | 11 +- .../server/protocol/BlockRecoveryCommand.java | 33 + .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 +- .../hadoop-hdfs/src/main/proto/HdfsServer.proto | 6 +- .../src/main/resources/hdfs-default.xml | 132 ++ .../src/main/webapps/hdfs/explorer.html | 9 +- .../src/main/webapps/hdfs/explorer.js | 30 + .../hadoop-hdfs/src/site/markdown/HdfsDesign.md | 163 +- .../src/site/markdown/HdfsMultihoming.md | 42 + .../src/site/markdown/HdfsPermissionsGuide.md | 72 +- .../org/apache/hadoop/TestRefreshCallQueue.java | 1 + .../java/org/apache/hadoop/cli/TestAclCLI.java | 2 + .../apache/hadoop/cli/TestCacheAdminCLI.java | 2 + .../apache/hadoop/cli/TestCryptoAdminCLI.java | 2 + .../org/apache/hadoop/cli/TestDeleteCLI.java | 2 + .../apache/hadoop/cli/TestErasureCodingCLI.java | 2 + .../java/org/apache/hadoop/cli/TestHDFSCLI.java | 2 + .../org/apache/hadoop/cli/TestXAttrCLI.java | 2 + .../apache/hadoop/fs/TestFcHdfsCreateMkdir.java | 4 +- .../apache/hadoop/fs/TestFcHdfsPermission.java | 4 +- .../apache/hadoop/fs/TestFcHdfsSetUMask.java | 4 +- .../org/apache/hadoop/fs/TestSymlinkHdfs.java | 4 +- .../hadoop/fs/contract/hdfs/HDFSContract.java | 1 + .../hadoop/fs/shell/TestHdfsTextCommand.java | 2 + .../fs/viewfs/TestViewFileSystemHdfs.java | 4 +- .../fs/viewfs/TestViewFileSystemWithAcls.java | 4 +- .../fs/viewfs/TestViewFileSystemWithXAttrs.java | 4 +- .../hadoop/fs/viewfs/TestViewFsAtHdfsRoot.java | 4 +- .../apache/hadoop/fs/viewfs/TestViewFsHdfs.java | 4 +- .../hadoop/fs/viewfs/TestViewFsWithAcls.java | 4 +- .../hadoop/fs/viewfs/TestViewFsWithXAttrs.java | 4 +- .../org/apache/hadoop/hdfs/FileAppendTest4.java | 4 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 41 +- .../apache/hadoop/hdfs/StripedFileTestUtil.java | 16 +- .../apache/hadoop/hdfs/TestAbandonBlock.java | 10 +- .../hadoop/hdfs/TestBlocksScheduledCounter.java | 2 + .../TestClientProtocolForPipelineRecovery.java | 4 +- .../hadoop/hdfs/TestClientReportBadBlock.java | 10 +- .../hadoop/hdfs/TestDFSClientExcludedNodes.java | 1 + .../hadoop/hdfs/TestDFSClientFailover.java | 1 + .../hadoop/hdfs/TestDFSClientSocketSize.java | 1 + .../org/apache/hadoop/hdfs/TestDFSFinalize.java | 5 +- .../apache/hadoop/hdfs/TestDFSOutputStream.java | 4 +- .../apache/hadoop/hdfs/TestDFSPermission.java | 1 + .../org/apache/hadoop/hdfs/TestDFSRollback.java | 5 +- .../hadoop/hdfs/TestDFSStartupVersions.java | 5 +- .../hdfs/TestDFSStorageStateRecovery.java | 5 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 1 + .../hadoop/hdfs/TestDFSStripedOutputStream.java | 1 + .../hadoop/hdfs/TestDataTransferKeepalive.java | 5 +- .../hadoop/hdfs/TestDatanodeRegistration.java | 140 +- .../apache/hadoop/hdfs/TestDecommission.java | 1 + .../hadoop/hdfs/TestDistributedFileSystem.java | 28 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 86 + .../hadoop/hdfs/TestEncryptionZonesWithHA.java | 1 + .../hadoop/hdfs/TestErasureCodingPolicies.java | 1 + .../TestErasureCodingPolicyWithSnapshot.java | 1 + .../org/apache/hadoop/hdfs/TestFileAppend.java | 66 + .../org/apache/hadoop/hdfs/TestFileAppend4.java | 18 +- .../hadoop/hdfs/TestFileConcurrentReader.java | 6 +- .../apache/hadoop/hdfs/TestFileCreation.java | 6 +- .../org/apache/hadoop/hdfs/TestFileStatus.java | 8 +- .../hadoop/hdfs/TestFileStatusWithECPolicy.java | 1 + .../apache/hadoop/hdfs/TestGetFileChecksum.java | 1 + .../hadoop/hdfs/TestHDFSFileSystemContract.java | 6 +- .../org/apache/hadoop/hdfs/TestHdfsAdmin.java | 1 + .../apache/hadoop/hdfs/TestLeaseRecovery.java | 1 + .../apache/hadoop/hdfs/TestLeaseRecovery2.java | 6 +- .../hadoop/hdfs/TestLeaseRecoveryStriped.java | 273 ++++ .../apache/hadoop/hdfs/TestListFilesInDFS.java | 6 +- .../hadoop/hdfs/TestListFilesInFileContext.java | 4 +- .../apache/hadoop/hdfs/TestMiniDFSCluster.java | 19 +- .../org/apache/hadoop/hdfs/TestPipelines.java | 5 +- .../hdfs/TestReadStripedFileWithDecoding.java | 1 + .../TestReadStripedFileWithMissingBlocks.java | 1 + .../hadoop/hdfs/TestRecoverStripedFile.java | 5 +- .../hadoop/hdfs/TestReservedRawPaths.java | 1 + .../apache/hadoop/hdfs/TestRollingUpgrade.java | 1 + .../hadoop/hdfs/TestRollingUpgradeRollback.java | 1 + .../org/apache/hadoop/hdfs/TestSafeMode.java | 4 +- .../hdfs/TestSafeModeWithStripedFile.java | 1 + .../org/apache/hadoop/hdfs/TestWriteRead.java | 5 +- .../hadoop/hdfs/TestWriteReadStripedFile.java | 1 + .../datatransfer/sasl/TestSaslDataTransfer.java | 1 + .../hadoop/hdfs/protocolPB/TestPBHelper.java | 15 +- .../hdfs/qjournal/MiniJournalCluster.java | 38 + .../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 1 + .../hdfs/qjournal/TestMiniJournalCluster.java | 1 + .../hadoop/hdfs/qjournal/TestNNWithQJM.java | 1 + .../hdfs/qjournal/TestSecureNNWithQJM.java | 3 + .../qjournal/client/TestEpochsAreUnique.java | 1 + .../hdfs/qjournal/client/TestQJMWithFaults.java | 3 + .../client/TestQuorumJournalManager.java | 15 +- .../qjournal/server/TestJournalNodeMXBean.java | 11 +- .../hdfs/security/TestDelegationToken.java | 1 + .../BaseReplicationPolicyTest.java | 2 + .../blockmanagement/BlockManagerTestUtil.java | 11 +- .../blockmanagement/TestBlockManager.java | 263 ++- .../TestBlockManagerSafeMode.java | 480 ++++++ .../blockmanagement/TestBlockStatsMXBean.java | 63 + .../TestBlocksWithNotEnoughRacks.java | 8 +- .../TestComputeInvalidateWork.java | 2 + .../TestPendingInvalidateBlock.java | 1 + .../blockmanagement/TestPendingReplication.java | 4 +- .../blockmanagement/TestReplicationPolicy.java | 56 +- .../TestReplicationPolicyWithNodeGroup.java | 32 + .../TestSequentialBlockGroupId.java | 1 + .../server/datanode/BlockReportTestBase.java | 12 +- .../hdfs/server/datanode/DataNodeTestUtils.java | 4 - .../server/datanode/FsDatasetTestUtils.java | 36 +- .../server/datanode/SimulatedFSDataset.java | 3 +- .../server/datanode/TestBPOfferService.java | 32 +- .../hdfs/server/datanode/TestBlockRecovery.java | 34 +- .../hdfs/server/datanode/TestDataNodeExit.java | 4 +- .../datanode/TestDataNodeReconfiguration.java | 259 +++ .../datanode/TestDataNodeVolumeFailure.java | 3 + .../TestDataNodeVolumeFailureReporting.java | 1 + .../TestDataNodeVolumeFailureToleration.java | 1 + .../server/datanode/TestDeleteBlockPool.java | 69 +- .../server/datanode/TestDirectoryScanner.java | 4 +- .../hdfs/server/datanode/TestDiskError.java | 14 +- .../server/datanode/TestFsDatasetCache.java | 2 + .../datanode/TestIncrementalBrVariations.java | 19 +- .../server/datanode/TestSimulatedFSDataset.java | 47 + .../fsdataset/impl/FsDatasetImplTestUtils.java | 81 + .../fsdataset/impl/FsDatasetTestUtil.java | 10 +- .../fsdataset/impl/LazyPersistTestCase.java | 2 +- .../fsdataset/impl/TestFsDatasetImpl.java | 4 + .../hdfs/server/namenode/FSImageTestUtil.java | 7 + .../hdfs/server/namenode/FSXAttrBaseTest.java | 2 +- .../server/namenode/NNThroughputBenchmark.java | 10 +- .../hdfs/server/namenode/NameNodeAdapter.java | 12 +- .../hdfs/server/namenode/TestAclConfigFlag.java | 1 + .../hdfs/server/namenode/TestAddBlock.java | 1 + .../hdfs/server/namenode/TestAddBlockRetry.java | 1 + .../TestAddOverReplicatedStripedBlocks.java | 1 + .../server/namenode/TestAddStripedBlocks.java | 9 +- .../hdfs/server/namenode/TestAuditLogs.java | 12 +- .../hdfs/server/namenode/TestBackupNode.java | 14 +- ...stBlockPlacementPolicyRackFaultTolerant.java | 1 + .../server/namenode/TestCacheDirectives.java | 1 + .../hdfs/server/namenode/TestCheckpoint.java | 2 +- .../TestCommitBlockSynchronization.java | 5 +- .../TestCommitBlockWithInvalidGenStamp.java | 1 + .../hdfs/server/namenode/TestDeadDatanode.java | 21 +- .../TestDefaultBlockPlacementPolicy.java | 1 + .../namenode/TestDiskspaceQuotaUpdate.java | 1 + .../server/namenode/TestEditLogAutoroll.java | 2 + .../namenode/TestEditLogJournalFailures.java | 5 +- .../hdfs/server/namenode/TestEditLogRace.java | 2 +- .../hdfs/server/namenode/TestFSDirectory.java | 1 + .../server/namenode/TestFSEditLogLoader.java | 4 +- .../server/namenode/TestFSImageWithAcl.java | 4 +- .../namenode/TestFSImageWithSnapshot.java | 1 + .../server/namenode/TestFSImageWithXAttr.java | 4 +- .../hdfs/server/namenode/TestFSNamesystem.java | 13 +- .../hdfs/server/namenode/TestFileTruncate.java | 10 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 6 +- .../hdfs/server/namenode/TestHDFSConcat.java | 21 + .../hdfs/server/namenode/TestHostsFiles.java | 8 +- .../namenode/TestINodeAttributeProvider.java | 1 + .../hdfs/server/namenode/TestINodeFile.java | 7 +- .../hdfs/server/namenode/TestMalformedURLs.java | 1 + .../namenode/TestMetadataVersionOutput.java | 1 + .../TestNameNodeMetadataConsistency.java | 61 +- .../namenode/TestNameNodeRetryCacheMetrics.java | 1 + .../namenode/TestNameNodeRpcServerMethods.java | 86 + .../server/namenode/TestNamenodeRetryCache.java | 5 +- .../server/namenode/TestQuotaByStorageType.java | 1 + .../namenode/TestQuotaWithStripedBlocks.java | 1 + .../namenode/TestRecoverStripedBlocks.java | 5 +- .../hdfs/server/namenode/TestStartup.java | 2 + .../server/namenode/TestXAttrConfigFlag.java | 1 + .../hdfs/server/namenode/ha/HATestUtil.java | 2 +- .../namenode/ha/TestBootstrapStandby.java | 12 + .../ha/TestBootstrapStandbyWithQJM.java | 2 + .../hdfs/server/namenode/ha/TestDNFencing.java | 1 + .../namenode/ha/TestDelegationTokensWithHA.java | 1 + .../ha/TestFailoverWithBlockTokensEnabled.java | 1 + .../namenode/ha/TestFailureToReadEdits.java | 3 + .../server/namenode/ha/TestGetGroupsWithHA.java | 1 + .../hdfs/server/namenode/ha/TestHASafeMode.java | 12 +- .../namenode/ha/TestInitializeSharedEdits.java | 1 + .../server/namenode/ha/TestNNHealthCheck.java | 1 + .../ha/TestPendingCorruptDnMessages.java | 2 +- .../namenode/ha/TestPipelinesFailover.java | 38 +- .../server/namenode/ha/TestQuotasWithHA.java | 1 + .../namenode/ha/TestRetryCacheWithHA.java | 1 + .../namenode/ha/TestSeveralNameNodes.java | 8 +- .../namenode/ha/TestStandbyCheckpoints.java | 1 + .../server/namenode/ha/TestXAttrsWithHA.java | 1 + .../TestNNMetricFilesInGetListingOps.java | 5 +- .../namenode/metrics/TestNameNodeMetrics.java | 5 +- .../snapshot/TestFileContextSnapshot.java | 1 + .../TestGetContentSummaryWithSnapshot.java | 1 + ...tINodeFileUnderConstructionWithSnapshot.java | 1 + .../namenode/snapshot/TestNestedSnapshots.java | 1 + .../snapshot/TestOpenFilesWithSnapshot.java | 2 + .../snapshot/TestRenameWithSnapshots.java | 1 + .../snapshot/TestSetQuotaWithSnapshot.java | 1 + .../server/namenode/snapshot/TestSnapshot.java | 1 + .../snapshot/TestSnapshotBlocksMap.java | 1 + .../namenode/snapshot/TestSnapshotDeletion.java | 1 + .../snapshot/TestSnapshotDiffReport.java | 1 + .../namenode/snapshot/TestSnapshotListing.java | 1 + .../namenode/snapshot/TestSnapshotMetrics.java | 1 + .../namenode/snapshot/TestSnapshotRename.java | 1 + .../snapshot/TestSnapshotReplication.java | 1 + .../snapshot/TestSnapshottableDirListing.java | 1 + .../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 2 +- .../hdfs/tools/TestDFSHAAdminMiniCluster.java | 5 +- .../hdfs/tools/TestDFSZKFailoverController.java | 8 +- .../apache/hadoop/hdfs/tools/TestGetGroups.java | 5 +- .../hdfs/tools/TestStoragePolicyCommands.java | 2 + .../hadoop/hdfs/util/TestStripedBlockUtil.java | 4 +- .../hadoop/hdfs/web/TestHttpsFileSystem.java | 4 +- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 158 ++ .../hdfs/web/TestWebHdfsFileSystemContract.java | 2 +- .../hadoop/hdfs/web/TestWebHdfsTokens.java | 6 +- .../TestWebHdfsWithAuthenticationFilter.java | 8 +- .../security/TestRefreshUserMappings.java | 2 + .../org/apache/hadoop/tools/TestJMXGet.java | 17 +- .../org/apache/hadoop/tracing/TestTracing.java | 5 +- hadoop-mapreduce-project/CHANGES.txt | 40 +- .../v2/app/job/impl/TaskAttemptImpl.java | 13 +- .../mapreduce/v2/app/webapp/TaskPage.java | 2 +- .../mapreduce/v2/app/webapp/TasksPage.java | 2 +- .../jobhistory/TestJobHistoryEventHandler.java | 14 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 21 +- .../org/apache/hadoop/mapreduce/tools/CLI.java | 41 +- .../src/main/resources/mapred-default.xml | 162 +- .../src/site/markdown/MapReduceTutorial.md | 6 +- .../org/apache/hadoop/mapred/TestTaskLog.java | 22 +- .../apache/hadoop/mapreduce/tools/TestCLI.java | 31 +- .../mapreduce/v2/hs/HistoryFileManager.java | 37 +- .../mapreduce/v2/hs/JobHistoryServer.java | 7 +- .../mapreduce/v2/hs/webapp/HsTaskPage.java | 2 +- .../mapreduce/v2/hs/webapp/HsTasksPage.java | 2 +- .../mapreduce/v2/hs/TestJobHistoryServer.java | 1 - .../mapreduce/v2/hs/webapp/TestHSWebApp.java | 26 + .../hadoop/mapred/ResourceMgrDelegate.java | 4 +- .../org/apache/hadoop/mapred/YARNRunner.java | 5 +- .../hadoop/mapred/TestMiniMRChildTask.java | 68 +- .../apache/hadoop/mapred/TestYARNRunner.java | 47 +- hadoop-project/pom.xml | 1 + .../hadoop/fs/s3/Jets3tFileSystemStore.java | 8 +- .../hadoop/fs/azure/NativeAzureFileSystem.java | 34 +- .../fs/azure/TestNativeAzureFileSystemLive.java | 22 + .../hadoop/yarn/sls/appmaster/AMSimulator.java | 2 +- .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 2 +- .../yarn/sls/scheduler/RMNodeWrapper.java | 5 +- .../sls/scheduler/ResourceSchedulerWrapper.java | 8 + hadoop-yarn-project/CHANGES.txt | 214 ++- .../dev-support/findbugs-exclude.xml | 5 + .../UpdateApplicationPriorityResponse.java | 18 +- .../yarn/api/records/ContainerStatus.java | 24 + .../hadoop/yarn/api/records/ExecutionType.java | 43 + .../api/records/FinalApplicationStatus.java | 5 +- .../hadoop/yarn/api/records/NodeReport.java | 25 + .../hadoop/yarn/api/records/QueueInfo.java | 16 +- .../yarn/api/records/ResourceUtilization.java | 150 ++ .../records/timeline/TimelineEntityGroupId.java | 163 ++ .../hadoop/yarn/conf/YarnConfiguration.java | 109 +- .../yarn/server/api/ContainerContext.java | 22 + .../src/main/proto/yarn_protos.proto | 16 + .../src/main/proto/yarn_service_protos.proto | 1 + .../yarn/conf/TestYarnConfigurationFields.java | 5 + .../hadoop/yarn/client/api/YarnClient.java | 4 +- .../yarn/client/api/impl/YarnClientImpl.java | 4 +- .../hadoop/yarn/client/cli/ApplicationCLI.java | 15 +- .../apache/hadoop/yarn/client/cli/NodeCLI.java | 97 +- .../apache/hadoop/yarn/client/cli/QueueCLI.java | 6 + .../hadoop/yarn/client/ProtocolHATestBase.java | 4 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 200 ++- ...UpdateApplicationPriorityResponsePBImpl.java | 63 + .../records/impl/pb/ContainerStatusPBImpl.java | 31 +- .../api/records/impl/pb/NodeReportPBImpl.java | 73 + .../yarn/api/records/impl/pb/ProtoUtils.java | 12 + .../api/records/impl/pb/QueueInfoPBImpl.java | 13 + .../impl/pb/ResourceUtilizationPBImpl.java | 104 ++ .../hadoop/yarn/client/api/TimelineClient.java | 43 + .../client/api/impl/DirectTimelineWriter.java | 66 + .../api/impl/FileSystemTimelineWriter.java | 847 ++++++++++ .../client/api/impl/TimelineClientImpl.java | 124 +- .../yarn/client/api/impl/TimelineWriter.java | 142 ++ .../hadoop/yarn/event/AsyncDispatcher.java | 7 +- .../nodelabels/CommonNodeLabelsManager.java | 18 +- .../nodelabels/FileSystemNodeLabelsStore.java | 65 +- .../hadoop/yarn/nodelabels/NodeLabelsStore.java | 16 +- .../NonAppendableFSNodeLabelStore.java | 134 ++ .../yarn/security/ContainerTokenIdentifier.java | 35 +- .../yarn/util/AbstractLivelinessMonitor.java | 4 + .../apache/hadoop/yarn/util/MonotonicClock.java | 46 + .../apache/hadoop/yarn/util/SystemClock.java | 4 + .../yarn/util/WindowsBasedProcessTree.java | 51 +- .../hadoop/yarn/webapp/view/JQueryUI.java | 21 +- .../src/main/proto/yarn_security_token.proto | 1 + .../webapps/static/dt-sorting/natural.js | 54 + .../src/main/resources/yarn-default.xml | 57 + .../hadoop/yarn/api/TestPBImplRecords.java | 4 +- .../yarn/api/TestTimelineEntityGroupId.java | 52 + .../client/api/impl/TestTimelineClient.java | 68 +- .../api/impl/TestTimelineClientForATS1_5.java | 225 +++ .../DummyCommonNodeLabelsManager.java | 6 +- .../TestFileSystemNodeLabelsStore.java | 28 +- .../yarn/security/TestYARNTokenIdentifier.java | 13 +- .../yarn/util/TestWindowsBasedProcessTree.java | 25 +- ...pplicationHistoryManagerOnTimelineStore.java | 7 + .../ApplicationHistoryServer.java | 9 +- .../webapp/AppAttemptPage.java | 2 +- .../webapp/AppPage.java | 2 +- .../server/timeline/TimelineDataManager.java | 6 +- .../timeline/webapp/TimelineWebServices.java | 2 + .../TestApplicationHistoryServer.java | 1 - .../webapp/TestAHSWebApp.java | 50 +- .../webapp/TestTimelineWebServices.java | 21 + .../webapp/TestTimelineWebServicesWithSSL.java | 21 +- .../impl/pb/NMContainerStatusPBImpl.java | 14 +- .../yarn/server/api/records/NodeStatus.java | 1 + .../server/api/records/ResourceUtilization.java | 133 -- .../api/records/impl/pb/NodeStatusPBImpl.java | 5 +- .../impl/pb/ResourceUtilizationPBImpl.java | 104 -- .../hadoop/yarn/server/utils/BuilderUtils.java | 15 +- .../hadoop/yarn/server/webapp/WebPageUtils.java | 6 +- .../main/proto/yarn_server_common_protos.proto | 5 - .../server/nodemanager/ContainerExecutor.java | 24 +- .../nodemanager/DockerContainerExecutor.java | 11 +- .../yarn/server/nodemanager/NodeManager.java | 7 +- .../server/nodemanager/NodeResourceMonitor.java | 2 +- .../nodemanager/NodeResourceMonitorImpl.java | 2 +- .../nodemanager/NodeStatusUpdaterImpl.java | 7 +- .../container/ContainerImpl.java | 7 +- .../launcher/ContainerLaunch.java | 169 +- .../linux/resources/CGroupsHandler.java | 9 +- .../CGroupsMemoryResourceHandlerImpl.java | 156 ++ .../linux/resources/MemoryResourceHandler.java | 27 + .../linux/resources/ResourceHandlerModule.java | 30 +- .../localizer/ResourceLocalizationService.java | 2 +- .../logaggregation/AppLogAggregatorImpl.java | 6 +- .../monitor/ContainersMonitor.java | 2 +- .../monitor/ContainersMonitorImpl.java | 2 +- .../nodemanager/webapp/AllApplicationsPage.java | 9 +- .../nodemanager/webapp/AllContainersPage.java | 8 +- .../nodemanager/TestNodeManagerResync.java | 3 + .../nodemanager/TestNodeManagerShutdown.java | 3 + .../amrmproxy/MockResourceManagerFacade.java | 6 +- .../container/TestContainer.java | 59 +- .../launcher/TestContainerLaunch.java | 252 ++- .../TestCGroupsMemoryResourceHandlerImpl.java | 172 ++ .../TestResourceLocalizationService.java | 2 + .../hadoop-yarn-server-resourcemanager/pom.xml | 11 + .../server/resourcemanager/AdminService.java | 49 +- .../server/resourcemanager/ClientRMService.java | 12 +- .../server/resourcemanager/ClusterMetrics.java | 20 +- .../resourcemanager/LeaderElectorService.java | 144 ++ .../resourcemanager/NMLivelinessMonitor.java | 3 +- .../resourcemanager/NodesListManager.java | 22 +- .../server/resourcemanager/RMAppManager.java | 29 +- .../yarn/server/resourcemanager/RMContext.java | 4 + .../server/resourcemanager/RMContextImpl.java | 11 + .../server/resourcemanager/ResourceManager.java | 60 +- .../resourcemanager/ResourceTrackerService.java | 5 + .../metrics/AppAttemptFinishedEvent.java | 10 +- .../metrics/SystemMetricsPublisher.java | 20 +- .../recovery/FileSystemRMStateStore.java | 12 + .../recovery/LeveldbRMStateStore.java | 32 + .../recovery/MemoryRMStateStore.java | 13 + .../recovery/NullRMStateStore.java | 6 + .../resourcemanager/recovery/RMStateStore.java | 63 +- .../recovery/RMStateStoreEventType.java | 1 + .../RMStateStoreRemoveAppAttemptEvent.java | 37 + .../recovery/ZKRMStateStore.java | 16 + .../recovery/records/ApplicationStateData.java | 10 + .../reservation/CapacityOverTimePolicy.java | 52 +- .../reservation/InMemoryPlan.java | 123 +- .../InMemoryReservationAllocation.java | 13 +- .../reservation/NoOverCommitPolicy.java | 8 + .../resourcemanager/reservation/PlanView.java | 66 +- .../reservation/ReservationAllocation.java | 12 +- .../reservation/SharingPolicy.java | 24 +- .../reservation/planning/IterativePlanner.java | 16 +- .../reservation/planning/PlanningAlgorithm.java | 34 +- .../reservation/planning/StageAllocator.java | 6 +- .../planning/StageAllocatorGreedy.java | 23 +- .../planning/StageAllocatorLowCostAligned.java | 14 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 81 +- .../rmapp/attempt/AMLivelinessMonitor.java | 3 +- .../rmapp/attempt/RMAppAttempt.java | 6 + .../rmapp/attempt/RMAppAttemptImpl.java | 22 +- .../rmcontainer/ContainerAllocationExpirer.java | 3 +- .../server/resourcemanager/rmnode/RMNode.java | 2 +- .../resourcemanager/rmnode/RMNodeImpl.java | 50 +- .../rmnode/RMNodeStatusEvent.java | 2 +- .../scheduler/AppSchedulingInfo.java | 379 ++--- .../scheduler/ResourceUsage.java | 62 +- .../scheduler/SchedulerApplicationAttempt.java | 124 +- .../scheduler/SchedulerNode.java | 2 +- .../scheduler/capacity/AbstractCSQueue.java | 8 +- ...CSAMContainerLaunchDiagnosticsConstants.java | 35 + .../scheduler/capacity/CapacityScheduler.java | 8 +- .../scheduler/capacity/LeafQueue.java | 41 +- .../scheduler/capacity/QueueCapacities.java | 9 + .../allocator/RegularContainerAllocator.java | 16 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 93 +- .../scheduler/fair/FSAppAttempt.java | 19 +- .../scheduler/fair/FairScheduler.java | 4 +- .../scheduler/fifo/FifoScheduler.java | 2 +- .../webapp/CapacitySchedulerPage.java | 2 +- .../webapp/MetricsOverviewTable.java | 30 +- .../resourcemanager/webapp/NodesPage.java | 3 + .../resourcemanager/webapp/RMWebServices.java | 384 ++++- .../webapp/dao/AppAttemptInfo.java | 8 + .../webapp/dao/ClusterMetricsInfo.java | 6 + .../resourcemanager/webapp/dao/NodeInfo.java | 8 + .../webapp/dao/ReservationDefinitionInfo.java | 82 + .../dao/ReservationDeleteRequestInfo.java | 49 + .../dao/ReservationDeleteResponseInfo.java | 36 + .../webapp/dao/ReservationRequestInfo.java | 78 + .../webapp/dao/ReservationRequestsInfo.java | 63 + .../dao/ReservationSubmissionRequestInfo.java | 60 + .../dao/ReservationSubmissionResponseInfo.java | 54 + .../dao/ReservationUpdateRequestInfo.java | 60 + .../dao/ReservationUpdateResponseInfo.java | 37 + .../webapp/dao/ResourceUtilizationInfo.java | 89 + .../yarn/server/resourcemanager/MockNodes.java | 2 +- .../server/resourcemanager/TestAppManager.java | 3 +- .../resourcemanager/TestClientRMService.java | 60 +- .../TestLeaderElectorService.java | 269 +++ .../yarn/server/resourcemanager/TestRMHA.java | 8 +- .../resourcemanager/TestRMNodeTransitions.java | 89 +- .../server/resourcemanager/TestRMRestart.java | 98 +- .../applicationsmanager/TestAMRestart.java | 8 + .../metrics/TestSystemMetricsPublisher.java | 34 +- .../nodelabels/NullRMNodeLabelsManager.java | 4 +- .../recovery/RMStateStoreTestBase.java | 57 +- .../recovery/TestFSRMStateStore.java | 17 + .../recovery/TestLeveldbRMStateStore.java | 16 + .../recovery/TestZKRMStateStore.java | 12 + .../TestCapacitySchedulerPlanFollower.java | 6 +- .../reservation/TestInMemoryPlan.java | 72 +- .../planning/TestGreedyReservationAgent.java | 94 +- .../rmapp/TestRMAppTransitions.java | 4 +- .../scheduler/TestResourceUsage.java | 22 +- .../TestSchedulerApplicationAttempt.java | 68 +- .../capacity/TestApplicationLimits.java | 9 +- .../TestApplicationLimitsByPartition.java | 44 +- .../TestNodeLabelContainerAllocation.java | 17 +- .../scheduler/capacity/TestQueueCapacities.java | 3 +- .../scheduler/capacity/TestReservations.java | 39 + .../scheduler/fair/TestFairScheduler.java | 98 ++ .../resourcemanager/webapp/TestNodesPage.java | 2 +- .../resourcemanager/webapp/TestRMWebApp.java | 5 + .../webapp/TestRMWebServices.java | 2 +- .../webapp/TestRMWebServicesApps.java | 6 +- .../TestRMWebServicesAppsModification.java | 10 + .../webapp/TestRMWebServicesNodes.java | 79 +- .../webapp/TestRMWebServicesReservation.java | 517 ++++++ .../src/test/resources/delete-reservation.json | 3 + .../src/test/resources/submit-reservation.json | 31 + .../src/test/resources/update-reservation.json | 31 + .../TestMiniYarnClusterNodeUtilization.java | 2 +- .../yarn/server/webproxy/WebAppProxyServer.java | 9 +- .../src/site/markdown/CapacityScheduler.md | 25 + .../src/site/markdown/NodeManager.md | 2 +- .../src/site/markdown/TimelineServer.md | 78 + .../src/site/markdown/YarnCommands.md | 2 + 623 files changed, 17741 insertions(+), 9099 deletions(-) ----------------------------------------------------------------------
