Merge remote-tracking branch 'apache/trunk' into YARN-2928
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e2786151 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e2786151 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e2786151 Branch: refs/heads/YARN-2928 Commit: e2786151da326ab763288c001677c17ca0de8dd3 Parents: f26941b f80c988 Author: Zhijie Shen <zjs...@apache.org> Authored: Wed Feb 11 11:40:48 2015 -0800 Committer: Zhijie Shen <zjs...@apache.org> Committed: Wed Feb 11 11:40:48 2015 -0800 ---------------------------------------------------------------------- dev-support/determine-flaky-tests-hadoop.py | 204 +++ .../main/resources/assemblies/hadoop-dist.xml | 5 + .../assemblies/hadoop-mapreduce-dist.xml | 6 + .../resources/assemblies/hadoop-yarn-dist.xml | 5 + hadoop-common-project/hadoop-common/CHANGES.txt | 139 ++ .../hadoop-common/dev-support/checkstyle.xml | 1 - hadoop-common-project/hadoop-common/pom.xml | 2 +- .../hadoop-common/src/CMakeLists.txt | 2 +- .../hadoop-common/src/main/bin/hadoop | 12 +- .../hadoop-common/src/main/bin/hadoop-config.sh | 26 +- .../src/main/bin/hadoop-functions.sh | 344 ++--- .../hadoop-common/src/main/bin/hadoop.cmd | 6 +- .../src/main/conf/log4j.properties | 4 +- .../src/main/conf/shellprofile.d/example | 106 ++ .../org/apache/hadoop/conf/Configuration.java | 142 +- .../apache/hadoop/fs/AbstractFileSystem.java | 19 +- .../java/org/apache/hadoop/fs/ChecksumFs.java | 5 + .../hadoop/fs/CommonConfigurationKeys.java | 3 - .../fs/CommonConfigurationKeysPublic.java | 7 +- .../org/apache/hadoop/fs/ContentSummary.java | 57 +- .../java/org/apache/hadoop/fs/CreateFlag.java | 24 +- .../apache/hadoop/fs/DelegateToFileSystem.java | 6 + .../org/apache/hadoop/fs/FSDataInputStream.java | 2 +- .../org/apache/hadoop/fs/FSOutputSummer.java | 2 +- .../java/org/apache/hadoop/fs/FileContext.java | 43 + .../java/org/apache/hadoop/fs/FilterFs.java | 8 + .../java/org/apache/hadoop/fs/shell/Count.java | 32 +- .../java/org/apache/hadoop/fs/shell/Ls.java | 183 ++- .../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 6 + .../org/apache/hadoop/fs/viewfs/ViewFs.java | 17 +- .../io/serializer/SerializationFactory.java | 2 +- .../main/java/org/apache/hadoop/ipc/Client.java | 2 +- .../apache/hadoop/ipc/ProtobufRpcEngine.java | 5 +- .../main/java/org/apache/hadoop/ipc/Server.java | 27 +- .../apache/hadoop/ipc/WritableRpcEngine.java | 4 +- .../hadoop/metrics/ganglia/GangliaContext.java | 33 +- .../apache/hadoop/metrics/ganglia/package.html | 6 + .../hadoop/metrics2/impl/MetricsSystemImpl.java | 2 +- .../sink/ganglia/AbstractGangliaSink.java | 36 +- .../org/apache/hadoop/net/NetworkTopology.java | 12 +- .../net/NetworkTopologyWithNodeGroup.java | 2 +- .../hadoop/security/LdapGroupsMapping.java | 2 + .../apache/hadoop/security/SaslRpcClient.java | 18 +- .../ZKDelegationTokenSecretManager.java | 26 +- .../apache/hadoop/tracing/SpanReceiverHost.java | 67 +- .../hadoop/tracing/TraceSamplerFactory.java | 53 - .../org/apache/hadoop/tracing/TraceUtils.java | 65 + .../apache/hadoop/util/ComparableVersion.java | 2 - .../java/org/apache/hadoop/util/FindClass.java | 388 ++++++ .../java/org/apache/hadoop/util/LogAdapter.java | 74 + .../java/org/apache/hadoop/util/ProtoUtil.java | 4 +- .../org/apache/hadoop/util/SignalLogger.java | 8 +- .../org/apache/hadoop/util/StringUtils.java | 16 + .../src/main/native/src/exception.c | 6 + .../src/main/native/src/exception.h | 20 +- .../hadoop/io/compress/bzip2/Bzip2Compressor.c | 28 +- .../io/compress/bzip2/Bzip2Decompressor.c | 28 +- .../org/apache/hadoop/io/nativeio/NativeIO.c | 7 +- .../src/main/resources/core-default.xml | 157 ++- .../src/site/apt/CLIMiniCluster.apt.vm | 83 -- .../src/site/apt/ClusterSetup.apt.vm | 651 --------- .../src/site/apt/CommandsManual.apt.vm | 327 ----- .../src/site/apt/Compatibility.apt.vm | 541 -------- .../src/site/apt/DeprecatedProperties.apt.vm | 552 -------- .../src/site/apt/FileSystemShell.apt.vm | 746 ---------- .../src/site/apt/HttpAuthentication.apt.vm | 98 -- .../src/site/apt/InterfaceClassification.apt.vm | 239 ---- .../hadoop-common/src/site/apt/Metrics.apt.vm | 879 ------------ .../src/site/apt/NativeLibraries.apt.vm | 205 --- .../src/site/apt/RackAwareness.apt.vm | 140 -- .../src/site/apt/SecureMode.apt.vm | 689 --------- .../src/site/apt/ServiceLevelAuth.apt.vm | 216 --- .../src/site/apt/SingleCluster.apt.vm | 286 ---- .../src/site/apt/SingleNodeSetup.apt.vm | 24 - .../src/site/apt/Superusers.apt.vm | 144 -- .../hadoop-common/src/site/apt/Tracing.apt.vm | 233 ---- .../src/site/markdown/CLIMiniCluster.md.vm | 68 + .../src/site/markdown/ClusterSetup.md | 339 +++++ .../src/site/markdown/CommandsManual.md | 237 ++++ .../src/site/markdown/Compatibility.md | 313 +++++ .../src/site/markdown/DeprecatedProperties.md | 288 ++++ .../src/site/markdown/FileSystemShell.md | 709 ++++++++++ .../src/site/markdown/HttpAuthentication.md | 58 + .../site/markdown/InterfaceClassification.md | 221 +++ .../hadoop-common/src/site/markdown/Metrics.md | 456 ++++++ .../src/site/markdown/NativeLibraries.md.vm | 145 ++ .../src/site/markdown/RackAwareness.md | 136 ++ .../src/site/markdown/SecureMode.md | 377 +++++ .../src/site/markdown/ServiceLevelAuth.md | 144 ++ .../src/site/markdown/SingleCluster.md.vm | 232 ++++ .../src/site/markdown/Superusers.md | 106 ++ .../hadoop-common/src/site/markdown/Tracing.md | 182 +++ .../src/site/markdown/filesystem/filesystem.md | 35 + .../apache/hadoop/conf/TestConfiguration.java | 109 +- .../apache/hadoop/fs/FileSystemTestHelper.java | 27 +- .../org/apache/hadoop/fs/SymlinkBaseTest.java | 2 +- .../org/apache/hadoop/fs/TestAfsCheckPath.java | 6 + .../apache/hadoop/fs/TestContentSummary.java | 6 +- .../org/apache/hadoop/fs/shell/TestCount.java | 115 +- .../java/org/apache/hadoop/fs/shell/TestLs.java | 1308 ++++++++++++++++++ .../io/serializer/TestSerializationFactory.java | 10 + .../java/org/apache/hadoop/ipc/TestIPC.java | 51 + .../metrics/ganglia/TestGangliaContext.java | 41 + .../hadoop/metrics2/impl/MetricsRecords.java | 92 ++ .../metrics2/impl/TestMetricsSystemImpl.java | 2 +- .../metrics2/sink/ganglia/TestGangliaSink.java | 81 ++ .../org/apache/hadoop/net/TestTableMapping.java | 43 +- .../org/apache/hadoop/test/TestJUnitSetup.java | 39 + .../apache/hadoop/tracing/TestTraceUtils.java | 51 + .../org/apache/hadoop/util/TestFindClass.java | 218 +++ .../src/test/resources/testConf.xml | 50 +- .../hadoop-kms/src/main/conf/kms-env.sh | 4 - .../hadoop/crypto/key/kms/server/KMSACLs.java | 7 +- .../crypto/key/kms/server/KMSConfiguration.java | 1 + .../hadoop-kms/src/main/libexec/kms-config.sh | 7 +- .../crypto/key/kms/server/TestKMSACLs.java | 18 +- .../hadoop/fs/http/client/HttpFSUtils.java | 3 +- .../http/server/HttpFSAuthenticationFilter.java | 8 +- .../service/hadoop/FileSystemAccessService.java | 2 +- .../apache/hadoop/lib/wsrs/JSONMapProvider.java | 3 +- .../apache/hadoop/lib/wsrs/JSONProvider.java | 3 +- .../src/main/libexec/httpfs-config.sh | 2 +- .../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 2 +- .../hadoop/hdfs/nfs/nfs3/AsyncDataService.java | 16 +- .../hadoop/hdfs/nfs/nfs3/OpenFileCtx.java | 78 +- .../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 147 +- .../hadoop/hdfs/nfs/nfs3/WriteManager.java | 31 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 144 +- .../hadoop-hdfs/dev-support/checkstyle.xml | 1 - hadoop-hdfs-project/hadoop-hdfs/pom.xml | 7 +- .../contrib/libwebhdfs/src/hdfs_http_client.c | 11 +- .../hadoop-hdfs/src/main/bin/hdfs-config.sh | 35 +- .../main/java/org/apache/hadoop/fs/Hdfs.java | 6 + .../apache/hadoop/hdfs/BlockReaderLocal.java | 6 +- .../hadoop/hdfs/BlockReaderLocalLegacy.java | 6 +- .../hadoop/hdfs/BlockStorageLocationUtil.java | 8 +- .../org/apache/hadoop/hdfs/ClientContext.java | 14 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 143 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 16 +- .../hadoop/hdfs/DFSInotifyEventInputStream.java | 6 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 90 +- .../org/apache/hadoop/hdfs/DFSOutputStream.java | 86 +- .../hadoop/hdfs/DistributedFileSystem.java | 55 +- .../apache/hadoop/hdfs/KeyProviderCache.java | 109 ++ .../apache/hadoop/hdfs/RemoteBlockReader.java | 6 +- .../apache/hadoop/hdfs/RemoteBlockReader2.java | 6 +- .../org/apache/hadoop/hdfs/StorageType.java | 32 +- .../apache/hadoop/hdfs/client/HdfsAdmin.java | 27 + .../hadoop/hdfs/client/HdfsDataInputStream.java | 8 +- .../hdfs/client/HdfsDataOutputStream.java | 8 +- .../org/apache/hadoop/hdfs/inotify/Event.java | 12 + .../hdfs/protocol/CacheDirectiveIterator.java | 6 +- .../hadoop/hdfs/protocol/CachePoolIterator.java | 6 +- .../hadoop/hdfs/protocol/ClientProtocol.java | 18 +- .../hdfs/protocol/DatanodeInfoWithStorage.java | 63 + .../hdfs/protocol/EncryptionZoneIterator.java | 6 +- .../hadoop/hdfs/protocol/LocatedBlock.java | 75 +- .../QuotaByStorageTypeExceededException.java | 56 + .../datatransfer/DataTransferProtoUtil.java | 8 +- .../hdfs/protocol/datatransfer/PipelineAck.java | 98 +- .../hdfs/protocol/datatransfer/Receiver.java | 2 +- .../hdfs/protocol/datatransfer/Sender.java | 4 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 18 +- .../ClientNamenodeProtocolTranslatorPB.java | 31 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 25 +- .../hadoop/hdfs/server/balancer/Balancer.java | 28 +- .../hdfs/server/balancer/NameNodeConnector.java | 26 +- .../server/blockmanagement/BlockCollection.java | 8 +- .../hdfs/server/blockmanagement/BlockInfo.java | 403 ------ .../blockmanagement/BlockInfoContiguous.java | 412 ++++++ .../BlockInfoContiguousUnderConstruction.java | 400 ++++++ .../BlockInfoUnderConstruction.java | 402 ------ .../server/blockmanagement/BlockManager.java | 343 +++-- .../hdfs/server/blockmanagement/BlocksMap.java | 32 +- .../CacheReplicationMonitor.java | 14 +- .../blockmanagement/CorruptReplicasMap.java | 19 +- .../blockmanagement/DatanodeDescriptor.java | 28 +- .../server/blockmanagement/DatanodeManager.java | 8 +- .../blockmanagement/DatanodeStorageInfo.java | 22 +- .../blockmanagement/InvalidateBlocks.java | 8 +- .../PendingReplicationBlocks.java | 4 +- .../blockmanagement/UnderReplicatedBlocks.java | 48 +- .../hdfs/server/datanode/BPServiceActor.java | 5 + .../server/datanode/BlockPoolSliceStorage.java | 12 +- .../hdfs/server/datanode/BlockReceiver.java | 49 +- .../hdfs/server/datanode/BlockSender.java | 6 +- .../hadoop/hdfs/server/datanode/DataNode.java | 19 + .../hdfs/server/datanode/DataStorage.java | 15 +- .../hdfs/server/datanode/VolumeScanner.java | 52 +- .../datanode/fsdataset/ReplicaInputStreams.java | 8 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 57 +- .../datanode/web/webhdfs/WebHdfsHandler.java | 3 +- .../apache/hadoop/hdfs/server/mover/Mover.java | 7 +- .../hdfs/server/namenode/BackupImage.java | 31 +- .../hadoop/hdfs/server/namenode/BackupNode.java | 2 +- .../ContentSummaryComputationContext.java | 5 + .../namenode/DirectoryWithQuotaFeature.java | 228 ++- .../hdfs/server/namenode/FSDirAttrOp.java | 64 +- .../hdfs/server/namenode/FSDirConcatOp.java | 280 ++-- .../hdfs/server/namenode/FSDirDeleteOp.java | 21 +- .../hdfs/server/namenode/FSDirRenameOp.java | 64 +- .../hdfs/server/namenode/FSDirectory.java | 222 ++- .../hadoop/hdfs/server/namenode/FSEditLog.java | 37 +- .../hdfs/server/namenode/FSEditLogLoader.java | 105 +- .../hdfs/server/namenode/FSEditLogOp.java | 186 ++- .../hdfs/server/namenode/FSEditLogOpCodes.java | 2 + .../hadoop/hdfs/server/namenode/FSImage.java | 60 +- .../hdfs/server/namenode/FSImageFormat.java | 33 +- .../server/namenode/FSImageFormatPBINode.java | 94 +- .../server/namenode/FSImageSerialization.java | 16 +- .../hdfs/server/namenode/FSNamesystem.java | 220 +-- .../namenode/FileUnderConstructionFeature.java | 17 +- .../hadoop/hdfs/server/namenode/INode.java | 124 +- .../hdfs/server/namenode/INodeDirectory.java | 129 +- .../namenode/INodeDirectoryAttributes.java | 30 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 183 ++- .../hadoop/hdfs/server/namenode/INodeMap.java | 27 +- .../hdfs/server/namenode/INodeReference.java | 117 +- .../hdfs/server/namenode/INodeSymlink.java | 25 +- .../namenode/INodeWithAdditionalFields.java | 4 +- .../namenode/InotifyFSEditLogOpTranslator.java | 4 + .../hdfs/server/namenode/LeaseManager.java | 6 +- .../hadoop/hdfs/server/namenode/NameNode.java | 92 +- .../server/namenode/NameNodeLayoutVersion.java | 4 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 26 +- .../hdfs/server/namenode/NamenodeFsck.java | 4 +- .../hadoop/hdfs/server/namenode/Namesystem.java | 4 +- .../hadoop/hdfs/server/namenode/Quota.java | 2 +- .../namenode/QuotaByStorageTypeEntry.java | 85 ++ .../hdfs/server/namenode/QuotaCounts.java | 179 +++ .../namenode/snapshot/AbstractINodeDiff.java | 11 +- .../snapshot/AbstractINodeDiffList.java | 54 +- .../snapshot/DirectorySnapshottableFeature.java | 13 +- .../snapshot/DirectoryWithSnapshotFeature.java | 135 +- .../snapshot/FSImageFormatPBSnapshot.java | 45 +- .../hdfs/server/namenode/snapshot/FileDiff.java | 22 +- .../server/namenode/snapshot/FileDiffList.java | 27 +- .../snapshot/FileWithSnapshotFeature.java | 79 +- .../namenode/snapshot/SnapshotManager.java | 3 +- .../web/resources/NamenodeWebHdfsMethods.java | 30 +- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 50 +- .../org/apache/hadoop/hdfs/tools/DFSck.java | 2 +- .../tools/offlineImageViewer/FSImageLoader.java | 4 +- .../OfflineImageViewerPB.java | 21 +- .../PBImageDelimitedTextWriter.java | 132 ++ .../offlineImageViewer/PBImageTextWriter.java | 586 ++++++++ .../hdfs/tools/snapshot/LsSnapshottableDir.java | 2 +- .../hdfs/tools/snapshot/SnapshotDiff.java | 2 +- .../apache/hadoop/hdfs/util/EnumCounters.java | 36 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 8 + .../hdfs/web/resources/NewLengthParam.java | 49 + .../hadoop/hdfs/web/resources/PostOpParam.java | 2 + .../src/main/native/libhdfs/exception.c | 5 + .../hadoop-hdfs/src/main/native/libhdfs/hdfs.c | 33 +- .../hadoop-hdfs/src/main/native/libhdfs/hdfs.h | 13 + .../main/native/libhdfs/test_libhdfs_threaded.c | 4 + .../src/main/proto/ClientNamenodeProtocol.proto | 3 + .../src/main/proto/datatransfer.proto | 3 +- .../hadoop-hdfs/src/main/proto/fsimage.proto | 10 + .../hadoop-hdfs/src/main/proto/inotify.proto | 1 + .../src/main/resources/hdfs-default.xml | 35 +- .../hadoop-hdfs/src/main/shellprofile.d/hdfs | 36 + .../src/main/webapps/hdfs/dfshealth.html | 9 +- .../src/site/apt/HDFSCommands.apt.vm | 5 +- .../hadoop-hdfs/src/site/apt/HdfsDesign.apt.vm | 12 +- .../src/site/apt/HdfsQuotaAdminGuide.apt.vm | 10 +- .../hadoop-hdfs/src/site/apt/WebHDFS.apt.vm | 46 + .../fs/TestHDFSFileContextMainOperations.java | 32 +- .../org/apache/hadoop/fs/TestSymlinkHdfs.java | 3 +- .../org/apache/hadoop/hdfs/AppendTestUtil.java | 56 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 49 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 4 +- .../hdfs/TestDFSInotifyEventInputStream.java | 9 +- .../hadoop/hdfs/TestDataTransferProtocol.java | 10 +- .../apache/hadoop/hdfs/TestDatanodeDeath.java | 17 +- .../apache/hadoop/hdfs/TestDecommission.java | 10 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 10 +- .../hadoop/hdfs/TestEncryptionZonesWithHA.java | 2 +- .../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 2 +- .../org/apache/hadoop/hdfs/TestFileAppend.java | 165 ++- .../org/apache/hadoop/hdfs/TestFileAppend2.java | 207 ++- .../org/apache/hadoop/hdfs/TestFileAppend3.java | 227 ++- .../org/apache/hadoop/hdfs/TestFileAppend4.java | 11 +- .../hadoop/hdfs/TestFileAppendRestart.java | 10 +- .../apache/hadoop/hdfs/TestFileCorruption.java | 15 +- .../apache/hadoop/hdfs/TestFileCreation.java | 5 +- .../hadoop/hdfs/TestFileCreationDelete.java | 9 +- .../java/org/apache/hadoop/hdfs/TestHFlush.java | 132 +- .../hadoop/hdfs/TestKeyProviderCache.java | 124 ++ .../apache/hadoop/hdfs/TestLeaseRecovery.java | 6 +- .../org/apache/hadoop/hdfs/TestPipelines.java | 10 +- .../apache/hadoop/hdfs/TestRenameWhileOpen.java | 9 +- .../hadoop/hdfs/TestReservedRawPaths.java | 4 +- .../org/apache/hadoop/hdfs/TestSafeMode.java | 17 +- .../hdfs/server/balancer/TestBalancer.java | 6 +- .../TestBalancerWithMultipleNameNodes.java | 8 +- .../server/blockmanagement/TestBlockInfo.java | 14 +- .../TestBlockInfoUnderConstruction.java | 4 +- .../blockmanagement/TestBlockManager.java | 20 +- .../blockmanagement/TestDatanodeDescriptor.java | 4 +- .../blockmanagement/TestDatanodeManager.java | 84 ++ .../blockmanagement/TestHeartbeatHandling.java | 6 +- .../TestPendingInvalidateBlock.java | 4 +- .../blockmanagement/TestReplicationPolicy.java | 12 +- .../server/datanode/BlockReportTestBase.java | 8 +- .../hdfs/server/datanode/TestBlockScanner.java | 36 +- .../server/datanode/TestCachingStrategy.java | 30 + .../datanode/TestDataNodeHotSwapVolumes.java | 34 + .../datanode/TestIncrementalBrVariations.java | 13 +- .../extdataset/ExternalDatasetImpl.java | 2 +- .../fsdataset/impl/LazyPersistTestCase.java | 8 +- .../fsdataset/impl/TestLazyPersistFiles.java | 5 +- .../hadoop/hdfs/server/mover/TestMover.java | 8 +- .../hdfs/server/namenode/CreateEditsLog.java | 8 +- .../hdfs/server/namenode/FSImageTestUtil.java | 2 +- .../server/namenode/NNThroughputBenchmark.java | 14 +- .../hdfs/server/namenode/TestAddBlock.java | 12 +- .../namenode/TestBlockUnderConstruction.java | 10 +- .../server/namenode/TestCacheDirectives.java | 2 +- .../TestCommitBlockSynchronization.java | 16 +- .../hdfs/server/namenode/TestDeleteRace.java | 32 +- .../namenode/TestDiskspaceQuotaUpdate.java | 34 +- .../hdfs/server/namenode/TestEditLog.java | 4 +- .../hdfs/server/namenode/TestFSImage.java | 4 +- .../namenode/TestFSImageWithSnapshot.java | 2 +- .../hdfs/server/namenode/TestFileTruncate.java | 219 ++- .../hadoop/hdfs/server/namenode/TestFsck.java | 4 +- .../namenode/TestFsckWithMultipleNameNodes.java | 5 +- .../hdfs/server/namenode/TestHDFSConcat.java | 78 +- .../hdfs/server/namenode/TestINodeFile.java | 4 +- .../namenode/TestListCorruptFileBlocks.java | 7 +- .../server/namenode/TestNamenodeRetryCache.java | 16 +- .../server/namenode/TestQuotaByStorageType.java | 524 +++++++ .../hdfs/server/namenode/ha/TestDNFencing.java | 11 +- .../hdfs/server/namenode/ha/TestHAAppend.java | 51 +- .../hdfs/server/namenode/ha/TestHASafeMode.java | 5 +- .../namenode/ha/TestPipelinesFailover.java | 15 +- .../namenode/ha/TestRetryCacheWithHA.java | 16 +- .../namenode/ha/TestStandbyBlockManagement.java | 20 +- .../server/namenode/ha/TestStandbyIsHot.java | 7 +- .../namenode/snapshot/SnapshotTestHelper.java | 10 +- .../namenode/snapshot/TestAclWithSnapshot.java | 3 - .../namenode/snapshot/TestNestedSnapshots.java | 41 +- .../snapshot/TestRenameWithSnapshots.java | 107 +- .../snapshot/TestSnapshotBlocksMap.java | 24 +- .../namenode/snapshot/TestSnapshotDeletion.java | 178 +-- .../snapshot/TestXAttrWithSnapshot.java | 71 - .../web/resources/TestWebHdfsDataLocality.java | 8 +- .../TestOfflineImageViewer.java | 54 +- .../hdfs/web/TestFSMainOperationsWebHdfs.java | 29 + .../web/TestWebHdfsWithMultipleNameNodes.java | 13 +- .../apache/hadoop/tracing/TestTraceAdmin.java | 4 +- .../org/apache/hadoop/tracing/TestTracing.java | 16 +- .../TestTracingShortCircuitLocalRead.java | 8 +- ....apache.hadoop.crypto.key.KeyProviderFactory | 16 + .../hadoop-hdfs/src/test/resources/editsStored | Bin 5586 -> 5850 bytes .../src/test/resources/editsStored.xml | 442 +++--- .../src/test/resources/testHDFSConf.xml | 246 +++- hadoop-mapreduce-project/CHANGES.txt | 49 + hadoop-mapreduce-project/bin/mapred-config.sh | 30 +- hadoop-mapreduce-project/conf/mapred-env.sh | 35 +- .../v2/app/rm/RMContainerAllocator.java | 5 +- .../v2/app/speculate/DefaultSpeculator.java | 62 +- .../mapreduce/v2/app/webapp/AppController.java | 1 - .../mapreduce/v2/app/webapp/NavBlock.java | 2 +- .../mapreduce/v2/app/TestRuntimeEstimators.java | 16 + .../v2/app/rm/TestRMContainerAllocator.java | 93 ++ .../v2/api/protocolrecords/package-info.java | 2 - .../apache/hadoop/mapred/jobcontrol/Job.java | 4 +- .../mapred/lib/CombineFileInputFormat.java | 4 +- .../apache/hadoop/mapred/lib/InputSampler.java | 2 +- .../hadoop/mapred/lib/db/DBInputFormat.java | 2 +- .../java/org/apache/hadoop/mapreduce/Job.java | 4 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 29 + .../hadoop/mapreduce/counters/package-info.java | 2 - .../lib/aggregate/ValueAggregatorJob.java | 2 +- .../hadoop/mapreduce/lib/db/DBInputFormat.java | 31 +- .../lib/db/DataDrivenDBInputFormat.java | 5 +- .../lib/db/OracleDataDrivenDBInputFormat.java | 2 +- .../lib/input/DelegatingInputFormat.java | 2 +- .../mapreduce/lib/jobcontrol/ControlledJob.java | 2 +- .../hadoop/mapreduce/lib/join/Parser.java | 2 +- .../mapreduce/lib/output/MultipleOutputs.java | 2 +- .../mapreduce/lib/partition/InputSampler.java | 2 +- .../lib/partition/TotalOrderPartitioner.java | 2 +- .../hadoop/mapreduce/util/ConfigUtil.java | 4 +- .../src/main/resources/mapred-default.xml | 41 +- .../src/site/apt/HadoopStreaming.apt.vm | 792 ----------- .../src/site/markdown/DistCp.md.vm | 512 ------- .../src/site/markdown/HadoopArchives.md.vm | 162 --- .../mapreduce/v2/hs/HistoryFileManager.java | 8 + .../hadoop/mapreduce/v2/hs/package-info.java | 2 - .../mapreduce/v2/hs/webapp/HsNavBlock.java | 2 +- .../resources/job_1329348432655_0001_conf.xml | 2 - .../org/apache/hadoop/RandomTextWriterJob.java | 2 +- .../java/org/apache/hadoop/fs/TestDFSIO.java | 91 +- .../hadoop/mapred/TestMiniMRChildTask.java | 4 +- .../hadoop/mapred/TestMiniMRClientCluster.java | 2 +- .../apache/hadoop/mapreduce/LargeSorter.java | 3 +- .../hadoop/mapreduce/RandomTextWriter.java | 2 +- .../apache/hadoop/mapreduce/RandomWriter.java | 2 +- .../apache/hadoop/mapreduce/TestLargeSort.java | 1 + .../apache/hadoop/mapreduce/v2/TestMRJobs.java | 2 +- .../test/java/testjar/UserNamePermission.java | 2 +- .../hadoop/examples/BaileyBorweinPlouffe.java | 2 +- .../java/org/apache/hadoop/examples/Grep.java | 6 +- .../java/org/apache/hadoop/examples/Join.java | 2 +- .../hadoop/examples/MultiFileWordCount.java | 2 +- .../apache/hadoop/examples/QuasiMonteCarlo.java | 2 +- .../hadoop/examples/RandomTextWriter.java | 2 +- .../apache/hadoop/examples/RandomWriter.java | 2 +- .../apache/hadoop/examples/SecondarySort.java | 2 +- .../java/org/apache/hadoop/examples/Sort.java | 2 +- .../org/apache/hadoop/examples/WordCount.java | 2 +- .../org/apache/hadoop/examples/WordMean.java | 3 +- .../org/apache/hadoop/examples/WordMedian.java | 3 +- .../hadoop/examples/WordStandardDeviation.java | 3 +- .../examples/dancing/DistributedPentomino.java | 2 +- .../org/apache/hadoop/examples/pi/DistSum.java | 3 +- .../shellprofile.d/mapreduce | 41 + hadoop-project/pom.xml | 62 +- hadoop-project/src/site/site.xml | 17 +- .../org/apache/hadoop/tools/HadoopArchives.java | 16 +- .../src/site/markdown/HadoopArchives.md.vm | 162 +++ .../src/site/resources/css/site.css | 30 + .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 30 +- .../src/site/markdown/tools/hadoop-aws/index.md | 2 +- .../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 3 + hadoop-tools/hadoop-azure/pom.xml | 13 + .../hadoop-azure/src/config/checkstyle.xml | 1 - .../fs/azure/AzureNativeFileSystemStore.java | 5 +- .../hadoop/fs/azure/NativeAzureFileSystem.java | 32 +- .../fs/azure/AzureBlobStorageTestAccount.java | 2 +- .../src/site/markdown/DistCp.md.vm | 512 +++++++ .../src/site/resources/css/site.css | 30 + .../hadoop/mapred/gridmix/GridmixJob.java | 6 +- .../src/site/markdown/GridMix.md.vm | 107 +- .../gridmix/TestCompressionEmulationUtils.java | 2 +- .../hadoop/tools/rumen/Hadoop20JHParser.java | 5 - .../hadoop/tools/rumen/HadoopLogsAnalyzer.java | 2 +- .../MapAttempt20LineHistoryEventEmitter.java | 2 +- .../hadoop/tools/rumen/ParsedConfigFile.java | 11 +- .../hadoop/tools/rumen/RandomSeedGenerator.java | 4 +- .../ReduceAttempt20LineHistoryEventEmitter.java | 2 +- .../hadoop-rumen/src/site/markdown/Rumen.md.vm | 135 +- hadoop-tools/hadoop-sls/README | 24 +- .../hadoop-sls/src/main/bin/rumen2sls.sh | 130 +- hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 144 +- .../src/main/data/2jobs2min-rumen-jh.json | 6 - .../src/site/apt/SchedulerLoadSimulator.apt.vm | 5 +- .../src/site/apt/HadoopStreaming.apt.vm | 792 +++++++++++ .../src/site/resources/css/site.css | 30 + .../mapreduce/TestStreamXmlRecordReader.java | 2 +- hadoop-yarn-project/CHANGES.txt | 140 +- .../hadoop-yarn/bin/start-yarn.sh | 6 +- .../hadoop-yarn/bin/stop-yarn.sh | 6 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 88 +- .../hadoop-yarn/bin/yarn-config.sh | 52 +- .../hadoop-yarn/conf/yarn-env.sh | 56 +- .../yarn/api/records/ResourceRequest.java | 21 +- .../hadoop/yarn/conf/YarnConfiguration.java | 20 +- .../hadoop/yarn/client/api/AMRMClient.java | 3 +- .../yarn/client/api/async/AMRMClientAsync.java | 11 + .../api/async/impl/AMRMClientAsyncImpl.java | 13 + .../yarn/client/api/impl/AMRMClientImpl.java | 45 +- .../yarn/client/api/impl/YarnClientImpl.java | 105 +- .../hadoop/yarn/client/cli/RMAdminCLI.java | 37 +- .../yarn/client/api/impl/TestAMRMClient.java | 72 +- .../yarn/client/api/impl/TestYarnClient.java | 161 ++- .../hadoop/yarn/client/cli/TestRMAdminCLI.java | 48 +- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 1 + .../client/api/impl/TimelineClientImpl.java | 37 +- .../nodelabels/CommonNodeLabelsManager.java | 143 +- .../hadoop/yarn/nodelabels/NodeLabel.java | 17 + .../apache/hadoop/yarn/security/AccessType.java | 33 + .../hadoop/yarn/security/AdminACLsManager.java | 31 - .../yarn/security/ConfiguredYarnAuthorizer.java | 97 ++ .../hadoop/yarn/security/PrivilegedEntity.java | 83 ++ .../security/YarnAuthorizationProvider.java | 112 ++ .../yarn/util/AbstractLivelinessMonitor.java | 8 + .../apache/hadoop/yarn/util/ConverterUtils.java | 2 +- .../org/apache/hadoop/yarn/util/FSDownload.java | 5 +- .../org/apache/hadoop/yarn/webapp/WebApps.java | 7 +- .../hadoop/yarn/webapp/view/NavBlock.java | 2 +- .../src/main/resources/yarn-default.xml | 131 +- .../client/api/impl/TestTimelineClient.java | 14 +- .../yarn/nodelabels/NodeLabelTestBase.java | 33 + .../nodelabels/TestCommonNodeLabelsManager.java | 207 ++- .../TestFileSystemNodeLabelsStore.java | 22 +- .../apache/hadoop/yarn/util/TestFSDownload.java | 33 +- .../apache/hadoop/registry/cli/RegistryCli.java | 425 +++--- .../hadoop/registry/cli/TestRegistryCli.java | 197 +++ .../ApplicationHistoryClientService.java | 27 +- ...pplicationHistoryManagerOnTimelineStore.java | 12 +- .../ApplicationHistoryServer.java | 2 + .../timeline/webapp/CrossOriginFilter.java | 18 + .../TestApplicationHistoryClientService.java | 4 + ...pplicationHistoryManagerOnTimelineStore.java | 9 + .../TestApplicationHistoryServer.java | 26 + .../metrics/ApplicationMetricsConstants.java | 6 + .../server/nodemanager/DeletionService.java | 40 +- .../container/ContainerImpl.java | 3 +- .../localizer/ResourceLocalizationService.java | 29 +- .../monitor/ContainerMetrics.java | 42 +- .../monitor/ContainerStartMonitoringEvent.java | 7 +- .../monitor/ContainersMonitorImpl.java | 30 +- .../util/CgroupsLCEResourcesHandler.java | 73 +- .../server/nodemanager/webapp/NavBlock.java | 2 +- .../impl/container-executor.c | 28 +- .../test/test-container-executor.c | 21 + .../TestLocalResourcesTrackerImpl.java | 5 +- .../TestResourceLocalizationService.java | 105 +- .../monitor/TestContainerMetrics.java | 40 +- .../metrics/TestNodeManagerMetrics.java | 6 + .../util/TestCgroupsLCEResourcesHandler.java | 80 +- .../server/resourcemanager/AdminService.java | 34 +- .../ApplicationMasterService.java | 7 +- .../server/resourcemanager/ClientRMService.java | 19 +- .../server/resourcemanager/RMAppManager.java | 3 +- .../server/resourcemanager/RMServerUtils.java | 25 +- .../server/resourcemanager/ResourceManager.java | 2 + .../metrics/ApplicationFinishedEvent.java | 9 +- .../metrics/SystemMetricsPublisher.java | 10 +- .../ProportionalCapacityPreemptionPolicy.java | 20 +- .../nodelabels/RMNodeLabelsManager.java | 23 +- .../recovery/ZKRMStateStore.java | 17 +- .../rmapp/attempt/AMLivelinessMonitor.java | 6 + .../rmapp/attempt/RMAppAttemptImpl.java | 18 +- .../resourcemanager/rmnode/RMNodeImpl.java | 5 +- .../scheduler/AbstractYarnScheduler.java | 70 +- .../scheduler/ResourceUsage.java | 80 +- .../scheduler/SchedulerUtils.java | 35 +- .../scheduler/YarnScheduler.java | 11 +- .../scheduler/capacity/AbstractCSQueue.java | 145 +- .../scheduler/capacity/CSQueue.java | 6 + .../scheduler/capacity/CapacityScheduler.java | 37 +- .../CapacitySchedulerConfiguration.java | 90 +- .../capacity/CapacitySchedulerContext.java | 2 + .../scheduler/capacity/LeafQueue.java | 174 ++- .../scheduler/capacity/ParentQueue.java | 72 +- .../scheduler/capacity/QueueCapacities.java | 191 +++ .../scheduler/fair/FSAppAttempt.java | 19 +- .../scheduler/fair/FairScheduler.java | 13 +- .../security/AMRMTokenSecretManager.java | 2 +- .../security/NMTokenSecretManagerInRM.java | 2 +- .../security/RMContainerTokenSecretManager.java | 4 +- .../webapp/CapacitySchedulerPage.java | 3 +- .../server/resourcemanager/webapp/NavBlock.java | 2 +- .../resourcemanager/webapp/RMWebServices.java | 10 +- .../dao/CapacitySchedulerLeafQueueInfo.java | 6 + .../yarn/server/resourcemanager/MockAM.java | 17 +- .../server/resourcemanager/TestAppManager.java | 3 +- .../server/resourcemanager/TestRMRestart.java | 6 +- .../TestWorkPreservingRMRestart.java | 2 +- .../metrics/TestSystemMetricsPublisher.java | 17 +- ...estProportionalCapacityPreemptionPolicy.java | 68 +- .../nodelabels/TestRMNodeLabelsManager.java | 77 +- .../recovery/TestZKRMStateStore.java | 4 +- .../rmapp/attempt/TestAMLivelinessMonitor.java | 81 ++ .../attempt/TestRMAppAttemptTransitions.java | 9 +- .../scheduler/TestAbstractYarnScheduler.java | 62 + .../scheduler/TestSchedulerUtils.java | 54 +- .../scheduler/capacity/TestCSQueueUtils.java | 24 +- .../capacity/TestCapacityScheduler.java | 355 ++++- .../capacity/TestContainerAllocation.java | 32 +- .../scheduler/capacity/TestParentQueue.java | 5 +- .../scheduler/capacity/TestQueueCapacities.java | 127 ++ .../fair/TestContinuousScheduling.java | 114 ++ .../scheduler/fair/TestFairScheduler.java | 112 +- .../webapp/TestRMWebServices.java | 55 + .../webapp/TestRMWebServicesCapacitySched.java | 2 +- .../webapp/TestRMWebServicesNodeLabels.java | 17 +- .../SCMAdminProtocolService.java | 10 +- .../src/site/apt/CapacityScheduler.apt.vm | 12 + .../src/site/markdown/registry/index.md | 1 + .../markdown/registry/registry-configuration.md | 453 ++++++ .../site/markdown/registry/registry-security.md | 4 +- .../registry/using-the-yarn-service-registry.md | 235 +++- .../src/site/markdown/registry/yarn-registry.md | 73 +- .../hadoop-yarn/shellprofile.d/yarn | 62 + pom.xml | 40 +- 581 files changed, 24402 insertions(+), 13496 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2786151/hadoop-project/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2786151/hadoop-yarn-project/CHANGES.txt ----------------------------------------------------------------------
