Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f62ba55 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f62ba55 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f62ba55 Branch: refs/heads/HDFS-7240 Commit: 3f62ba558d33a3d385359e04a445a54441d6d53b Parents: fedb22d a62637a Author: Anu Engineer <aengin...@apache.org> Authored: Thu Apr 7 14:43:39 2016 -0700 Committer: Anu Engineer <aengin...@apache.org> Committed: Thu Apr 7 14:43:39 2016 -0700 ---------------------------------------------------------------------- BUILDING.txt | 4 +- .../server/AuthenticationFilter.java | 33 +- .../hadoop-common/src/main/bin/hadoop | 12 +- .../hadoop-common/src/main/bin/hadoop-config.sh | 6 +- .../hadoop-common/src/main/bin/hadoop-daemon.sh | 6 +- .../src/main/bin/hadoop-daemons.sh | 6 +- .../src/main/bin/hadoop-functions.sh | 37 +- .../src/main/bin/hadoop-layout.sh.example | 14 +- .../hadoop-common/src/main/bin/slaves.sh | 6 +- .../hadoop-common/src/main/bin/start-all.sh | 4 +- .../hadoop-common/src/main/bin/stop-all.sh | 4 +- .../hadoop-common/src/main/conf/hadoop-env.sh | 10 +- .../org/apache/hadoop/conf/Configuration.java | 13 + .../hadoop/fs/CommonConfigurationKeys.java | 14 +- .../java/org/apache/hadoop/fs/FileUtil.java | 41 - .../org/apache/hadoop/fs/PathIOException.java | 9 + .../hadoop/fs/shell/CommandWithDestination.java | 3 +- .../apache/hadoop/fs/shell/MoveCommands.java | 6 +- .../java/org/apache/hadoop/fs/shell/Touch.java | 3 +- .../org/apache/hadoop/ipc/CallQueueManager.java | 134 ++- .../main/java/org/apache/hadoop/ipc/Client.java | 130 ++- .../apache/hadoop/ipc/DecayRpcScheduler.java | 396 +++++-- .../hadoop/ipc/DecayRpcSchedulerMXBean.java | 2 + .../apache/hadoop/ipc/DefaultRpcScheduler.java | 45 + .../org/apache/hadoop/ipc/FairCallQueue.java | 45 +- .../apache/hadoop/ipc/ProtobufRpcEngine.java | 8 +- .../org/apache/hadoop/ipc/RpcScheduler.java | 8 +- .../java/org/apache/hadoop/ipc/Schedulable.java | 5 +- .../main/java/org/apache/hadoop/ipc/Server.java | 77 +- .../apache/hadoop/ipc/WritableRpcEngine.java | 45 +- .../hadoop/metrics2/lib/MutableQuantiles.java | 7 +- .../hadoop/metrics2/util/QuantileEstimator.java | 32 + .../hadoop/metrics2/util/SampleQuantiles.java | 2 +- .../java/org/apache/hadoop/net/NetUtils.java | 15 +- .../apache/hadoop/security/SecurityUtil.java | 53 +- .../security/http/RestCsrfPreventionFilter.java | 2 +- .../apache/hadoop/security/ssl/SSLFactory.java | 42 +- .../org/apache/hadoop/tracing/TraceUtils.java | 4 +- .../hadoop/util/NativeLibraryChecker.java | 8 +- .../apache/hadoop/util/ShutdownHookManager.java | 116 +- .../hadoop/io/compress/lz4/Lz4Compressor.c | 4 +- .../hadoop/io/erasurecode/erasure_coder.c | 1 + .../apache/hadoop/io/erasurecode/isal_load.c | 46 +- .../apache/hadoop/io/erasurecode/isal_load.h | 6 +- .../io/erasurecode/jni_erasure_code_native.c | 11 +- .../src/main/resources/core-default.xml | 9 +- .../src/site/markdown/ClusterSetup.md | 40 +- .../src/site/markdown/CommandsManual.md | 2 +- .../src/site/markdown/UnixShellGuide.md | 2 +- .../apache/hadoop/conf/TestConfiguration.java | 8 +- .../crypto/TestCryptoStreamsForLocalFS.java | 5 +- .../apache/hadoop/crypto/key/TestKeyShell.java | 5 +- .../org/apache/hadoop/fs/FSTestWrapper.java | 3 +- .../fs/FileContextMainOperationsBaseTest.java | 4 +- .../apache/hadoop/fs/FileContextTestHelper.java | 6 +- .../apache/hadoop/fs/FileContextURIBase.java | 6 +- .../apache/hadoop/fs/FileSystemTestHelper.java | 4 +- .../org/apache/hadoop/fs/TestAvroFSInput.java | 9 +- .../hadoop/fs/TestChecksumFileSystem.java | 5 +- .../org/apache/hadoop/fs/TestDFVariations.java | 2 +- .../test/java/org/apache/hadoop/fs/TestDU.java | 4 +- .../hadoop/fs/TestFileContextResolveAfs.java | 8 +- .../java/org/apache/hadoop/fs/TestFileUtil.java | 66 +- .../org/apache/hadoop/fs/TestFsShellCopy.java | 56 +- .../apache/hadoop/fs/TestFsShellReturnCode.java | 8 +- .../org/apache/hadoop/fs/TestFsShellTouch.java | 88 ++ .../hadoop/fs/TestGetFileBlockLocations.java | 5 +- .../hadoop/fs/TestHarFileSystemBasics.java | 5 +- .../java/org/apache/hadoop/fs/TestHardLink.java | 5 +- .../org/apache/hadoop/fs/TestListFiles.java | 14 +- .../apache/hadoop/fs/TestLocalFileSystem.java | 7 +- .../fs/TestLocalFileSystemPermission.java | 18 +- .../java/org/apache/hadoop/fs/TestPath.java | 6 +- .../java/org/apache/hadoop/fs/TestTrash.java | 6 +- .../apache/hadoop/fs/TestTruncatedInputBug.java | 4 +- .../AbstractContractGetFileStatusTest.java | 23 + .../hadoop/fs/sftp/TestSFTPFileSystem.java | 3 +- .../apache/hadoop/fs/shell/TestPathData.java | 5 +- .../apache/hadoop/fs/shell/TestTextCommand.java | 4 +- .../hadoop/fs/viewfs/TestViewfsFileStatus.java | 7 +- .../apache/hadoop/ha/ClientBaseWithFixes.java | 4 +- .../http/TestAuthenticationSessionCookie.java | 5 +- .../apache/hadoop/http/TestHttpCookieFlag.java | 5 +- .../hadoop/http/TestHttpServerLifecycle.java | 5 +- .../apache/hadoop/http/TestSSLHttpServer.java | 5 +- .../org/apache/hadoop/io/TestArrayFile.java | 6 +- .../org/apache/hadoop/io/TestBloomMapFile.java | 6 +- .../java/org/apache/hadoop/io/TestMapFile.java | 6 +- .../org/apache/hadoop/io/TestSequenceFile.java | 48 +- .../hadoop/io/TestSequenceFileAppend.java | 4 +- .../io/TestSequenceFileSerialization.java | 4 +- .../apache/hadoop/io/TestSequenceFileSync.java | 5 +- .../java/org/apache/hadoop/io/TestSetFile.java | 5 +- .../apache/hadoop/io/compress/TestCodec.java | 23 +- .../apache/hadoop/io/file/tfile/TestTFile.java | 4 +- .../io/file/tfile/TestTFileByteArrays.java | 4 +- .../io/file/tfile/TestTFileComparator2.java | 4 +- .../io/file/tfile/TestTFileComparators.java | 5 +- .../hadoop/io/file/tfile/TestTFileSeek.java | 4 +- .../file/tfile/TestTFileSeqFileComparison.java | 5 +- .../hadoop/io/file/tfile/TestTFileSplit.java | 4 +- .../hadoop/io/file/tfile/TestTFileStreams.java | 4 +- .../file/tfile/TestTFileUnsortedByteArrays.java | 5 +- .../apache/hadoop/io/file/tfile/TestVLong.java | 4 +- .../apache/hadoop/io/nativeio/TestNativeIO.java | 10 +- .../TestSharedFileDescriptorFactory.java | 4 +- .../org/apache/hadoop/ipc/TestAsyncIPC.java | 346 ++++++ .../apache/hadoop/ipc/TestCallQueueManager.java | 156 ++- .../hadoop/ipc/TestDecayRpcScheduler.java | 42 +- .../apache/hadoop/ipc/TestFairCallQueue.java | 79 +- .../java/org/apache/hadoop/ipc/TestIPC.java | 29 +- .../hadoop/ipc/TestIdentityProviders.java | 18 +- .../java/org/apache/hadoop/ipc/TestRPC.java | 159 ++- .../sink/RollingFileSystemSinkTestBase.java | 6 +- .../org/apache/hadoop/net/TestNetUtils.java | 47 +- .../apache/hadoop/security/TestCredentials.java | 4 +- .../hadoop/security/TestLdapGroupsMapping.java | 7 +- .../hadoop/security/TestSecurityUtil.java | 16 +- .../hadoop/security/alias/TestCredShell.java | 4 +- .../alias/TestCredentialProviderFactory.java | 5 +- .../hadoop/security/ssl/KeyStoreTestUtil.java | 4 +- .../ssl/TestReloadingX509TrustManager.java | 6 +- .../hadoop/security/ssl/TestSSLFactory.java | 145 ++- .../apache/hadoop/test/GenericTestUtils.java | 84 +- .../java/org/apache/hadoop/util/JarFinder.java | 4 +- .../hadoop/util/TestApplicationClassLoader.java | 4 +- .../org/apache/hadoop/util/TestClasspath.java | 5 +- .../hadoop/util/TestGenericOptionsParser.java | 2 +- .../apache/hadoop/util/TestHostsFileReader.java | 4 +- .../org/apache/hadoop/util/TestJarFinder.java | 9 +- .../java/org/apache/hadoop/util/TestRunJar.java | 8 +- .../java/org/apache/hadoop/util/TestShell.java | 4 +- .../hadoop/util/TestShutdownHookManager.java | 57 +- .../apache/hadoop/util/TestSysInfoLinux.java | 5 +- .../org/apache/hadoop/util/TestWinUtils.java | 5 +- .../java/org/apache/hadoop/util/TestZKUtil.java | 5 +- .../scripts/hadoop-functions_test_helper.bash | 3 +- .../src/test/scripts/hadoop_basic_init.bats | 2 +- .../src/test/scripts/hadoop_bootstrap.bats | 4 +- .../src/test/scripts/hadoop_confdir.bats | 24 +- .../src/test/scripts/hadoop_finalize.bats | 2 +- .../hadoop-kms/src/main/conf/kms-env.sh | 4 +- .../hadoop-kms/src/main/libexec/kms-config.sh | 8 +- .../hadoop-kms/src/main/sbin/kms.sh | 4 +- .../apache/hadoop/hdfs/BlockReaderFactory.java | 4 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 4 +- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 6 +- .../hadoop/hdfs/client/impl/DfsClientConf.java | 2 +- .../hdfs/protocol/ClientDatanodeProtocol.java | 7 + .../ClientDatanodeProtocolTranslatorPB.java | 12 + .../hadoop/hdfs/util/StripedBlockUtil.java | 22 +- .../src/main/proto/ClientDatanodeProtocol.proto | 10 + .../src/main/conf/httpfs-env.sh | 4 +- .../src/main/libexec/httpfs-config.sh | 8 +- .../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 4 +- .../src/main/native/fuse-dfs/doc/README | 6 +- .../main/native/fuse-dfs/fuse_dfs_wrapper.sh | 12 +- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 + .../bkjournal/TestBookKeeperHACheckpoints.java | 46 +- .../src/main/bin/distribute-exclude.sh | 4 +- .../hadoop-hdfs/src/main/bin/hdfs | 4 +- .../hadoop-hdfs/src/main/bin/hdfs-config.sh | 6 +- .../src/main/bin/refresh-namenodes.sh | 4 +- .../hadoop-hdfs/src/main/bin/start-balancer.sh | 4 +- .../hadoop-hdfs/src/main/bin/start-dfs.sh | 4 +- .../src/main/bin/start-secure-dns.sh | 4 +- .../hadoop-hdfs/src/main/bin/stop-balancer.sh | 4 +- .../hadoop-hdfs/src/main/bin/stop-dfs.sh | 4 +- .../hadoop-hdfs/src/main/bin/stop-secure-dns.sh | 4 +- ...tDatanodeProtocolServerSideTranslatorPB.java | 15 + .../hadoop/hdfs/server/balancer/Balancer.java | 2 +- .../server/blockmanagement/BlockManager.java | 2 +- .../blockmanagement/DatanodeStorageInfo.java | 10 + .../hdfs/server/datanode/BPServiceActor.java | 5 + .../hdfs/server/datanode/BlockReceiver.java | 5 + .../hdfs/server/datanode/BlockSender.java | 6 +- .../hadoop/hdfs/server/datanode/DataNode.java | 7 + .../server/datanode/DataNodeFaultInjector.java | 2 + .../hdfs/server/datanode/DataXceiver.java | 48 +- .../hdfs/server/datanode/DataXceiverServer.java | 6 + .../hdfs/server/datanode/DirectoryScanner.java | 12 +- .../datanode/IncrementalBlockReportManager.java | 9 + .../hdfs/server/datanode/ReplicaInPipeline.java | 54 +- .../erasurecode/ErasureCodingWorker.java | 1017 +--------------- .../erasurecode/StripedBlockReader.java | 202 ++++ .../erasurecode/StripedBlockWriter.java | 196 ++++ .../datanode/erasurecode/StripedReader.java | 466 ++++++++ .../erasurecode/StripedReconstructor.java | 273 +++++ .../datanode/erasurecode/StripedWriter.java | 313 +++++ .../datanode/erasurecode/package-info.java | 26 + .../datanode/fsdataset/impl/FsDatasetImpl.java | 149 ++- .../datanode/fsdataset/impl/ReplicaMap.java | 2 +- .../hdfs/server/namenode/FSDirWriteFileOp.java | 4 + .../hdfs/server/namenode/FSDirectory.java | 48 +- .../hdfs/server/namenode/FSNamesystem.java | 1 + .../hadoop/hdfs/server/namenode/NameNode.java | 9 +- .../hdfs/server/namenode/NamenodeFsck.java | 4 +- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 21 + .../src/main/webapps/hdfs/dfshealth.html | 2 +- .../hadoop-hdfs/src/site/markdown/Federation.md | 18 +- .../markdown/HDFSHighAvailabilityWithNFS.md | 4 +- .../markdown/HDFSHighAvailabilityWithQJM.md | 4 +- .../src/site/markdown/HdfsNfsGateway.md | 8 +- .../org/apache/hadoop/TestRefreshCallQueue.java | 2 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 2 +- .../hadoop/hdfs/TestBlocksScheduledCounter.java | 51 + .../TestClientProtocolForPipelineRecovery.java | 102 ++ .../org/apache/hadoop/hdfs/TestDFSShell.java | 31 + .../hadoop/hdfs/TestReconstructStripedFile.java | 11 +- .../datatransfer/sasl/TestSaslDataTransfer.java | 48 + .../TestBlockManagerSafeMode.java | 14 +- .../blockmanagement/TestBlockTokenWithDFS.java | 2 +- .../server/datanode/TestBPOfferService.java | 96 +- .../hdfs/server/datanode/TestBlockRecovery.java | 259 ++++- .../datanode/TestDataNodeVolumeFailure.java | 2 +- .../server/datanode/TestFsDatasetCache.java | 5 +- .../namenode/TestDecommissioningStatus.java | 12 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 122 +- .../namenode/TestProtectedDirectories.java | 44 + .../hdfs/server/namenode/ha/TestHASafeMode.java | 35 + .../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 2 +- .../apache/hadoop/tracing/TestTraceAdmin.java | 2 +- .../TestTracingShortCircuitLocalRead.java | 4 +- .../src/test/resources/testHDFSConf.xml | 6 +- hadoop-mapreduce-project/bin/mapred | 4 +- hadoop-mapreduce-project/bin/mapred-config.sh | 6 +- .../bin/mr-jobhistory-daemon.sh | 4 +- .../v2/app/job/impl/TaskAttemptImpl.java | 41 +- .../apache/hadoop/mapreduce/v2/app/MRApp.java | 10 +- .../hadoop/mapreduce/v2/app/TestRecovery.java | 29 +- .../v2/app/job/impl/TestTaskAttempt.java | 37 +- .../apache/hadoop/mapred/pipes/Submitter.java | 2 +- .../mapreduce/v2/hs/TestHistoryFileManager.java | 5 +- .../mapreduce/v2/hs/TestJobListCache.java | 2 +- .../java/org/apache/hadoop/fs/DFSCIOTest.java | 10 +- .../org/apache/hadoop/fs/TestFileSystem.java | 20 +- .../java/org/apache/hadoop/fs/TestJHLA.java | 6 +- .../java/org/apache/hadoop/hdfs/NNBench.java | 239 ++-- .../org/apache/hadoop/hdfs/TestNNBench.java | 84 ++ .../io/TestSequenceFileMergeProgress.java | 12 +- .../hadoop/mapred/ClusterMapReduceTestCase.java | 14 +- .../apache/hadoop/mapred/ReliabilityTest.java | 2 +- .../apache/hadoop/mapred/TestAuditLogger.java | 9 +- .../apache/hadoop/mapred/TestBadRecords.java | 8 +- .../mapred/TestClusterMapReduceTestCase.java | 10 + .../org/apache/hadoop/mapred/TestCollect.java | 10 +- .../mapred/TestCommandLineJobSubmission.java | 9 +- .../hadoop/mapred/TestFieldSelection.java | 11 +- .../mapred/TestFileInputFormatPathFilter.java | 19 +- .../apache/hadoop/mapred/TestGetSplitHosts.java | 7 +- .../apache/hadoop/mapred/TestIFileStreams.java | 13 +- .../org/apache/hadoop/mapred/TestInputPath.java | 7 +- .../hadoop/mapred/TestJavaSerialization.java | 10 +- .../org/apache/hadoop/mapred/TestJobName.java | 6 + .../hadoop/mapred/TestJobSysDirWithDFS.java | 10 +- .../mapred/TestKeyValueTextInputFormat.java | 15 +- .../apache/hadoop/mapred/TestLazyOutput.java | 7 +- .../hadoop/mapred/TestMRCJCFileInputFormat.java | 32 +- .../mapred/TestMRCJCFileOutputCommitter.java | 28 +- .../mapred/TestMRTimelineEventHandling.java | 19 +- .../apache/hadoop/mapred/TestMapProgress.java | 9 +- .../org/apache/hadoop/mapred/TestMerge.java | 7 +- .../apache/hadoop/mapred/TestMiniMRBringup.java | 6 +- .../hadoop/mapred/TestMiniMRChildTask.java | 16 +- .../hadoop/mapred/TestMiniMRDFSCaching.java | 14 +- .../hadoop/mapred/TestMultiFileInputFormat.java | 19 +- .../hadoop/mapred/TestMultiFileSplit.java | 10 +- .../hadoop/mapred/TestMultipleLevelCaching.java | 12 +- .../mapred/TestMultipleTextOutputFormat.java | 23 +- .../hadoop/mapred/TestOldCombinerGrouping.java | 5 +- .../apache/hadoop/mapred/TestReduceFetch.java | 10 +- .../mapred/TestReduceFetchFromPartialMem.java | 46 +- .../apache/hadoop/mapred/TestReduceTask.java | 18 +- .../TestSequenceFileAsBinaryInputFormat.java | 19 +- .../TestSequenceFileAsBinaryOutputFormat.java | 31 +- .../TestSequenceFileAsTextInputFormat.java | 33 +- .../mapred/TestSequenceFileInputFilter.java | 32 +- .../mapred/TestSequenceFileInputFormat.java | 29 +- .../apache/hadoop/mapred/TestSortedRanges.java | 19 +- .../TestSpecialCharactersInOutputPath.java | 21 +- .../hadoop/mapred/TestStatisticsCollector.java | 10 +- .../hadoop/mapred/TestUserDefinedCounters.java | 24 +- .../hadoop/mapred/TestWritableJobConf.java | 20 +- .../apache/hadoop/mapred/TestYARNRunner.java | 8 +- .../hadoop/mapred/join/TestDatamerge.java | 42 +- .../hadoop/mapred/join/TestTupleWritable.java | 24 +- .../TestWrappedRecordReaderClassloader.java | 7 +- .../mapred/lib/TestDelegatingInputFormat.java | 9 +- .../hadoop/mapred/lib/TestLineInputFormat.java | 7 +- .../hadoop/mapred/lib/TestMultipleInputs.java | 2 - .../mapred/lib/aggregate/TestAggregates.java | 14 +- .../mapred/lib/db/TestConstructQuery.java | 16 +- .../apache/hadoop/mapred/pipes/TestPipes.java | 9 +- .../hadoop/mapreduce/TestLocalRunner.java | 34 +- .../hadoop/mapreduce/TestMRJobClient.java | 49 +- .../mapreduce/TestMapReduceLazyOutput.java | 9 +- .../mapreduce/TestNewCombinerGrouping.java | 5 +- .../hadoop/mapreduce/TestValueIterReset.java | 8 +- .../TestYarnClientProtocolProvider.java | 5 +- .../lib/aggregate/TestMapReduceAggregates.java | 28 +- .../mapreduce/lib/db/TestDBOutputFormat.java | 17 +- .../mapreduce/lib/db/TestIntegerSplitter.java | 15 +- .../mapreduce/lib/db/TestTextSplitter.java | 18 +- .../lib/fieldsel/TestMRFieldSelection.java | 20 +- .../lib/input/TestCombineFileInputFormat.java | 4 +- .../TestMRSequenceFileAsBinaryInputFormat.java | 21 +- .../TestMRSequenceFileAsTextInputFormat.java | 27 +- .../input/TestMRSequenceFileInputFilter.java | 39 +- .../lib/input/TestNLineInputFormat.java | 34 +- .../mapreduce/lib/join/TestJoinDatamerge.java | 52 +- .../mapreduce/lib/join/TestJoinProperties.java | 44 +- .../lib/join/TestJoinTupleWritable.java | 24 +- .../lib/join/TestWrappedRRClassloader.java | 17 +- .../TestMRSequenceFileAsBinaryOutputFormat.java | 35 +- .../lib/partition/TestBinaryPartitioner.java | 16 +- .../lib/partition/TestKeyFieldHelper.java | 9 +- .../TestMRKeyFieldBasedPartitioner.java | 6 +- .../partition/TestTotalOrderPartitioner.java | 11 +- .../mapreduce/util/TestMRAsyncDiskService.java | 15 +- .../mapreduce/v2/TestMiniMRProxyUser.java | 30 +- .../hadoop/mapreduce/v2/TestNonExistentJob.java | 18 +- .../apache/hadoop/tools/HadoopArchiveLogs.java | 4 +- .../hadoop/tools/TestHadoopArchiveLogs.java | 4 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 77 +- .../src/site/markdown/tools/hadoop-aws/index.md | 7 + .../s3a/TestS3AContractGetFileStatus.java | 31 + .../src/test/resources/contract/s3a.xml | 5 + .../apache/hadoop/contrib/utils/join/README.txt | 2 +- .../native/pipes/debug/pipes-default-script | 5 +- .../hadoop-sls/src/main/bin/rumen2sls.sh | 4 +- hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 8 +- .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 9 + .../yarn/sls/scheduler/RMNodeWrapper.java | 9 + .../apache/hadoop/streaming/DumpTypedBytes.java | 2 +- .../hadoop/streaming/HadoopStreaming.java | 2 +- .../apache/hadoop/streaming/LoadTypedBytes.java | 2 +- .../org/apache/hadoop/streaming/StreamJob.java | 16 +- .../streaming/TestStreamingBadRecords.java | 9 +- .../hadoop-yarn/bin/start-yarn.sh | 4 +- .../hadoop-yarn/bin/stop-yarn.sh | 4 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 4 +- .../hadoop-yarn/bin/yarn-config.sh | 6 +- .../hadoop-yarn/bin/yarn-daemon.sh | 4 +- .../hadoop-yarn/bin/yarn-daemons.sh | 4 +- .../yarn/api/records/ResourceUtilization.java | 22 + .../hadoop/yarn/conf/YarnConfiguration.java | 9 + .../distributedshell/ApplicationMaster.java | 13 +- .../apache/hadoop/yarn/util/ConverterUtils.java | 16 +- .../hadoop/yarn/util/resource/Resources.java | 2 +- .../hadoop/yarn/webapp/util/WebAppUtils.java | 22 + .../src/main/resources/yarn-default.xml | 26 +- .../api/records/TestResourceUtilization.java | 63 + .../ApplicationHistoryServer.java | 15 +- .../hadoop/yarn/server/webapp/AppBlock.java | 20 + .../hadoop/yarn/server/webapp/WebPageUtils.java | 6 + .../hadoop/yarn/server/webapp/WebServices.java | 22 +- .../hadoop-yarn-server-nodemanager/pom.xml | 5 + .../container/ContainerImpl.java | 22 + .../logaggregation/AppLogAggregatorImpl.java | 14 +- .../monitor/ContainerMetrics.java | 87 ++ .../nodemanager/webapp/NMWebServices.java | 6 +- .../TestDockerContainerExecutorWithMocks.java | 2 +- .../containermanager/TestAuxServices.java | 2 +- .../containermanager/TestNMProxy.java | 3 +- .../container/TestContainer.java | 11 + .../TestLogAggregationService.java | 12 +- .../monitor/TestContainerMetrics.java | 58 +- .../webapp/TestNMWebServicesApps.java | 9 +- .../server/resourcemanager/AdminService.java | 40 +- .../resourcemanager/NodesListManager.java | 104 +- .../server/resourcemanager/RMServerUtils.java | 2 +- .../resourcemanager/ResourceTrackerService.java | 88 +- .../monitor/SchedulingEditPolicy.java | 8 +- .../monitor/SchedulingMonitor.java | 4 - .../CapacitySchedulerPreemptionContext.java | 52 + .../CapacitySchedulerPreemptionUtils.java | 65 ++ .../capacity/FifoCandidatesSelector.java | 364 ++++++ .../capacity/PreemptableResourceCalculator.java | 370 ++++++ .../capacity/PreemptionCandidatesSelector.java | 52 + .../ProportionalCapacityPreemptionPolicy.java | 1086 ++++-------------- .../monitor/capacity/TempQueuePerPartition.java | 159 +++ .../resource/DynamicResourceConfiguration.java | 13 +- .../server/resourcemanager/rmnode/RMNode.java | 4 + .../resourcemanager/rmnode/RMNodeImpl.java | 22 +- .../server/resourcemanager/scheduler/Queue.java | 20 + .../scheduler/capacity/AbstractCSQueue.java | 32 +- .../scheduler/capacity/CSQueueUtils.java | 15 +- .../CapacitySchedulerConfiguration.java | 45 + .../scheduler/capacity/LeafQueue.java | 16 +- .../scheduler/capacity/QueueCapacities.java | 38 +- .../capacity/preemption/PreemptableQueue.java | 6 - .../capacity/preemption/PreemptionManager.java | 2 +- .../resourcemanager/scheduler/fair/FSQueue.java | 8 + .../scheduler/fifo/FifoScheduler.java | 12 + .../resourcemanager/webapp/AppAttemptPage.java | 4 +- .../webapp/CapacitySchedulerPage.java | 18 +- .../resourcemanager/webapp/NodesPage.java | 53 +- .../webapp/RMAppAttemptBlock.java | 61 +- .../resourcemanager/webapp/RMWebServices.java | 32 +- .../yarn/server/resourcemanager/MockNodes.java | 9 + .../yarn/server/resourcemanager/MockRM.java | 7 +- .../resourcemanager/TestRMAdminService.java | 77 +- .../server/resourcemanager/TestRMRestart.java | 2 - .../TestResourceTrackerService.java | 222 +++- ...estProportionalCapacityPreemptionPolicy.java | 133 +-- ...pacityPreemptionPolicyForNodePartitions.java | 78 +- .../recovery/TestZKRMStateStorePerf.java | 9 +- .../rmapp/TestNodesListManager.java | 5 +- .../TestCapacitySchedulerNodeLabelUpdate.java | 40 +- .../TestCapacitySchedulerPreemption.java | 14 +- .../capacity/TestContainerAllocation.java | 13 +- .../TestNodeLabelContainerAllocation.java | 79 +- .../scheduler/capacity/TestQueueCapacities.java | 4 +- .../security/TestDelegationTokenRenewer.java | 3 - .../resourcemanager/webapp/TestNodesPage.java | 37 +- .../webapp/TestRMWebServicesApps.java | 24 +- .../TestRMWebServicesAppsModification.java | 10 +- .../webapp/TestRMWebServicesNodes.java | 12 +- .../src/site/markdown/CapacityScheduler.md | 10 +- .../site/markdown/DockerContainerExecutor.md.vm | 2 +- .../src/site/markdown/SecureContainer.md | 2 +- .../src/site/markdown/TimelineServer.md | 8 +- .../src/site/markdown/YarnCommands.md | 45 +- 423 files changed, 9534 insertions(+), 4160 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f62ba55/hadoop-hdfs-project/hadoop-hdfs/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f62ba55/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f62ba55/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ----------------------------------------------------------------------
