Merge branch 'trunk' into HDFS-7240
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
hadoop-hdfs-project/hadoop-hdfs/pom.xml
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
hadoop-project/pom.xml
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0bd0f62
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0bd0f62
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0bd0f62
Branch: refs/heads/HDFS-7240
Commit: d0bd0f623338dbb558d0dee5e747001d825d92c5
Parents: 7d13259 b6bfb2f
Author: Anu Engineer <aengin...@apache.org>
Authored: Mon Aug 21 18:57:15 2017 -0700
Committer: Anu Engineer <aengin...@apache.org>
Committed: Mon Aug 21 18:57:15 2017 -0700
----------------------------------------------------------------------
LICENSE.txt | 1 +
dev-support/docker/hadoop_env_checks.sh | 2 +-
dev-support/findHangingTest.sh | 2 +-
dev-support/verify-xml.sh | 2 +-
.../resources/assemblies/hadoop-yarn-dist.xml | 1 +
.../hadoop-client-check-invariants/pom.xml | 1 -
.../hadoop-client-check-test-invariants/pom.xml | 1 -
.../hadoop-client-minicluster/pom.xml | 33 +
.../hadoop-client-runtime/pom.xml | 7 -
.../dev-support/findbugsExcludeFile.xml | 4 +
hadoop-common-project/hadoop-common/pom.xml | 3 +-
.../hadoop-common/src/main/bin/hadoop | 28 +-
.../src/main/bin/hadoop-functions.sh | 184 +-
.../org/apache/hadoop/conf/Configuration.java | 51 +-
.../key/kms/LoadBalancingKMSClientProvider.java | 4 +-
.../hadoop/fs/CommonConfigurationKeys.java | 21 +
.../fs/CommonConfigurationKeysPublic.java | 2 +
.../src/main/java/org/apache/hadoop/fs/DF.java | 9 +-
.../hadoop/fs/FSDataOutputStreamBuilder.java | 203 +-
.../apache/hadoop/fs/FileEncryptionInfo.java | 6 +-
.../java/org/apache/hadoop/fs/FileStatus.java | 189 +-
.../java/org/apache/hadoop/fs/FileSystem.java | 24 +-
.../java/org/apache/hadoop/fs/FileUtil.java | 4 +-
.../org/apache/hadoop/fs/FsUrlConnection.java | 10 +
.../hadoop/fs/FsUrlStreamHandlerFactory.java | 26 +-
.../org/apache/hadoop/fs/LocatedFileStatus.java | 64 +-
.../org/apache/hadoop/fs/ftp/FTPFileSystem.java | 4 +-
.../hadoop/fs/permission/FsPermission.java | 14 +-
.../apache/hadoop/fs/protocolPB/PBHelper.java | 131 +
.../hadoop/fs/protocolPB/package-info.java | 18 +
.../apache/hadoop/fs/sftp/SFTPFileSystem.java | 6 +
.../org/apache/hadoop/fs/shell/AclCommands.java | 6 +-
.../hadoop/fs/shell/CommandWithDestination.java | 4 +-
.../java/org/apache/hadoop/fs/shell/Ls.java | 4 +-
.../java/org/apache/hadoop/fs/shell/Stat.java | 19 +-
.../hadoop/fs/viewfs/ViewFsFileStatus.java | 8 +-
.../fs/viewfs/ViewFsLocatedFileStatus.java | 6 -
.../org/apache/hadoop/http/HttpServer2.java | 9 +-
.../main/java/org/apache/hadoop/io/IOUtils.java | 2 +-
.../java/org/apache/hadoop/io/SequenceFile.java | 2 +-
.../hadoop/io/erasurecode/CodecRegistry.java | 2 +-
.../apache/hadoop/io/erasurecode/ECSchema.java | 6 +-
.../io/erasurecode/ErasureCodeConstants.java | 8 +
.../hadoop/io/retry/RetryInvocationHandler.java | 11 +
.../hadoop/security/CompositeGroupsMapping.java | 4 +-
.../org/apache/hadoop/util/GenericsUtil.java | 15 +
.../hadoop/util/curator/ZKCuratorManager.java | 340 +
.../hadoop/util/curator/package-info.java | 27 +
.../hadoop-common/src/main/proto/FSProtos.proto | 69 +
.../src/main/resources/core-default.xml | 95 +-
.../src/site/markdown/FileSystemShell.md | 4 +-
.../hadoop-common/src/site/markdown/Metrics.md | 20 +-
.../src/site/markdown/SecureMode.md | 9 +-
.../src/site/markdown/ServiceLevelAuth.md | 32 +-
.../src/site/markdown/UnixShellGuide.md | 4 +-
.../src/site/markdown/filesystem/filesystem.md | 57 +-
.../filesystem/fsdataoutputstreambuilder.md | 182 +
.../src/site/markdown/filesystem/index.md | 1 +
.../conf/TestCommonConfigurationFields.java | 6 +
.../apache/hadoop/conf/TestConfiguration.java | 229 +-
.../org/apache/hadoop/fs/TestFileStatus.java | 1 +
.../apache/hadoop/fs/TestLocalFileSystem.java | 78 +-
.../fs/contract/AbstractContractAppendTest.java | 33 +-
.../fs/contract/AbstractContractCreateTest.java | 90 +-
.../hadoop/fs/contract/ContractTestUtils.java | 88 +-
.../fs/protocolPB/TestFSSerialization.java | 85 +
.../org/apache/hadoop/http/TestHttpServer.java | 13 +
.../apache/hadoop/util/TestGenericsUtil.java | 5 +
.../util/curator/TestZKCuratorManager.java | 95 +
.../src/test/resources/testConf.xml | 10 +-
.../scripts/hadoop-functions_test_helper.bash | 2 +-
.../test/scripts/hadoop_add_array_param.bats | 37 +
.../src/test/scripts/hadoop_array_contains.bats | 47 +
.../src/test/scripts/hadoop_escape_chars.bats | 32 -
.../src/test/scripts/hadoop_sort_array.bats | 37 +
.../main/libexec/shellprofile.d/hadoop-kms.sh | 4 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 31 +-
.../hadoop/hdfs/DFSClientFaultInjector.java | 2 +
.../org/apache/hadoop/hdfs/DFSInputStream.java | 156 +-
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 23 +-
.../hadoop/hdfs/DistributedFileSystem.java | 14 +-
.../hadoop/hdfs/protocol/ClientProtocol.java | 10 +-
.../hdfs/protocol/ErasureCodingPolicy.java | 13 +-
.../hdfs/protocol/FsPermissionExtension.java | 5 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 4 +-
.../hadoop/hdfs/protocol/HdfsFileStatus.java | 228 +-
.../hdfs/protocol/HdfsLocatedFileStatus.java | 42 +-
.../protocol/SnapshottableDirectoryStatus.java | 10 +-
.../protocol/SystemErasureCodingPolicies.java | 14 +
.../ClientNamenodeProtocolTranslatorPB.java | 9 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 67 +-
.../apache/hadoop/hdfs/web/JsonUtilClient.java | 50 +-
.../hadoop/hdfs/web/WebHdfsConstants.java | 8 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 16 +-
.../hadoop-hdfs-client/src/main/proto/acl.proto | 7 +-
.../src/main/proto/erasurecoding.proto | 2 +-
.../src/main/proto/hdfs.proto | 16 +-
.../hadoop/fs/http/client/HttpFSFileSystem.java | 92 +-
.../hadoop/fs/http/server/FSOperations.java | 107 +-
.../http/server/HttpFSParametersProvider.java | 45 +
.../hadoop/fs/http/server/HttpFSServer.java | 36 +
.../libexec/shellprofile.d/hadoop-httpfs.sh | 2 +-
.../fs/http/client/BaseTestHttpFSWith.java | 130 +-
.../hadoop/fs/http/server/TestHttpFSServer.java | 140 +-
.../apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java | 6 +-
.../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 8 +-
.../dev-support/findbugsExcludeFile.xml | 33 +
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 +-
.../hadoop-hdfs/src/main/bin/hdfs | 82 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 +-
.../hadoop/hdfs/protocol/SnapshotException.java | 4 +
.../hadoop/hdfs/protocol/SnapshotInfo.java | 2 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 7 +-
.../hadoop/hdfs/protocolPB/package-info.java | 18 +
.../hadoop/hdfs/qjournal/server/Journal.java | 3 +-
.../hdfs/qjournal/server/JournalMetrics.java | 11 +
.../hdfs/qjournal/server/JournalNodeSyncer.java | 4 +
.../server/blockmanagement/BlockManager.java | 49 +-
.../blockmanagement/DatanodeAdminManager.java | 756 +
.../blockmanagement/DatanodeDescriptor.java | 6 +-
.../server/blockmanagement/DatanodeManager.java | 97 +-
.../blockmanagement/DecommissionManager.java | 741 -
.../hadoop/hdfs/server/datanode/DataNode.java | 21 +
.../hdfs/server/datanode/DataXceiver.java | 9 +-
.../erasurecode/ErasureCodingWorker.java | 15 +-
.../erasurecode/StripedBlockReconstructor.java | 3 +-
.../datanode/erasurecode/StripedReader.java | 20 +
.../erasurecode/StripedReconstructionInfo.java | 15 +
.../erasurecode/StripedReconstructor.java | 8 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 48 +-
.../apache/hadoop/hdfs/server/mover/Mover.java | 2 +-
.../hadoop/hdfs/server/namenode/BackupNode.java | 2 +-
.../namenode/ErasureCodingPolicyManager.java | 25 +-
.../server/namenode/FSDirErasureCodingOp.java | 17 +-
.../server/namenode/FSDirStatAndListingOp.java | 58 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 2 +-
.../hdfs/server/namenode/FSDirectory.java | 20 +-
.../hdfs/server/namenode/FSNamesystem.java | 176 +-
.../hdfs/server/namenode/LeaseManager.java | 41 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 24 +-
.../hdfs/server/namenode/NamenodeFsck.java | 4 +-
.../namenode/metrics/ECBlockGroupsMBean.java | 59 +
.../metrics/ECBlockGroupsStatsMBean.java | 59 -
.../namenode/metrics/ReplicatedBlocksMBean.java | 63 +
.../metrics/ReplicatedBlocksStatsMBean.java | 63 -
.../snapshot/DirectorySnapshottableFeature.java | 16 +-
.../namenode/snapshot/SnapshotManager.java | 7 +-
.../org/apache/hadoop/hdfs/tools/ECAdmin.java | 41 +-
.../offlineEditsViewer/XmlEditsVisitor.java | 41 +-
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 28 +-
.../src/main/resources/hdfs-default.xml | 40 +-
.../src/main/webapps/hdfs/explorer.js | 6 +
.../src/site/markdown/HDFSErasureCoding.md | 43 +-
.../src/site/markdown/HdfsPermissionsGuide.md | 2 +-
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 39 +
.../org/apache/hadoop/TestGenericRefresh.java | 28 +-
.../java/org/apache/hadoop/cli/TestAclCLI.java | 2 +
.../apache/hadoop/fs/TestUrlStreamHandler.java | 48 +-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 8 +-
.../hadoop/hdfs/TestDFSClientRetries.java | 6 +-
.../hadoop/hdfs/TestDFSClientSocketSize.java | 20 +-
.../org/apache/hadoop/hdfs/TestDFSShell.java | 36 +-
.../hadoop/hdfs/TestDFSUpgradeFromImage.java | 3 +-
.../apache/hadoop/hdfs/TestDecommission.java | 44 +-
.../hadoop/hdfs/TestDistributedFileSystem.java | 50 +
.../apache/hadoop/hdfs/TestEncryptionZones.java | 3 +-
.../hadoop/hdfs/TestErasureCodingPolicies.java | 129 +-
.../hdfs/TestFileStatusSerialization.java | 153 +
.../java/org/apache/hadoop/hdfs/TestLease.java | 7 +-
.../java/org/apache/hadoop/hdfs/TestPread.java | 89 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 74 +-
.../org/apache/hadoop/hdfs/TestSafeMode.java | 25 +
.../hdfs/qjournal/TestJournalNodeSync.java | 265 -
.../hdfs/qjournal/server/TestJournalNode.java | 6 +-
.../qjournal/server/TestJournalNodeSync.java | 439 +
.../blockmanagement/BlockManagerTestUtil.java | 2 +-
.../blockmanagement/TestBlockManager.java | 54 +
.../blockmanagement/TestDatanodeManager.java | 96 +-
...constructStripedBlocksWithRackAwareness.java | 5 +-
.../TestReplicationPolicyConsiderLoad.java | 2 +-
.../TestDataXceiverBackwardsCompat.java | 212 +
.../hdfs/server/mover/TestStorageMover.java | 2 +-
.../hdfs/server/namenode/AclTestHelpers.java | 13 +-
.../hdfs/server/namenode/FSAclBaseTest.java | 14 +-
.../server/namenode/TestAddStripedBlocks.java | 4 +-
.../namenode/TestDecommissioningStatus.java | 6 +-
.../TestDefaultBlockPlacementPolicy.java | 4 +-
.../server/namenode/TestEnabledECPolicies.java | 10 +-
.../hdfs/server/namenode/TestFSImage.java | 87 +
.../server/namenode/TestFSImageWithAcl.java | 14 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 17 +-
.../hdfs/server/namenode/TestMetaSave.java | 2 +
.../server/namenode/TestNameNodeMXBean.java | 14 +-
.../namenode/TestNameNodeOptionParsing.java | 27 +-
.../namenode/TestNameNodeReconfigure.java | 36 +
.../namenode/TestNamenodeCapacityReport.java | 8 +-
.../namenode/TestReconstructStripedBlocks.java | 4 +-
.../hdfs/server/namenode/TestStartup.java | 4 +-
.../server/namenode/ha/TestEditLogTailer.java | 4 +-
.../namenode/ha/TestFailureToReadEdits.java | 6 +-
.../namenode/ha/TestInitializeSharedEdits.java | 2 +-
.../namenode/metrics/TestNameNodeMetrics.java | 134 +-
.../snapshot/TestOpenFilesWithSnapshot.java | 347 +
.../apache/hadoop/hdfs/web/TestJsonUtil.java | 11 +-
.../hadoop-hdfs/src/test/resources/editsStored | Bin 5850 -> 5850 bytes
.../src/test/resources/editsStored.xml | 62 +-
.../test/resources/testErasureCodingConf.xml | 113 +-
hadoop-mapreduce-project/bin/mapred | 18 +-
.../jobhistory/JobHistoryEventHandler.java | 27 +-
.../hadoop/mapreduce/v2/app/AppContext.java | 4 +
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 14 +-
.../mapreduce/v2/app/job/impl/JobImpl.java | 35 +-
.../mapreduce/v2/app/rm/RMCommunicator.java | 4 +-
.../mapreduce/v2/app/webapp/AppController.java | 10 +-
.../hadoop/mapreduce/v2/app/webapp/AppView.java | 4 +-
.../mapreduce/v2/app/webapp/ConfBlock.java | 36 +-
.../mapreduce/v2/app/webapp/CountersBlock.java | 38 +-
.../mapreduce/v2/app/webapp/CountersPage.java | 4 +-
.../mapreduce/v2/app/webapp/InfoPage.java | 2 +-
.../mapreduce/v2/app/webapp/JobBlock.java | 95 +-
.../mapreduce/v2/app/webapp/JobConfPage.java | 3 +-
.../hadoop/mapreduce/v2/app/webapp/JobPage.java | 2 +-
.../mapreduce/v2/app/webapp/JobsBlock.java | 24 +-
.../mapreduce/v2/app/webapp/NavBlock.java | 38 +-
.../v2/app/webapp/SingleCounterBlock.java | 28 +-
.../v2/app/webapp/SingleCounterPage.java | 3 +-
.../mapreduce/v2/app/webapp/TaskPage.java | 20 +-
.../mapreduce/v2/app/webapp/TasksBlock.java | 12 +-
.../mapreduce/v2/app/webapp/TasksPage.java | 2 +-
.../jobhistory/TestJobHistoryEventHandler.java | 102 +
.../hadoop/mapreduce/v2/app/MockAppContext.java | 10 +
.../mapreduce/v2/app/TestJobEndNotifier.java | 16 +-
.../mapreduce/v2/app/TestRuntimeEstimators.java | 10 +
.../mapreduce/v2/app/job/impl/TestJobImpl.java | 139 +-
.../v2/app/launcher/TestContainerLauncher.java | 9 +
.../app/launcher/TestContainerLauncherImpl.java | 9 +
.../v2/app/rm/TestRMContainerAllocator.java | 3 +-
.../hadoop-mapreduce-client-common/pom.xml | 4 -
.../org/apache/hadoop/mapreduce/Cluster.java | 16 +-
.../apache/hadoop/mapreduce/JobSubmitter.java | 2 -
.../apache/hadoop/mapreduce/MRJobConfig.java | 6 +-
.../mapreduce/counters/AbstractCounters.java | 4 -
.../hadoop/mapreduce/counters/Limits.java | 12 -
.../ClientDistributedCacheManager.java | 6 +-
.../mapreduce/jobhistory/HistoryViewer.java | 16 -
.../TaskAttemptUnsuccessfulCompletionEvent.java | 28 +-
.../src/main/resources/mapred-default.xml | 8 +
.../org/apache/hadoop/mapred/TestQueue.java | 8 +-
.../hadoop/mapreduce/v2/hs/CompletedJob.java | 15 -
.../hadoop/mapreduce/v2/hs/JobHistory.java | 10 +
.../mapreduce/v2/hs/webapp/HsAboutPage.java | 7 +-
.../mapreduce/v2/hs/webapp/HsConfPage.java | 2 +-
.../mapreduce/v2/hs/webapp/HsCountersPage.java | 4 +-
.../mapreduce/v2/hs/webapp/HsJobBlock.java | 98 +-
.../mapreduce/v2/hs/webapp/HsJobPage.java | 2 +-
.../mapreduce/v2/hs/webapp/HsJobsBlock.java | 46 +-
.../mapreduce/v2/hs/webapp/HsLogsPage.java | 2 +-
.../mapreduce/v2/hs/webapp/HsNavBlock.java | 30 +-
.../v2/hs/webapp/HsSingleCounterPage.java | 2 +-
.../mapreduce/v2/hs/webapp/HsTaskPage.java | 53 +-
.../mapreduce/v2/hs/webapp/HsTasksBlock.java | 57 +-
.../mapreduce/v2/hs/webapp/HsTasksPage.java | 2 +-
.../hadoop/mapreduce/v2/hs/webapp/HsView.java | 4 +-
.../java/org/apache/hadoop/io/FileBench.java | 2 +-
.../mapred/MiniMRClientClusterFactory.java | 4 +-
.../mapred/TestCombineFileInputFormat.java | 6 +-
.../TestCombineSequenceFileInputFormat.java | 7 +-
.../mapred/TestCombineTextInputFormat.java | 7 +-
.../mapred/TestConcatenatedCompressedInput.java | 6 +-
.../org/apache/hadoop/mapred/TestMapRed.java | 4 +-
.../hadoop/mapred/TestMiniMRChildTask.java | 4 +-
.../hadoop/mapred/TestTextInputFormat.java | 8 +-
.../apache/hadoop/mapred/TestYARNRunner.java | 8 +-
.../TestWrappedRecordReaderClassloader.java | 4 +-
.../apache/hadoop/mapreduce/TestCounters.java | 39 +-
.../lib/input/TestCombineFileInputFormat.java | 2 +-
.../mapreduce/lib/input/TestMultipleInputs.java | 10 +-
.../lib/join/TestWrappedRRClassloader.java | 4 +-
.../TestUmbilicalProtocolWithJobToken.java | 22 +-
.../mapreduce/util/MRAsyncDiskService.java | 2 +-
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 91 +-
.../v2/TestMRJobsWithHistoryService.java | 4 +-
.../hadoop/mapred/FadvisedFileRegion.java | 5 +-
.../hadoop-mapreduce-client/pom.xml | 10 +-
hadoop-maven-plugins/pom.xml | 5 +-
hadoop-project-dist/pom.xml | 12 +-
hadoop-project/pom.xml | 49 +-
hadoop-project/src/site/site.xml | 1 +
.../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 4 +
.../main/shellprofile.d/hadoop-archive-logs.sh | 2 +-
.../org/apache/hadoop/tools/HadoopArchives.java | 2 +-
.../src/main/shellprofile.d/hadoop-archives.sh | 4 +-
hadoop-tools/hadoop-aws/pom.xml | 22 +-
hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +-
.../org/apache/hadoop/fs/adl/AdlConfKeys.java | 8 +
.../org/apache/hadoop/fs/adl/AdlFileStatus.java | 69 +
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 50 +-
.../apache/hadoop/fs/adl/TokenProviderType.java | 2 +
.../src/site/markdown/index.md | 98 +-
.../hadoop/fs/adl/TestAzureADTokenProvider.java | 40 +
.../apache/hadoop/fs/adl/TestGetFileStatus.java | 58 +-
.../apache/hadoop/fs/adl/TestListStatus.java | 8 +-
.../fs/azure/AzureNativeFileSystemStore.java | 123 +-
.../fs/azure/BlobOperationDescriptor.java | 222 +
.../hadoop/fs/azure/BlockBlobInputStream.java | 91 +-
.../fs/azure/ClientThrottlingAnalyzer.java | 284 +
.../fs/azure/ClientThrottlingIntercept.java | 221 +
.../hadoop/fs/azure/NativeAzureFileSystem.java | 47 +-
.../services/org.apache.hadoop.fs.FileSystem | 17 -
.../hadoop/fs/azure/AbstractWasbTestBase.java | 4 +
.../fs/azure/TestBlobOperationDescriptor.java | 305 +
.../fs/azure/TestBlockBlobInputStream.java | 85 +-
.../fs/azure/TestClientThrottlingAnalyzer.java | 177 +
.../TestFileSystemOperationsWithThreads.java | 61 +-
.../azure/TestOutOfBandAzureBlobOperations.java | 8 +-
.../fs/azure/TestWasbRemoteCallHelper.java | 7 +-
.../src/test/resources/azure-test.xml | 11 +-
.../hadoop/tools/CopyListingFileStatus.java | 4 +-
.../apache/hadoop/tools/util/DistCpUtils.java | 4 +-
.../src/main/shellprofile.d/hadoop-distcp.sh | 4 +-
.../src/main/shellprofile.d/hadoop-extras.sh | 2 +-
.../apache/hadoop/mapred/gridmix/Gridmix.java | 2 +-
.../hadoop/mapred/gridmix/PseudoLocalFs.java | 8 +-
.../src/main/shellprofile.d/hadoop-gridmix.sh | 2 +-
.../hadoop/mapred/gridmix/TestFilePool.java | 4 +-
.../hadoop/mapred/gridmix/TestFileQueue.java | 8 +-
.../mapred/gridmix/TestPseudoLocalFs.java | 2 +-
.../hadoop/mapred/gridmix/TestUserResolve.java | 4 +-
.../fs/swift/snative/SwiftFileStatus.java | 16 +-
.../snative/SwiftNativeFileSystemStore.java | 4 +-
.../hadoop/fs/swift/util/SwiftTestUtils.java | 2 +-
.../fs/swift/SwiftFileSystemBaseTest.java | 2 +-
.../swift/TestSwiftFileSystemDirectories.java | 4 +-
.../TestSwiftFileSystemPartitionedUploads.java | 6 +-
.../src/main/shellprofile.d/hadoop-rumen.sh | 4 +-
.../hadoop/tools/rumen/TestHistograms.java | 6 +-
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 2 +-
.../yarn/sls/scheduler/RMNodeWrapper.java | 2 +-
.../org/apache/hadoop/streaming/StreamJob.java | 27 +-
.../src/main/shellprofile.d/hadoop-streaming.sh | 2 +-
.../MySQL/FederationStateStoreDatabase.sql | 21 +
.../MySQL/FederationStateStoreStoredProcs.sql | 162 +
.../MySQL/FederationStateStoreTables.sql | 47 +
.../MySQL/FederationStateStoreUser.sql | 25 +
.../FederationStateStore/MySQL/dropDatabase.sql | 21 +
.../MySQL/dropStoreProcedures.sql | 47 +
.../FederationStateStore/MySQL/dropTables.sql | 27 +
.../bin/FederationStateStore/MySQL/dropUser.sql | 21 +
.../FederationStateStoreStoreProcs.sql | 511 +
.../SQLServer/FederationStateStoreTables.sql | 122 +
hadoop-yarn-project/hadoop-yarn/bin/yarn | 47 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 13 +-
.../hadoop-yarn/conf/yarn-env.sh | 12 +
.../dev-support/findbugs-exclude.xml | 22 +-
.../yarn/api/ContainerManagementProtocol.java | 23 +
.../protocolrecords/ContainerUpdateRequest.java | 77 +
.../ContainerUpdateResponse.java | 95 +
.../org/apache/hadoop/yarn/conf/HAUtil.java | 30 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 190 +-
.../proto/containermanagement_protocol.proto | 1 +
.../src/main/proto/yarn_service_protos.proto | 9 +
.../yarn/conf/TestYarnConfigurationFields.java | 60 +
.../yarn/client/api/impl/NMClientImpl.java | 18 +-
.../TestFederationRMFailoverProxyProvider.java | 223 +
.../yarn/client/api/impl/TestAMRMClient.java | 401 +-
.../yarn/client/api/impl/TestAMRMProxy.java | 10 +-
.../yarn/client/api/impl/TestNMClient.java | 44 +-
.../yarn/client/api/impl/TestYarnClient.java | 9 +-
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 22 +
...ContainerManagementProtocolPBClientImpl.java | 40 +-
...ontainerManagementProtocolPBServiceImpl.java | 28 +-
.../impl/pb/ContainerUpdateRequestPBImpl.java | 171 +
.../impl/pb/ContainerUpdateResponsePBImpl.java | 241 +
.../hadoop/yarn/client/ClientRMProxy.java | 4 +-
.../org/apache/hadoop/yarn/client/RMProxy.java | 11 +-
.../api/impl/FileSystemTimelineWriter.java | 40 +-
.../apache/hadoop/yarn/util/AsyncCallback.java | 35 +
.../hadoop/yarn/util/LRUCacheHashMap.java | 49 +
.../apache/hadoop/yarn/webapp/ResponseInfo.java | 4 +-
.../hadoop/yarn/webapp/example/HelloWorld.java | 4 +-
.../hadoop/yarn/webapp/example/MyApp.java | 6 +-
.../hadoop/yarn/webapp/hamlet/Hamlet.java | 4 +
.../hadoop/yarn/webapp/hamlet/HamletGen.java | 2 +
.../hadoop/yarn/webapp/hamlet/HamletImpl.java | 2 +
.../hadoop/yarn/webapp/hamlet/HamletSpec.java | 2 +
.../hadoop/yarn/webapp/hamlet/package-info.java | 6 +
.../hadoop/yarn/webapp/hamlet2/Hamlet.java | 30557 +++++++++++++++++
.../hadoop/yarn/webapp/hamlet2/HamletGen.java | 449 +
.../hadoop/yarn/webapp/hamlet2/HamletImpl.java | 385 +
.../hadoop/yarn/webapp/hamlet2/HamletSpec.java | 3101 ++
.../yarn/webapp/hamlet2/package-info.java | 27 +
.../yarn/webapp/log/AggregatedLogsBlock.java | 66 +-
.../yarn/webapp/log/AggregatedLogsNavBlock.java | 4 +-
.../yarn/webapp/log/AggregatedLogsPage.java | 2 +-
.../hadoop/yarn/webapp/util/WebAppUtils.java | 14 +
.../hadoop/yarn/webapp/view/ErrorPage.java | 12 +-
.../hadoop/yarn/webapp/view/FooterBlock.java | 2 +-
.../hadoop/yarn/webapp/view/HeaderBlock.java | 6 +-
.../hadoop/yarn/webapp/view/HtmlBlock.java | 2 +-
.../hadoop/yarn/webapp/view/HtmlPage.java | 12 +-
.../hadoop/yarn/webapp/view/InfoBlock.java | 28 +-
.../hadoop/yarn/webapp/view/JQueryUI.java | 14 +-
.../hadoop/yarn/webapp/view/LipsumBlock.java | 4 +-
.../hadoop/yarn/webapp/view/NavBlock.java | 10 +-
.../yarn/webapp/view/TwoColumnCssLayout.java | 20 +-
.../yarn/webapp/view/TwoColumnLayout.java | 20 +-
.../src/main/resources/yarn-default.xml | 224 +-
.../hadoop/yarn/TestContainerLaunchRPC.java | 9 +
.../yarn/TestContainerResourceIncreaseRPC.java | 20 +-
.../hadoop/yarn/api/TestPBImplRecords.java | 4 +
.../api/impl/TestTimelineClientForATS1_5.java | 81 +-
.../hadoop/yarn/conf/TestYarnConfiguration.java | 1 -
.../hadoop/yarn/util/TestLRUCacheHashMap.java | 74 +
.../yarn/util/TestProcfsBasedProcessTree.java | 2 +-
.../hadoop/yarn/util/TestYarnVersionInfo.java | 11 +-
.../apache/hadoop/yarn/webapp/TestSubViews.java | 10 +-
.../apache/hadoop/yarn/webapp/TestWebApp.java | 28 +-
.../hadoop/yarn/webapp/view/TestHtmlBlock.java | 10 +-
.../hadoop/yarn/webapp/view/TestHtmlPage.java | 9 +-
.../hadoop/yarn/webapp/view/TestInfoBlock.java | 6 +-
.../yarn/webapp/view/TestTwoColumnCssPage.java | 6 +-
.../pom.xml | 4 -
.../ApplicationHistoryClientService.java | 8 +-
.../ApplicationHistoryManagerImpl.java | 8 +-
...pplicationHistoryManagerOnTimelineStore.java | 8 +-
.../ApplicationHistoryServer.java | 10 +-
.../FileSystemApplicationHistoryStore.java | 22 +-
.../webapp/AHSErrorsAndWarningsPage.java | 2 +-
.../webapp/AHSLogsPage.java | 2 +-
.../webapp/AHSView.java | 4 +-
.../webapp/AHSWebServices.java | 7 +-
.../webapp/AboutBlock.java | 8 +-
.../webapp/AboutPage.java | 5 +-
.../webapp/AppAttemptPage.java | 2 +-
.../webapp/AppPage.java | 2 +-
.../webapp/ContainerPage.java | 2 +-
.../webapp/NavBlock.java | 36 +-
.../timeline/KeyValueBasedTimelineStore.java | 8 +-
.../server/timeline/LeveldbTimelineStore.java | 35 +-
.../yarn/server/timeline/RollingLevelDB.java | 15 +-
.../timeline/RollingLevelDBTimelineStore.java | 22 +-
.../server/timeline/TimelineDataManager.java | 7 +-
.../recovery/LeveldbTimelineStateStore.java | 30 +-
.../timeline/security/TimelineACLsManager.java | 7 +-
...lineDelegationTokenSecretManagerService.java | 8 +-
.../timeline/webapp/TimelineWebServices.java | 7 +-
.../TestFileSystemApplicationHistoryStore.java | 8 +-
.../timeline/TestLeveldbTimelineStore.java | 2 +-
.../hadoop-yarn-server-common/pom.xml | 62 +
.../hadoop/yarn/server/api/ServerRMProxy.java | 4 +-
.../protocolrecords/NodeHeartbeatResponse.java | 6 +-
.../impl/pb/NodeHeartbeatResponsePBImpl.java | 42 +-
.../failover/FederationProxyProviderUtil.java | 133 +
.../FederationRMFailoverProxyProvider.java | 221 +
.../federation/failover/package-info.java | 17 +
.../AbstractConfigurableFederationPolicy.java | 155 +
.../policies/ConfigurableFederationPolicy.java | 41 +
.../FederationPolicyInitializationContext.java | 130 +
...ionPolicyInitializationContextValidator.java | 76 +
.../policies/FederationPolicyUtils.java | 203 +
.../federation/policies/RouterPolicyFacade.java | 265 +
.../amrmproxy/AbstractAMRMProxyPolicy.java | 47 +
.../amrmproxy/BroadcastAMRMProxyPolicy.java | 85 +
.../amrmproxy/FederationAMRMProxyPolicy.java | 65 +
.../LocalityMulticastAMRMProxyPolicy.java | 608 +
.../amrmproxy/RejectAMRMProxyPolicy.java | 67 +
.../policies/amrmproxy/package-info.java | 19 +
.../policies/dao/WeightedPolicyInfo.java | 251 +
.../federation/policies/dao/package-info.java | 19 +
.../exceptions/FederationPolicyException.java | 33 +
...FederationPolicyInitializationException.java | 33 +
.../NoActiveSubclustersException.java | 27 +
.../exceptions/UnknownSubclusterException.java | 28 +
.../policies/exceptions/package-info.java | 19 +
.../policies/manager/AbstractPolicyManager.java | 190 +
.../manager/FederationPolicyManager.java | 118 +
.../manager/HashBroadcastPolicyManager.java | 38 +
.../manager/PriorityBroadcastPolicyManager.java | 66 +
.../manager/RejectAllPolicyManager.java | 40 +
.../manager/UniformBroadcastPolicyManager.java | 44 +
.../manager/WeightedLocalityPolicyManager.java | 67 +
.../policies/manager/package-info.java | 19 +
.../federation/policies/package-info.java | 19 +
.../policies/router/AbstractRouterPolicy.java | 66 +
.../policies/router/FederationRouterPolicy.java | 52 +
.../policies/router/HashBasedRouterPolicy.java | 102 +
.../policies/router/LoadBasedRouterPolicy.java | 112 +
.../policies/router/PriorityRouterPolicy.java | 72 +
.../policies/router/RejectRouterPolicy.java | 76 +
.../router/UniformRandomRouterPolicy.java | 104 +
.../router/WeightedRandomRouterPolicy.java | 94 +
.../policies/router/package-info.java | 19 +
.../resolver/AbstractSubClusterResolver.java | 67 +
.../resolver/DefaultSubClusterResolverImpl.java | 164 +
.../federation/resolver/SubClusterResolver.java | 58 +
.../federation/resolver/package-info.java | 17 +
...ederationApplicationHomeSubClusterStore.java | 121 +
.../store/FederationMembershipStateStore.java | 117 +
.../federation/store/FederationPolicyStore.java | 77 +
.../federation/store/FederationStateStore.java | 64 +
.../FederationStateStoreException.java | 50 +
...derationStateStoreInvalidInputException.java | 48 +
.../FederationStateStoreRetriableException.java | 44 +
.../store/exception/package-info.java | 17 +
.../store/impl/MemoryFederationStateStore.java | 315 +
.../store/impl/SQLFederationStateStore.java | 942 +
.../impl/ZookeeperFederationStateStore.java | 634 +
.../federation/store/impl/package-info.java | 17 +
.../server/federation/store/package-info.java | 17 +
.../AddApplicationHomeSubClusterRequest.java | 72 +
.../AddApplicationHomeSubClusterResponse.java | 65 +
.../records/ApplicationHomeSubCluster.java | 124 +
.../DeleteApplicationHomeSubClusterRequest.java | 65 +
...DeleteApplicationHomeSubClusterResponse.java | 43 +
.../GetApplicationHomeSubClusterRequest.java | 64 +
.../GetApplicationHomeSubClusterResponse.java | 73 +
.../GetApplicationsHomeSubClusterRequest.java | 40 +
.../GetApplicationsHomeSubClusterResponse.java | 75 +
.../store/records/GetSubClusterInfoRequest.java | 62 +
.../records/GetSubClusterInfoResponse.java | 62 +
...SubClusterPoliciesConfigurationsRequest.java | 35 +
...ubClusterPoliciesConfigurationsResponse.java | 66 +
...GetSubClusterPolicyConfigurationRequest.java | 63 +
...etSubClusterPolicyConfigurationResponse.java | 65 +
.../records/GetSubClustersInfoRequest.java | 70 +
.../records/GetSubClustersInfoResponse.java | 66 +
...SetSubClusterPolicyConfigurationRequest.java | 61 +
...etSubClusterPolicyConfigurationResponse.java | 36 +
.../records/SubClusterDeregisterRequest.java | 89 +
.../records/SubClusterDeregisterResponse.java | 42 +
.../records/SubClusterHeartbeatRequest.java | 149 +
.../records/SubClusterHeartbeatResponse.java | 45 +
.../federation/store/records/SubClusterId.java | 100 +
.../store/records/SubClusterIdInfo.java | 75 +
.../store/records/SubClusterInfo.java | 325 +
.../records/SubClusterPolicyConfiguration.java | 162 +
.../records/SubClusterRegisterRequest.java | 74 +
.../records/SubClusterRegisterResponse.java | 44 +
.../store/records/SubClusterState.java | 85 +
.../UpdateApplicationHomeSubClusterRequest.java | 74 +
...UpdateApplicationHomeSubClusterResponse.java | 43 +
...dApplicationHomeSubClusterRequestPBImpl.java | 132 +
...ApplicationHomeSubClusterResponsePBImpl.java | 117 +
.../pb/ApplicationHomeSubClusterPBImpl.java | 167 +
...eApplicationHomeSubClusterRequestPBImpl.java | 130 +
...ApplicationHomeSubClusterResponsePBImpl.java | 78 +
...tApplicationHomeSubClusterRequestPBImpl.java | 139 +
...ApplicationHomeSubClusterResponsePBImpl.java | 132 +
...ApplicationsHomeSubClusterRequestPBImpl.java | 78 +
...pplicationsHomeSubClusterResponsePBImpl.java | 190 +
.../impl/pb/GetSubClusterInfoRequestPBImpl.java | 125 +
.../pb/GetSubClusterInfoResponsePBImpl.java | 134 +
...sterPoliciesConfigurationsRequestPBImpl.java | 95 +
...terPoliciesConfigurationsResponsePBImpl.java | 191 +
...ClusterPolicyConfigurationRequestPBImpl.java | 103 +
...lusterPolicyConfigurationResponsePBImpl.java | 143 +
.../pb/GetSubClustersInfoRequestPBImpl.java | 108 +
.../pb/GetSubClustersInfoResponsePBImpl.java | 184 +
...ClusterPolicyConfigurationRequestPBImpl.java | 142 +
...lusterPolicyConfigurationResponsePBImpl.java | 93 +
.../pb/SubClusterDeregisterRequestPBImpl.java | 156 +
.../pb/SubClusterDeregisterResponsePBImpl.java | 77 +
.../pb/SubClusterHeartbeatRequestPBImpl.java | 192 +
.../pb/SubClusterHeartbeatResponsePBImpl.java | 77 +
.../records/impl/pb/SubClusterIdPBImpl.java | 75 +
.../records/impl/pb/SubClusterInfoPBImpl.java | 251 +
.../pb/SubClusterPolicyConfigurationPBImpl.java | 138 +
.../pb/SubClusterRegisterRequestPBImpl.java | 134 +
.../pb/SubClusterRegisterResponsePBImpl.java | 77 +
...eApplicationHomeSubClusterRequestPBImpl.java | 132 +
...ApplicationHomeSubClusterResponsePBImpl.java | 78 +
.../store/records/impl/pb/package-info.java | 17 +
.../federation/store/records/package-info.java | 17 +
...cationHomeSubClusterStoreInputValidator.java | 180 +
...ationMembershipStateStoreInputValidator.java | 315 +
.../FederationPolicyStoreInputValidator.java | 143 +
.../store/utils/FederationStateStoreUtils.java | 214 +
.../federation/store/utils/package-info.java | 17 +
.../utils/FederationStateStoreFacade.java | 602 +
.../server/federation/utils/package-info.java | 17 +
.../yarn/server/uam/UnmanagedAMPoolManager.java | 311 +
.../server/uam/UnmanagedApplicationManager.java | 607 +
.../hadoop/yarn/server/uam/package-info.java | 18 +
.../yarn/server/utils/AMRMClientUtils.java | 189 +
.../server/utils/YarnServerSecurityUtils.java | 41 +-
.../yarn/server/webapp/AppAttemptBlock.java | 31 +-
.../hadoop/yarn/server/webapp/AppBlock.java | 80 +-
.../hadoop/yarn/server/webapp/AppsBlock.java | 14 +-
.../yarn/server/webapp/ContainerBlock.java | 23 +-
.../server/webapp/ErrorsAndWarningsBlock.java | 50 +-
.../yarn_server_common_service_protos.proto | 3 +
.../proto/yarn_server_federation_protos.proto | 165 +
.../java/org/apache/hadoop/yarn/TestRPC.java | 9 +
.../hadoop/yarn/TestYarnServerApiClasses.java | 6 +-
.../yarn/server/MockResourceManagerFacade.java | 855 +
.../policies/BaseFederationPoliciesTest.java | 186 +
...ionPolicyInitializationContextValidator.java | 138 +
.../policies/TestRouterPolicyFacade.java | 222 +
.../TestBroadcastAMRMProxyFederationPolicy.java | 112 +
.../TestLocalityMulticastAMRMProxyPolicy.java | 602 +
.../amrmproxy/TestRejectAMRMProxyPolicy.java | 78 +
.../policies/manager/BasePolicyManagerTest.java | 101 +
.../TestHashBasedBroadcastPolicyManager.java | 40 +
.../TestPriorityBroadcastPolicyManager.java | 72 +
.../manager/TestRejectAllPolicyManager.java | 40 +
.../TestUniformBroadcastPolicyManager.java | 40 +
.../TestWeightedLocalityPolicyManager.java | 79 +
.../policies/router/BaseRouterPoliciesTest.java | 118 +
.../router/TestHashBasedRouterPolicy.java | 83 +
.../router/TestLoadBasedRouterPolicy.java | 106 +
.../router/TestPriorityRouterPolicy.java | 85 +
.../policies/router/TestRejectRouterPolicy.java | 63 +
.../router/TestUniformRandomRouterPolicy.java | 64 +
.../router/TestWeightedRandomRouterPolicy.java | 137 +
.../resolver/TestDefaultSubClusterResolver.java | 189 +
.../impl/FederationStateStoreBaseTest.java | 578 +
.../store/impl/HSQLDBFederationStateStore.java | 252 +
.../impl/TestMemoryFederationStateStore.java | 35 +
.../store/impl/TestSQLFederationStateStore.java | 49 +
.../impl/TestZookeeperFederationStateStore.java | 89 +
.../records/TestFederationProtocolRecords.java | 265 +
.../TestFederationStateStoreInputValidator.java | 1260 +
.../utils/FederationPoliciesTestUtil.java | 227 +
.../utils/FederationStateStoreTestUtil.java | 181 +
.../utils/TestFederationStateStoreFacade.java | 192 +
.../TestFederationStateStoreFacadeRetry.java | 140 +
.../uam/TestUnmanagedApplicationManager.java | 335 +
.../src/test/resources/nodes | 8 +
.../src/test/resources/nodes-malformed | 3 +
.../hadoop-yarn-server-nodemanager/pom.xml | 45 +
.../src/CMakeLists.txt | 33 +
.../CMgrDecreaseContainersResourceEvent.java | 37 -
.../nodemanager/CMgrUpdateContainersEvent.java | 48 +
.../nodemanager/ContainerManagerEventType.java | 2 +-
.../server/nodemanager/DirectoryCollection.java | 61 +-
.../nodemanager/LinuxContainerExecutor.java | 2 +-
.../nodemanager/LocalDirsHandlerService.java | 59 +-
.../nodemanager/NodeStatusUpdaterImpl.java | 19 +-
.../amrmproxy/DefaultRequestInterceptor.java | 30 +-
.../amrmproxy/FederationInterceptor.java | 1150 +
.../containermanager/ContainerManagerImpl.java | 180 +-
.../containermanager/container/Container.java | 4 +-
.../container/ContainerImpl.java | 41 +-
.../CGroupsCpuResourceHandlerImpl.java | 20 +-
.../linux/resources/CGroupsHandler.java | 15 +
.../linux/resources/CGroupsHandlerImpl.java | 26 +-
.../CGroupsMemoryResourceHandlerImpl.java | 35 +-
.../linux/resources/ResourceHandlerModule.java | 58 +-
.../localizer/ContainerLocalizer.java | 15 +-
.../monitor/ContainerMetrics.java | 2 +-
.../monitor/ContainersMonitorImpl.java | 15 -
...locationBasedResourceUtilizationTracker.java | 31 +-
.../scheduler/ContainerScheduler.java | 83 +-
.../scheduler/ContainerSchedulerEventType.java | 1 +
.../UpdateContainerSchedulerEvent.java | 85 +
.../util/CgroupsLCEResourcesHandler.java | 53 +-
.../nodemanager/webapp/AllApplicationsPage.java | 28 +-
.../nodemanager/webapp/AllContainersPage.java | 30 +-
.../nodemanager/webapp/ApplicationPage.java | 25 +-
.../nodemanager/webapp/ContainerLogsPage.java | 17 +-
.../nodemanager/webapp/ContainerPage.java | 31 +-
.../webapp/NMErrorsAndWarningsPage.java | 2 +-
.../yarn/server/nodemanager/webapp/NMView.java | 4 +-
.../server/nodemanager/webapp/NavBlock.java | 28 +-
.../server/nodemanager/webapp/NodePage.java | 28 +-
.../container-executor/impl/configuration.c | 672 +-
.../container-executor/impl/configuration.h | 182 +-
.../impl/container-executor.c | 204 +-
.../impl/container-executor.h | 63 +-
.../container-executor/impl/get_executable.c | 1 +
.../main/native/container-executor/impl/main.c | 30 +-
.../impl/modules/cgroups/cgroups-operations.c | 161 +
.../impl/modules/cgroups/cgroups-operations.h | 55 +
.../impl/modules/common/constants.h | 29 +
.../impl/modules/common/module-configs.c | 41 +
.../impl/modules/common/module-configs.h | 33 +
.../impl/modules/gpu/gpu-module.c | 229 +
.../impl/modules/gpu/gpu-module.h | 45 +
.../main/native/container-executor/impl/util.c | 134 +
.../main/native/container-executor/impl/util.h | 122 +
.../container-executor/impl/utils/path-utils.c | 52 +
.../container-executor/impl/utils/path-utils.h | 35 +
.../impl/utils/string-utils.c | 159 +
.../impl/utils/string-utils.h | 37 +
.../test-configurations/configuration-1.cfg | 31 +
.../test-configurations/configuration-2.cfg | 28 +
.../test/test-configurations/old-config.cfg | 25 +
.../test/modules/cgroups/test-cgroups-module.cc | 121 +
.../test/modules/gpu/test-gpu-module.cc | 203 +
.../test/test-container-executor-common.h | 36 +
.../test/test-container-executor.c | 246 +-
.../test/test_configuration.cc | 432 +
.../native/container-executor/test/test_main.cc | 32 +
.../native/container-executor/test/test_util.cc | 138 +
.../test/utils/test-path-utils.cc | 67 +
.../test/utils/test-string-utils.cc | 93 +
.../nodemanager/TestDirectoryCollection.java | 23 +
.../nodemanager/TestNodeManagerResync.java | 41 +-
.../amrmproxy/MockRequestInterceptor.java | 14 +-
.../amrmproxy/MockResourceManagerFacade.java | 514 -
.../amrmproxy/TestAMRMProxyService.java | 1 +
.../amrmproxy/TestFederationInterceptor.java | 496 +
.../TestableFederationInterceptor.java | 139 +
.../BaseContainerManagerTest.java | 43 +-
.../containermanager/TestAuxServices.java | 2 +-
.../containermanager/TestContainerManager.java | 308 +-
.../TestContainerManagerRecovery.java | 24 +-
.../TestCGroupsCpuResourceHandlerImpl.java | 23 +
.../linux/resources/TestCGroupsHandlerImpl.java | 25 +
.../TestCGroupsMemoryResourceHandlerImpl.java | 45 +-
...locationBasedResourceUtilizationTracker.java | 93 +
.../TestContainerSchedulerQueuing.java | 96 +
.../util/TestCgroupsLCEResourcesHandler.java | 33 +-
.../nodemanager/webapp/MockContainer.java | 2 +-
...ActiveStandbyElectorBasedElectorService.java | 5 +-
.../ApplicationMasterService.java | 13 +-
.../server/resourcemanager/RMAppManager.java | 56 +-
.../yarn/server/resourcemanager/RMZKUtils.java | 81 -
.../server/resourcemanager/ResourceManager.java | 125 +-
.../resourcemanager/ResourceTrackerService.java | 2 +-
.../FederationStateStoreHeartbeat.java | 108 +
.../federation/FederationStateStoreService.java | 304 +
.../federation/package-info.java | 17 +
.../monitor/SchedulingEditPolicy.java | 4 +-
.../monitor/SchedulingMonitor.java | 4 +-
.../ProportionalCapacityPreemptionPolicy.java | 4 +-
.../monitor/invariants/InvariantsChecker.java | 10 +-
.../invariants/MetricsInvariantChecker.java | 7 +-
.../recovery/FileSystemRMStateStore.java | 2 +-
.../recovery/LeveldbRMStateStore.java | 2 +-
.../recovery/MemoryRMStateStore.java | 1 +
.../resourcemanager/recovery/RMStateStore.java | 5 +
.../recovery/ZKRMStateStore.java | 59 +-
.../reservation/ReservationInputValidator.java | 18 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 3 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 19 +-
.../rmapp/attempt/RMAppAttemptMetrics.java | 19 +-
.../rmcontainer/RMContainerImpl.java | 8 +-
.../server/resourcemanager/rmnode/RMNode.java | 6 +-
.../rmnode/RMNodeDecreaseContainerEvent.java | 39 -
.../resourcemanager/rmnode/RMNodeEventType.java | 2 +-
.../resourcemanager/rmnode/RMNodeImpl.java | 29 +-
.../rmnode/RMNodeUpdateContainerEvent.java | 44 +
.../scheduler/AbstractYarnScheduler.java | 23 +-
.../scheduler/AppSchedulingInfo.java | 6 +-
.../scheduler/SchedulerApplicationAttempt.java | 43 +-
.../scheduler/SchedulerUtils.java | 17 +-
.../scheduler/capacity/AbstractCSQueue.java | 8 +
.../scheduler/capacity/CapacityScheduler.java | 66 +-
.../CapacitySchedulerConfiguration.java | 23 +
.../capacity/CapacitySchedulerQueueManager.java | 4 +
.../scheduler/capacity/LeafQueue.java | 5 -
.../scheduler/capacity/ParentQueue.java | 45 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 13 +
.../scheduler/fair/AllocationConfiguration.java | 3 -
.../fair/AllocationFileLoaderService.java | 10 +-
.../scheduler/fair/FSAppAttempt.java | 17 +-
.../scheduler/fair/FSPreemptionThread.java | 28 +-
.../scheduler/fair/FSSchedulerNode.java | 68 +-
.../scheduler/fair/FairScheduler.java | 24 +-
.../scheduler/fifo/FifoScheduler.java | 15 +-
.../security/RMContainerTokenSecretManager.java | 30 +-
.../resourcemanager/webapp/AboutBlock.java | 20 +-
.../resourcemanager/webapp/AboutPage.java | 2 +-
.../resourcemanager/webapp/AppAttemptPage.java | 2 +-
.../webapp/AppLogAggregationStatusPage.java | 2 +-
.../server/resourcemanager/webapp/AppPage.java | 2 +-
.../webapp/AppsBlockWithMetrics.java | 5 +-
.../webapp/CapacitySchedulerPage.java | 220 +-
.../resourcemanager/webapp/ContainerPage.java | 2 +-
.../webapp/DefaultSchedulerPage.java | 58 +-
.../resourcemanager/webapp/ErrorBlock.java | 2 +-
.../webapp/FairSchedulerAppsBlock.java | 12 +-
.../webapp/FairSchedulerPage.java | 118 +-
.../webapp/MetricsOverviewTable.java | 124 +-
.../server/resourcemanager/webapp/NavBlock.java | 34 +-
.../resourcemanager/webapp/NodeLabelsPage.java | 18 +-
.../resourcemanager/webapp/NodesPage.java | 16 +-
.../webapp/RMAppAttemptBlock.java | 48 +-
.../resourcemanager/webapp/RMAppBlock.java | 28 +-
.../webapp/RMAppLogAggregationStatusBlock.java | 34 +-
.../resourcemanager/webapp/RMAppsBlock.java | 15 +-
.../webapp/RMErrorsAndWarningsPage.java | 2 +-
.../resourcemanager/webapp/RMWSConsts.java | 15 +
.../resourcemanager/webapp/RMWebAppUtil.java | 224 +
.../webapp/RMWebServiceProtocol.java | 133 +-
.../resourcemanager/webapp/RMWebServices.java | 183 +-
.../webapp/RedirectionErrorPage.java | 2 +-
.../server/resourcemanager/webapp/RmView.java | 4 +-
.../webapp/SchedulerPageUtil.java | 8 +-
.../webapp/dao/AppAttemptInfo.java | 5 +-
.../webapp/dao/ClusterMetricsInfo.java | 5 +-
.../webapp/dao/FairSchedulerQueueInfo.java | 41 +-
.../yarn/server/resourcemanager/MockNodes.java | 2 +-
.../server/resourcemanager/NodeManager.java | 9 +
.../server/resourcemanager/RMHATestBase.java | 5 +-
.../resourcemanager/TestAMAuthorization.java | 9 +
.../TestApplicationMasterLauncher.java | 15 +-
.../resourcemanager/TestClientRMService.java | 4 +-
...pportunisticContainerAllocatorAMService.java | 168 +
.../server/resourcemanager/TestRMRestart.java | 8 +-
.../TestWorkPreservingRMRestart.java | 10 +-
.../TestFederationRMStateStoreService.java | 176 +
.../recovery/RMStateStoreTestBase.java | 8 +-
.../recovery/TestFSRMStateStore.java | 1 +
.../recovery/TestLeveldbRMStateStore.java | 1 +
.../recovery/TestZKRMStateStore.java | 1 +
.../TestReservationInputValidator.java | 103 +-
.../scheduler/TestSchedulerUtils.java | 4 +-
.../capacity/TestCapacityScheduler.java | 360 +-
.../TestCapacitySchedulerAsyncScheduling.java | 147 +
.../TestCapacitySchedulerDynamicBehavior.java | 2 +-
.../capacity/TestContainerResizing.java | 7 +-
.../capacity/TestIncreaseAllocationExpirer.java | 4 +-
.../scheduler/capacity/TestLeafQueue.java | 2 -
.../scheduler/fair/TestFSSchedulerNode.java | 52 +
.../scheduler/fair/TestFairScheduler.java | 46 +-
.../hadoop/yarn/webapp/TestRMWithXFSFilter.java | 14 +-
.../hadoop-yarn-server-router/pom.xml | 110 +
.../hadoop/yarn/server/router/Router.java | 179 +
.../yarn/server/router/RouterServerUtil.java | 63 +
.../AbstractClientRequestInterceptor.java | 127 +
.../clientrm/ClientRequestInterceptor.java | 65 +
.../DefaultClientRequestInterceptor.java | 311 +
.../clientrm/FederationClientInterceptor.java | 677 +
.../router/clientrm/RouterClientRMService.java | 546 +
.../server/router/clientrm/package-info.java | 20 +
.../hadoop/yarn/server/router/package-info.java | 20 +
.../AbstractRMAdminRequestInterceptor.java | 90 +
.../DefaultRMAdminRequestInterceptor.java | 215 +
.../rmadmin/RMAdminRequestInterceptor.java | 65 +
.../router/rmadmin/RouterRMAdminService.java | 423 +
.../server/router/rmadmin/package-info.java | 20 +
.../webapp/AbstractRESTRequestInterceptor.java | 89 +
.../webapp/DefaultRequestInterceptorREST.java | 510 +
.../webapp/FederationInterceptorREST.java | 750 +
.../yarn/server/router/webapp/HTTPMethods.java | 34 +
.../router/webapp/RESTRequestInterceptor.java | 125 +
.../yarn/server/router/webapp/RouterWebApp.java | 48 +
.../router/webapp/RouterWebServiceUtil.java | 227 +
.../server/router/webapp/RouterWebServices.java | 876 +
.../yarn/server/router/webapp/package-info.java | 20 +
.../router/clientrm/BaseRouterClientRMTest.java | 586 +
.../clientrm/MockClientRequestInterceptor.java | 36 +
.../PassThroughClientRequestInterceptor.java | 267 +
.../TestFederationClientInterceptor.java | 403 +
.../TestFederationClientInterceptorRetry.java | 295 +
.../clientrm/TestRouterClientRMService.java | 210 +
.../TestableFederationClientInterceptor.java | 75 +
.../router/rmadmin/BaseRouterRMAdminTest.java | 346 +
.../rmadmin/MockRMAdminRequestInterceptor.java | 36 +
.../PassThroughRMAdminRequestInterceptor.java | 148 +
.../rmadmin/TestRouterRMAdminService.java | 219 +
.../webapp/BaseRouterWebServicesTest.java | 614 +
.../yarn/server/router/webapp/JavaProcess.java | 52 +
.../MockDefaultRequestInterceptorREST.java | 136 +
.../webapp/MockRESTRequestInterceptor.java | 340 +
.../PassThroughRESTRequestInterceptor.java | 339 +
.../webapp/TestFederationInterceptorREST.java | 379 +
.../TestFederationInterceptorRESTRetry.java | 274 +
.../router/webapp/TestRouterWebServices.java | 269 +
.../webapp/TestRouterWebServicesREST.java | 1298 +
.../TestableFederationInterceptorREST.java | 54 +
.../src/test/resources/capacity-scheduler.xml | 111 +
.../src/test/resources/log4j.properties | 19 +
.../src/test/resources/yarn-site.xml | 30 +
.../sharedcachemanager/CleanerService.java | 7 +-
.../server/sharedcachemanager/CleanerTask.java | 7 +-
.../ClientProtocolService.java | 7 +-
.../SCMAdminProtocolService.java | 8 +-
.../sharedcachemanager/SharedCacheManager.java | 9 +-
.../metrics/CleanerMetrics.java | 7 +-
.../metrics/ClientSCMMetrics.java | 7 +-
.../metrics/SharedCacheUploaderMetrics.java | 8 +-
.../store/InMemorySCMStore.java | 7 +-
.../webapp/SCMOverviewPage.java | 26 +-
.../sharedcachemanager/webapp/SCMWebServer.java | 7 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 13 +-
.../timeline/EntityGroupFSTimelineStore.java | 23 +-
.../timeline/LevelDBCacheTimelineStore.java | 14 +-
.../TestEntityGroupFSTimelineStore.java | 99 +-
...stTimelineReaderWebServicesHBaseStorage.java | 30 +-
.../storage/TestHBaseTimelineStorageApps.java | 4 +-
.../TestHBaseTimelineStorageEntities.java | 14 +-
.../reader/filter/TimelineFilterUtils.java | 7 +-
.../storage/HBaseTimelineReaderImpl.java | 8 +-
.../storage/HBaseTimelineWriterImpl.java | 8 +-
.../storage/TimelineSchemaCreator.java | 7 +-
.../storage/application/ApplicationTable.java | 7 +-
.../storage/apptoflow/AppToFlowTable.java | 7 +-
.../storage/common/AppIdKeyConverter.java | 3 +-
.../storage/common/ColumnHelper.java | 8 +-
.../common/HBaseTimelineStorageUtils.java | 41 +-
.../storage/entity/EntityTable.java | 7 +-
.../storage/flow/FlowActivityTable.java | 7 +-
.../storage/flow/FlowRunCoprocessor.java | 7 +-
.../storage/flow/FlowRunTable.java | 7 +-
.../storage/flow/FlowScanner.java | 7 +-
.../storage/reader/TimelineEntityReader.java | 7 +-
.../TestCustomApplicationIdConversion.java | 39 +
.../collector/AppLevelTimelineCollector.java | 7 +-
.../collector/NodeTimelineCollectorManager.java | 8 +-
.../PerNodeTimelineCollectorsAuxService.java | 10 +-
.../collector/TimelineCollector.java | 7 +-
.../collector/TimelineCollectorManager.java | 8 +-
.../collector/TimelineCollectorWebService.java | 8 +-
.../reader/TimelineReaderServer.java | 9 +-
.../reader/TimelineReaderWebServices.java | 8 +-
.../storage/FileSystemTimelineReaderImpl.java | 8 +-
.../storage/common/TimelineStorageUtils.java | 4 -
.../hadoop/yarn/server/webproxy/ProxyUtils.java | 16 +-
.../server/webproxy/WebAppProxyServlet.java | 14 +-
.../hadoop-yarn/hadoop-yarn-server/pom.xml | 1 +
.../src/site/markdown/Federation.md | 309 +
.../src/site/markdown/GracefulDecommission.md | 12 +-
.../src/site/markdown/NodeManagerCgroups.md | 17 +-
.../site/markdown/WritingYarnApplications.md | 4 +-
.../hadoop-yarn-site/src/site/markdown/YARN.md | 5 +-
.../src/site/markdown/registry/yarn-registry.md | 14 +-
.../resources/images/amrmproxy_architecture.png | Bin 0 -> 35010 bytes
.../images/federation_architecture.png | Bin 0 -> 46964 bytes
.../images/federation_sequence_diagram.png | Bin 0 -> 47910 bytes
.../src/main/webapp/app/adapters/yarn-queue.js | 30 -
.../app/adapters/yarn-queue/capacity-queue.js | 23 +
.../app/adapters/yarn-queue/fair-queue.js | 23 +
.../app/adapters/yarn-queue/fifo-queue.js | 23 +
.../app/adapters/yarn-queue/yarn-queue.js | 30 +
.../main/webapp/app/components/tree-selector.js | 19 +-
.../src/main/webapp/app/initializers/loader.js | 42 +-
.../src/main/webapp/app/models/yarn-queue.js | 94 -
.../app/models/yarn-queue/capacity-queue.js | 95 +
.../webapp/app/models/yarn-queue/fair-queue.js | 79 +
.../webapp/app/models/yarn-queue/fifo-queue.js | 52 +
.../webapp/app/models/yarn-queue/yarn-queue.js | 23 +
.../main/webapp/app/routes/cluster-overview.js | 4 +-
.../src/main/webapp/app/routes/yarn-queue.js | 26 +-
.../src/main/webapp/app/routes/yarn-queues.js | 12 +-
.../main/webapp/app/routes/yarn-queues/index.js | 25 -
.../app/routes/yarn-queues/queues-selector.js | 25 -
.../main/webapp/app/serializers/yarn-queue.js | 129 -
.../serializers/yarn-queue/capacity-queue.js | 128 +
.../app/serializers/yarn-queue/fair-queue.js | 92 +
.../app/serializers/yarn-queue/fifo-queue.js | 59 +
.../app/serializers/yarn-queue/yarn-queue.js | 47 +
.../components/queue-configuration-table.hbs | 54 -
.../templates/components/queue-navigator.hbs | 7 +-
.../yarn-queue/capacity-queue-conf-table.hbs | 54 +
.../yarn-queue/capacity-queue-info.hbs | 84 +
.../components/yarn-queue/capacity-queue.hbs | 63 +
.../yarn-queue/fair-queue-conf-table.hbs | 52 +
.../components/yarn-queue/fair-queue-info.hbs | 66 +
.../components/yarn-queue/fair-queue.hbs | 63 +
.../yarn-queue/fifo-queue-conf-table.hbs | 56 +
.../components/yarn-queue/fifo-queue-info.hbs | 47 +
.../components/yarn-queue/fifo-queue.hbs | 48 +
.../webapp/app/templates/yarn-queue/info.hbs | 73 +-
.../main/webapp/app/templates/yarn-queues.hbs | 54 +-
.../src/main/webapp/app/utils/color-utils.js | 1 -
hadoop-yarn-project/hadoop-yarn/pom.xml | 12 +-
hadoop-yarn-project/pom.xml | 4 +
pom.xml | 5 +-
start-build-env.sh | 2 +-
962 files changed, 94710 insertions(+), 7270 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0bd0f62/LICENSE.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0bd0f62/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0bd0f62/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0bd0f62/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0bd0f62/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0bd0f62/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0bd0f62/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 914635e,e6405b5..61e4808
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@@ -1,4 -1,4 +1,4 @@@
--#!/usr/bin/env bash
++ #!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@@ -31,45 -31,37 +31,47 @@@ function hadoop_usag
hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
hadoop_add_option "--workers" "turn on worker mode"
- hadoop_add_subcommand "balancer" "run a cluster balancing utility"
- hadoop_add_subcommand "cacheadmin" "configure the HDFS cache"
- hadoop_add_subcommand "cblock" "cblock CLI"
- hadoop_add_subcommand "cblockserver" "run cblock server"
- hadoop_add_subcommand "classpath" "prints the class path needed to get the
hadoop jar and the required libraries"
- hadoop_add_subcommand "corona" "run an ozone data generator"
- hadoop_add_subcommand "crypto" "configure HDFS encryption zones"
- hadoop_add_subcommand "datanode" "run a DFS datanode"
- hadoop_add_subcommand "debug" "run a Debug Admin to execute HDFS debug
commands"
- hadoop_add_subcommand "dfs" "run a filesystem command on the file system"
- hadoop_add_subcommand "dfsadmin" "run a DFS admin client"
- hadoop_add_subcommand "diskbalancer" "Distributes data evenly among disks
on a given node"
- hadoop_add_subcommand "envvars" "display computed Hadoop environment
variables"
- hadoop_add_subcommand "ec" "run a HDFS ErasureCoding CLI"
- hadoop_add_subcommand "fetchdt" "fetch a delegation token from the NameNode"
- hadoop_add_subcommand "fsck" "run a DFS filesystem checking utility"
- hadoop_add_subcommand "getconf" "get config values from configuration"
- hadoop_add_subcommand "groups" "get the groups which users belong to"
- hadoop_add_subcommand "haadmin" "run a DFS HA admin client"
- hadoop_add_subcommand "jmxget" "get JMX exported values from NameNode or
DataNode."
- hadoop_add_subcommand "journalnode" "run the DFS journalnode"
- hadoop_add_subcommand "jscsi" "run cblock jscsi server"
- hadoop_add_subcommand "ksm" "Ozone keyspace manager"
- hadoop_add_subcommand "lsSnapshottableDir" "list all snapshottable dirs
owned by the current user"
- hadoop_add_subcommand "mover" "run a utility to move block replicas across
storage types"
- hadoop_add_subcommand "namenode" "run the DFS namenode"
- hadoop_add_subcommand "nfs3" "run an NFS version 3 gateway"
- hadoop_add_subcommand "oev" "apply the offline edits viewer to an edits
file"
- hadoop_add_subcommand "oiv" "apply the offline fsimage viewer to an fsimage"
- hadoop_add_subcommand "oiv_legacy" "apply the offline fsimage viewer to a
legacy fsimage"
- hadoop_add_subcommand "oz" "command line interface for ozone"
- hadoop_add_subcommand "oz_debug" "ozone debug tool, convert ozone meta data
db into sqlite db"
- hadoop_add_subcommand "portmap" "run a portmap service"
- hadoop_add_subcommand "scm" "run the Storage Container Manager service"
- hadoop_add_subcommand "secondarynamenode" "run the DFS secondary namenode"
- hadoop_add_subcommand "snapshotDiff" "diff two snapshots of a directory or
diff the current directory contents with a snapshot"
- hadoop_add_subcommand "storagepolicies" "list/get/set block storage
policies"
- hadoop_add_subcommand "version" "print the version"
- hadoop_add_subcommand "zkfc" "run the ZK Failover Controller daemon"
++
+ hadoop_add_subcommand "balancer" daemon "run a cluster balancing utility"
+ hadoop_add_subcommand "cacheadmin" admin "configure the HDFS cache"
++ hadoop_add_subcommand "cblock" admin "cblock CLI"
++ hadoop_add_subcommand "cblockserver" daemon "run cblock server"
+ hadoop_add_subcommand "classpath" client "prints the class path needed to
get the hadoop jar and the required libraries"
++ hadoop_add_subcommand "corona" client "run an ozone data generator"
+ hadoop_add_subcommand "crypto" admin "configure HDFS encryption zones"
+ hadoop_add_subcommand "datanode" daemon "run a DFS datanode"
+ hadoop_add_subcommand "debug" admin "run a Debug Admin to execute HDFS
debug commands"
+ hadoop_add_subcommand "dfs" client "run a filesystem command on the file
system"
+ hadoop_add_subcommand "dfsadmin" admin "run a DFS admin client"
+ hadoop_add_subcommand "diskbalancer" daemon "Distributes data evenly among
disks on a given node"
+ hadoop_add_subcommand "envvars" client "display computed Hadoop environment
variables"
+ hadoop_add_subcommand "ec" admin "run a HDFS ErasureCoding CLI"
+ hadoop_add_subcommand "fetchdt" client "fetch a delegation token from the
NameNode"
+ hadoop_add_subcommand "fsck" admin "run a DFS filesystem checking utility"
+ hadoop_add_subcommand "getconf" client "get config values from
configuration"
+ hadoop_add_subcommand "groups" client "get the groups which users belong to"
+ hadoop_add_subcommand "haadmin" admin "run a DFS HA admin client"
+ hadoop_add_subcommand "jmxget" admin "get JMX exported values from NameNode
or DataNode."
+ hadoop_add_subcommand "journalnode" daemon "run the DFS journalnode"
++ hadoop_add_subcommand "jscsi" daemon "run cblock jscsi server"
++ hadoop_add_subcommand "ksm" daemon "Ozone keyspace manager"
+ hadoop_add_subcommand "lsSnapshottableDir" client "list all snapshottable
dirs owned by the current user"
+ hadoop_add_subcommand "mover" daemon "run a utility to move block replicas
across storage types"
+ hadoop_add_subcommand "namenode" daemon "run the DFS namenode"
+ hadoop_add_subcommand "nfs3" daemon "run an NFS version 3 gateway"
+ hadoop_add_subcommand "oev" admin "apply the offline edits viewer to an
edits file"
+ hadoop_add_subcommand "oiv" admin "apply the offline fsimage viewer to an
fsimage"
+ hadoop_add_subcommand "oiv_legacy" admin "apply the offline fsimage viewer
to a legacy fsimage"
++ hadoop_add_subcommand "oz" client "command line interface for ozone"
++ hadoop_add_subcommand "oz_debug" client "ozone debug tool, convert ozone
metadata into relational data"
+ hadoop_add_subcommand "portmap" daemon "run a portmap service"
++ hadoop_add_subcommand "scm" daemon "run the Storage Container Manager
service"
+ hadoop_add_subcommand "secondarynamenode" daemon "run the DFS secondary
namenode"
+ hadoop_add_subcommand "snapshotDiff" client "diff two snapshots of a
directory or diff the current directory contents with a snapshot"
+ hadoop_add_subcommand "storagepolicies" admin "list/get/set block storage
policies"
+ hadoop_add_subcommand "version" client "print the version"
+ hadoop_add_subcommand "zkfc" daemon "run the ZK Failover Controller daemon"
++
hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0bd0f62/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0bd0f62/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0bd0f62/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --cc hadoop-project/pom.xml
index 8b99379,8c1d374..e1d22b4
mode 100644,100755..100755
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@@ -98,8 -96,10 +96,12 @@@
<apacheds.version>2.0.0-M21</apacheds.version>
<ldap-api.version>1.0.0-M33</ldap-api.version>
+ <!-- Apache Ratis version -->
+ <ratis.version>0.1.1-alpha-SNAPSHOT</ratis.version>
+ <jcache.version>1.0-alpha-1</jcache.version>
+ <ehcache.version>3.3.1</ehcache.version>
+ <hikari.version>2.4.12</hikari.version>
+ <mssql.version>6.2.1.jre7</mssql.version>
<!-- define the Java language version used by the compiler -->
<javac.version>1.8</javac.version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0bd0f62/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0bd0f62/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
----------------------------------------------------------------------
diff --cc
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
index fbb56b0,09b12f2..1c39fa7
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@@@ -815,8 -821,8 +821,8 @@@ public class TestAMRMClient
amClient.updateBlacklist(nodeList02, null);
assertEquals(3, amClient.blacklistAdditions.size());
assertEquals(0, amClient.blacklistRemovals.size());
-
- // Add nodes[1] and nodes[2] to removal list,
+
- // Add nodes[1] and nodes[2] to removal list,
++ // Add nodes[1] and nodes[2] to removal list,
// Verify addition list remove these two nodes.
List<String> nodeList12 = new ArrayList<String>();
nodeList12.add(nodes[1]);
@@@ -824,8 -830,8 +830,8 @@@
amClient.updateBlacklist(null, nodeList12);
assertEquals(1, amClient.blacklistAdditions.size());
assertEquals(2, amClient.blacklistRemovals.size());
-
- // Add nodes[1] again to addition list,
+
- // Add nodes[1] again to addition list,
++ // Add nodes[1] again to addition list,
// Verify removal list will remove this node.
List<String> nodeList1 = new ArrayList<String>();
nodeList1.add(nodes[1]);
@@@ -933,10 -939,10 +939,10 @@@
1), null, null, Priority.UNDEFINED, true, "x"));
client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024,
1), null, null, Priority.UNDEFINED, true, "a"));
- Assert.assertEquals(1, client.ask.size());
- Assert.assertEquals("a", client.ask.iterator().next()
+ assertEquals(1, client.ask.size());
+ assertEquals("a", client.ask.iterator().next()
.getNodeLabelExpression());
-
+
// add exp=x to ANY, rack and node, only resource request has ANY resource
// name will be assigned the label expression
// add exp=x then add exp=a to ANY in same priority, only exp=a should
kept
@@@ -971,9 -977,9 +977,9 @@@
} catch (InvalidContainerRequestException e) {
return;
}
- Assert.fail();
+ fail();
}
-
+
@Test(timeout=30000)
public void testAskWithInvalidNodeLabels() {
AMRMClientImpl<ContainerRequest> client =
@@@ -1170,11 -1507,11 +1507,11 @@@
int allocatedContainerCount = 0;
int iterationsLeft = 3;
Set<ContainerId> releases = new TreeSet<ContainerId>();
-
+
amClient.getNMTokenCache().clearCache();
- Assert.assertEquals(0,
amClient.getNMTokenCache().numberOfTokensInCache());
+ assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache());
HashMap<String, Token> receivedNMTokens = new HashMap<String, Token>();
-
+
while (allocatedContainerCount < containersRequestedAny
&& iterationsLeft-- > 0) {
AllocateResponse allocResponse = amClient.allocate(0.1f);
@@@ -1202,11 -1539,11 +1539,11 @@@
triggerSchedulingWithNMHeartBeat();
}
}
-
+
// Should receive atleast 1 token
- Assert.assertTrue(receivedNMTokens.size() > 0
+ assertTrue(receivedNMTokens.size() > 0
&& receivedNMTokens.size() <= nodeCount);
-
+
assertEquals(allocatedContainerCount, containersRequestedAny);
assertEquals(2, releases.size());
assertEquals(0, amClient.ask.size());
@@@ -1254,7 -1591,7 +1591,7 @@@
assertEquals(2, amClient.release.size());
assertEquals(3, amClient.ask.size());
snoopRequest = amClient.ask.iterator().next();
-- // verify that the remove request made in between makeRequest and
allocate
++ // verify that the remove request made in between makeRequest and allocate
// has not been lost
assertEquals(0, snoopRequest.getNumContainers());
@@@ -1444,8 -1781,8 +1781,8 @@@
org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken_1
=
getAMRMToken();
Assert.assertNotNull(amrmToken_1);
- Assert.assertEquals(amrmToken_1.decodeIdentifier().getKeyId(),
+ assertEquals(amrmToken_1.decodeIdentifier().getKeyId(),
- amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
+ amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
// Wait for enough time and make sure the roll_over happens
// At mean time, the old AMRMToken should continue to work
@@@ -1459,8 -1796,8 +1796,8 @@@
org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken_2
=
getAMRMToken();
Assert.assertNotNull(amrmToken_2);
- Assert.assertEquals(amrmToken_2.decodeIdentifier().getKeyId(),
+ assertEquals(amrmToken_2.decodeIdentifier().getKeyId(),
- amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
+ amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
Assert.assertNotEquals(amrmToken_1, amrmToken_2);
@@@ -1470,22 -1807,22 +1807,22 @@@
// Verify latest AMRMToken can be used to send allocation request.
UserGroupInformation testUser1 =
UserGroupInformation.createRemoteUser("testUser1");
-
- AMRMTokenIdentifierForTest newVersionTokenIdentifier =
+
+ AMRMTokenIdentifierForTest newVersionTokenIdentifier =
new AMRMTokenIdentifierForTest(amrmToken_2.decodeIdentifier(),
"message");
-
+
- Assert.assertEquals("Message is changed after set to
newVersionTokenIdentifier",
+ assertEquals("Message is changed after set to
newVersionTokenIdentifier",
"message", newVersionTokenIdentifier.getMessage());
- org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>
newVersionToken =
+ org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>
newVersionToken =
new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> (
- newVersionTokenIdentifier.getBytes(),
+ newVersionTokenIdentifier.getBytes(),
amrmTokenSecretManager.retrievePassword(newVersionTokenIdentifier),
newVersionTokenIdentifier.getKind(), new Text());
-
+
SecurityUtil.setTokenService(newVersionToken, yarnCluster
- .getResourceManager().getApplicationMasterService().getBindAddress());
+
.getResourceManager().getApplicationMasterService().getBindAddress());
testUser1.addToken(newVersionToken);
-
+
AllocateRequest request = Records.newRecord(AllocateRequest.class);
request.setResponseId(response.getResponseId());
testUser1.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
@@@ -1525,17 -1862,17 +1862,17 @@@
@Override
public ApplicationMasterProtocol run() {
return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy(
- ApplicationMasterProtocol.class,
- yarnCluster.getResourceManager().getApplicationMasterService()
- .getBindAddress(), conf);
+ ApplicationMasterProtocol.class,
+ yarnCluster.getResourceManager().getApplicationMasterService()
+ .getBindAddress(), conf);
}
}).allocate(Records.newRecord(AllocateRequest.class));
- Assert.fail("The old Token should not work");
+ fail("The old Token should not work");
} catch (Exception ex) {
- Assert.assertTrue(ex instanceof InvalidToken);
- Assert.assertTrue(ex.getMessage().contains(
+ assertTrue(ex instanceof InvalidToken);
+ assertTrue(ex.getMessage().contains(
- "Invalid AMRMToken from "
- + amrmToken_2.decodeIdentifier().getApplicationAttemptId()));
+ "Invalid AMRMToken from "
+ + amrmToken_2.decodeIdentifier().getApplicationAttemptId()));
}
amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
