Merge trunk into branch
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/MR-2841@1612742 13f79535-47bb-0310-9956-ffa450edef68 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/341695e7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/341695e7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/341695e7 Branch: refs/heads/trunk Commit: 341695e7317d156c8cd869f52978100d4b88a6ea Parents: e788e8b e0f00e1 Author: Todd Lipcon <t...@apache.org> Authored: Wed Jul 23 01:47:28 2014 +0000 Committer: Todd Lipcon <t...@apache.org> Committed: Wed Jul 23 01:47:28 2014 +0000 ---------------------------------------------------------------------- hadoop-common-project/hadoop-auth/pom.xml | 11 + hadoop-common-project/hadoop-common/CHANGES.txt | 81 +- .../hadoop-common/src/main/bin/hadoop-daemon.sh | 1 + .../hadoop/crypto/key/CachingKeyProvider.java | 174 +++++ .../crypto/key/KeyProviderCryptoExtension.java | 235 ++++-- .../hadoop/crypto/key/KeyProviderFactory.java | 10 + .../org/apache/hadoop/crypto/key/KeyShell.java | 41 +- .../crypto/key/kms/KMSClientProvider.java | 161 +++- .../hadoop/crypto/key/kms/KMSRESTConstants.java | 8 + .../hadoop/crypto/key/kms/ValueQueue.java | 317 ++++++++ .../fs/CommonConfigurationKeysPublic.java | 33 + .../main/java/org/apache/hadoop/fs/Stat.java | 13 +- .../hadoop/fs/s3native/NativeS3FileSystem.java | 9 +- .../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 34 + .../apache/hadoop/fs/viewfs/ViewFileSystem.java | 42 + .../org/apache/hadoop/fs/viewfs/ViewFs.java | 89 +++ .../main/java/org/apache/hadoop/io/Text.java | 19 +- .../apache/hadoop/io/compress/BZip2Codec.java | 6 +- .../hadoop/io/compress/CompressionCodec.java | 55 ++ .../io/compress/CompressionInputStream.java | 10 + .../io/compress/CompressionOutputStream.java | 18 +- .../apache/hadoop/io/compress/DefaultCodec.java | 14 +- .../apache/hadoop/io/compress/GzipCodec.java | 14 +- .../org/apache/hadoop/io/compress/Lz4Codec.java | 6 +- .../apache/hadoop/io/compress/SnappyCodec.java | 6 +- .../apache/hadoop/ipc/ProtobufRpcEngine.java | 29 +- .../main/java/org/apache/hadoop/ipc/Server.java | 12 +- .../apache/hadoop/ipc/WritableRpcEngine.java | 41 +- .../apache/hadoop/metrics2/MetricsSystem.java | 6 + .../hadoop/metrics2/impl/MetricsConfig.java | 2 +- .../hadoop/metrics2/impl/MetricsSystemImpl.java | 11 + .../org/apache/hadoop/net/NetworkTopology.java | 9 +- .../net/NetworkTopologyWithNodeGroup.java | 7 +- .../java/org/apache/hadoop/security/Groups.java | 52 +- .../hadoop/security/LdapGroupsMapping.java | 6 +- .../security/ShellBasedUnixGroupsMapping.java | 3 +- .../hadoop/security/alias/CredentialShell.java | 4 +- .../security/alias/JavaKeyStoreProvider.java | 3 + .../authorize/DefaultImpersonationProvider.java | 65 +- .../authorize/ImpersonationProvider.java | 15 + .../hadoop/security/authorize/ProxyUsers.java | 29 +- .../main/java/org/apache/hadoop/util/Timer.java | 51 ++ .../main/java/org/apache/hadoop/util/Tool.java | 2 +- .../src/main/resources/core-default.xml | 47 ++ .../src/site/apt/CommandsManual.apt.vm | 4 +- .../src/site/apt/NativeLibraries.apt.vm | 2 + .../crypto/key/TestCachingKeyProvider.java | 152 ++++ .../key/TestKeyProviderCryptoExtension.java | 124 ++- .../apache/hadoop/crypto/key/TestKeyShell.java | 16 +- .../hadoop/crypto/key/TestValueQueue.java | 190 +++++ .../java/org/apache/hadoop/fs/TestStat.java | 28 +- .../fs/viewfs/ViewFileSystemBaseTest.java | 55 +- .../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 52 ++ .../java/org/apache/hadoop/io/TestText.java | 22 + .../org/apache/hadoop/ipc/MiniRPCBenchmark.java | 8 +- .../java/org/apache/hadoop/ipc/TestRPC.java | 2 + .../metrics2/impl/TestGangliaMetrics.java | 12 +- .../metrics2/impl/TestMetricsSystemImpl.java | 50 +- .../net/TestNetworkTopologyWithNodeGroup.java | 8 +- .../hadoop/security/TestDoAsEffectiveUser.java | 23 +- .../hadoop/security/TestGroupsCaching.java | 56 ++ .../hadoop/security/alias/TestCredShell.java | 20 +- .../security/authorize/TestProxyUsers.java | 126 ++- .../java/org/apache/hadoop/util/FakeTimer.java | 52 ++ .../hadoop-kms/src/main/conf/kms-acls.xml | 15 + ...rKeyGeneratorKeyProviderCryptoExtension.java | 149 ++++ .../hadoop/crypto/key/kms/server/KMS.java | 97 ++- .../hadoop/crypto/key/kms/server/KMSACLs.java | 36 +- .../key/kms/server/KMSCacheKeyProvider.java | 177 ----- .../crypto/key/kms/server/KMSConfiguration.java | 14 +- .../key/kms/server/KMSServerJSONUtils.java | 21 +- .../hadoop/crypto/key/kms/server/KMSWebApp.java | 50 +- .../hadoop-kms/src/site/apt/index.apt.vm | 110 ++- .../hadoop/crypto/key/kms/server/TestKMS.java | 111 ++- .../key/kms/server/TestKMSCacheKeyProvider.java | 120 --- .../java/org/apache/hadoop/nfs/NfsExports.java | 25 +- .../org/apache/hadoop/nfs/TestNfsExports.java | 12 + .../apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java | 2 + .../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 20 +- .../nfs/nfs3/TestClientAccessPrivilege.java | 120 +++ .../hadoop/hdfs/nfs/nfs3/TestNfs3Utils.java | 7 + .../hadoop/hdfs/nfs/nfs3/TestReaddir.java | 8 +- .../apache/hadoop/hdfs/nfs/nfs3/TestWrites.java | 6 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 58 ++ .../bkjournal/TestBootstrapStandbyWithBKJM.java | 169 ++++ .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 + .../org/apache/hadoop/hdfs/DFSOutputStream.java | 50 +- .../java/org/apache/hadoop/hdfs/HAUtil.java | 35 +- .../org/apache/hadoop/hdfs/NameNodeProxies.java | 57 +- .../hadoop/hdfs/protocol/DatanodeInfo.java | 2 +- .../hadoop/hdfs/protocol/HdfsConstants.java | 2 +- .../datatransfer/DataTransferProtocol.java | 21 +- .../hdfs/protocol/datatransfer/Receiver.java | 11 +- .../hdfs/protocol/datatransfer/Sender.java | 11 +- .../DatanodeProtocolClientSideTranslatorPB.java | 29 +- .../NamenodeProtocolTranslatorPB.java | 8 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 67 +- .../hadoop/hdfs/server/balancer/Balancer.java | 6 +- .../server/blockmanagement/BlockManager.java | 38 +- .../blockmanagement/BlockPlacementPolicy.java | 65 +- .../BlockPlacementPolicyDefault.java | 59 +- .../BlockPlacementPolicyWithNodeGroup.java | 40 +- .../server/blockmanagement/DatanodeManager.java | 11 +- .../blockmanagement/DatanodeStatistics.java | 6 + .../blockmanagement/DatanodeStorageInfo.java | 18 + .../blockmanagement/HeartbeatManager.java | 21 +- .../hdfs/server/common/HdfsServerConstants.java | 3 +- .../hdfs/server/datanode/BPOfferService.java | 3 +- .../hdfs/server/datanode/BlockReceiver.java | 10 +- .../hadoop/hdfs/server/datanode/DataNode.java | 150 +++- .../hdfs/server/datanode/DataXceiver.java | 25 +- .../server/datanode/fsdataset/FsDatasetSpi.java | 9 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 74 +- .../datanode/fsdataset/impl/FsVolumeList.java | 21 +- .../web/resources/DatanodeWebHdfsMethods.java | 3 +- .../hdfs/server/namenode/FSClusterStats.java | 9 + .../hadoop/hdfs/server/namenode/FSImage.java | 6 +- .../hdfs/server/namenode/FSNamesystem.java | 35 +- .../hadoop/hdfs/server/namenode/NNStorage.java | 2 +- .../hadoop/hdfs/server/namenode/NameNode.java | 19 +- .../hdfs/server/namenode/NamenodeFsck.java | 9 +- .../hdfs/server/namenode/XAttrStorage.java | 40 +- .../server/namenode/ha/BootstrapStandby.java | 7 +- .../web/resources/NamenodeWebHdfsMethods.java | 117 ++- .../hdfs/server/protocol/BlockCommand.java | 15 +- .../hdfs/tools/DFSZKFailoverController.java | 5 + .../org/apache/hadoop/hdfs/tools/DFSck.java | 6 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 31 +- .../web/resources/ExcludeDatanodesParam.java | 42 + .../src/main/proto/DatanodeProtocol.proto | 1 + .../src/main/proto/datatransfer.proto | 4 + .../hadoop-hdfs/src/main/proto/hdfs.proto | 7 + .../src/main/resources/hdfs-default.xml | 13 + .../src/main/webapps/hdfs/explorer.html | 31 +- .../site/apt/HDFSHighAvailabilityWithQJM.apt.vm | 4 +- .../src/site/apt/HdfsMultihoming.apt.vm | 145 ++++ .../fs/viewfs/TestViewFileSystemWithAcls.java | 190 +++++ .../hadoop/fs/viewfs/TestViewFsWithAcls.java | 190 +++++ .../org/apache/hadoop/hdfs/DFSTestUtil.java | 7 +- .../hadoop/hdfs/TestDataTransferProtocol.java | 54 +- .../apache/hadoop/hdfs/TestFileCreation.java | 17 +- .../hadoop/hdfs/TestIsMethodSupported.java | 34 +- .../hadoop/hdfs/protocolPB/TestPBHelper.java | 4 +- .../TestDelegationTokenForProxyUser.java | 6 +- .../blockmanagement/TestReplicationPolicy.java | 41 +- .../TestReplicationPolicyConsiderLoad.java | 24 +- .../TestReplicationPolicyWithNodeGroup.java | 37 +- .../hdfs/server/common/TestJspHelper.java | 6 +- .../server/datanode/BlockReportTestBase.java | 5 +- .../server/datanode/SimulatedFSDataset.java | 10 +- .../hdfs/server/datanode/TestBlockRecovery.java | 6 +- .../server/datanode/TestBlockReplacement.java | 4 +- .../hdfs/server/datanode/TestDiskError.java | 5 +- .../server/datanode/TestSimulatedFSDataset.java | 4 +- .../fsdataset/impl/TestWriteToReplica.java | 27 +- .../namenode/TestFavoredNodesEndToEnd.java | 62 +- .../namenode/TestNamenodeCapacityReport.java | 186 ++++- .../namenode/TestStartupOptionUpgrade.java | 19 +- .../hdfs/server/namenode/ha/TestDNFencing.java | 13 +- .../namenode/ha/TestDelegationTokensWithHA.java | 10 +- .../web/resources/TestWebHdfsDataLocality.java | 89 ++- .../hadoop/hdfs/tools/TestDFSAdminWithHA.java | 42 +- .../apache/hadoop/net/TestNetworkTopology.java | 45 +- .../security/TestRefreshUserMappings.java | 6 +- hadoop-mapreduce-project/CHANGES.txt | 18 + .../bin/mr-jobhistory-daemon.sh | 1 + .../hadoop/mapred/LocalContainerLauncher.java | 79 +- .../hadoop/mapreduce/v2/app/MRAppMaster.java | 268 +++++-- .../v2/app/commit/CommitterEventHandler.java | 27 +- .../v2/app/local/LocalContainerAllocator.java | 6 + .../mapreduce/v2/app/rm/RMCommunicator.java | 34 +- .../v2/app/rm/RMContainerAllocator.java | 12 + .../v2/app/rm/RMContainerRequestor.java | 36 +- .../mapred/TestLocalContainerLauncher.java | 73 ++ .../v2/app/rm/TestRMContainerAllocator.java | 244 ++++++ .../apache/hadoop/mapreduce/v2/util/MRApps.java | 46 +- .../lib/input/CombineFileInputFormat.java | 2 +- .../lib/input/TestCombineFileInputFormat.java | 55 ++ .../apache/hadoop/mapreduce/v2/TestMRJobs.java | 88 ++- .../apache/hadoop/mapred/ShuffleHandler.java | 84 +- .../hadoop/mapred/TestShuffleHandler.java | 90 +++ hadoop-project/src/site/site.xml | 1 + .../hadoop/fs/azure/NativeAzureFileSystem.java | 27 +- .../metrics/AzureFileSystemMetricsSystem.java | 19 +- .../fs/azure/AzureBlobStorageTestAccount.java | 4 +- .../fs/azure/NativeAzureFileSystemBaseTest.java | 7 + .../apache/hadoop/tools/DistCpOptionSwitch.java | 1 + .../org/apache/hadoop/tools/OptionsParser.java | 2 +- hadoop-yarn-project/CHANGES.txt | 44 ++ hadoop-yarn-project/hadoop-yarn/bin/yarn | 36 +- .../hadoop-yarn/bin/yarn-daemon.sh | 1 + .../distributedshell/ApplicationMaster.java | 52 +- .../applications/distributedshell/Client.java | 20 +- .../yarn/security/AMRMTokenIdentifier.java | 14 + .../org/apache/hadoop/yarn/util/FSDownload.java | 5 +- .../hadoop/yarn/event/DrainDispatcher.java | 14 +- .../apache/hadoop/yarn/util/TestFSDownload.java | 7 + .../security/BaseNMTokenSecretManager.java | 2 +- .../nodemanager/DefaultContainerExecutor.java | 20 +- .../nodemanager/LinuxContainerExecutor.java | 19 +- .../yarn/server/nodemanager/NodeManager.java | 13 +- .../recovery/NMLeveldbStateStoreService.java | 195 ++++- .../recovery/NMNullStateStoreService.java | 28 + .../recovery/NMStateStoreService.java | 35 + .../recovery/records/NMDBSchemaVersion.java | 80 ++ .../impl/pb/NMDBSchemaVersionPBImpl.java | 81 ++ .../security/NMTokenSecretManagerInNM.java | 116 ++- .../server/nodemanager/webapp/NodePage.java | 2 +- .../yarn_server_nodemanager_recovery.proto | 6 + .../TestDefaultContainerExecutor.java | 121 ++- .../TestLinuxContainerExecutorWithMocks.java | 41 +- .../recovery/NMMemoryStateStoreService.java | 49 ++ .../TestNMLeveldbStateStoreService.java | 114 +++ .../security/TestNMTokenSecretManagerInNM.java | 154 ++++ .../hadoop-yarn-server-resourcemanager/pom.xml | 15 + .../server/resourcemanager/ResourceManager.java | 4 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 26 +- .../rmapp/attempt/RMAppAttemptImpl.java | 32 +- .../rmcontainer/RMContainer.java | 5 + .../rmcontainer/RMContainerImpl.java | 28 +- .../scheduler/AbstractYarnScheduler.java | 39 + .../scheduler/AppSchedulingInfo.java | 69 +- .../scheduler/SchedulerApplicationAttempt.java | 9 +- .../scheduler/capacity/CapacityScheduler.java | 47 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 10 +- .../scheduler/event/AppAddedSchedulerEvent.java | 10 + .../event/AppAttemptAddedSchedulerEvent.java | 12 +- .../scheduler/fair/FSSchedulerApp.java | 10 +- .../scheduler/fair/FairScheduler.java | 114 ++- .../fair/policies/FairSharePolicy.java | 6 +- .../scheduler/fifo/FifoScheduler.java | 48 +- .../security/AMRMTokenSecretManager.java | 232 ++++-- .../RMDelegationTokenSecretManager.java | 12 + .../resourcemanager/webapp/RMWebServices.java | 279 ++++++- .../webapp/dao/DelegationToken.java | 99 +++ .../resourcemanager/TestApplicationCleanup.java | 101 ++- .../resourcemanager/TestFifoScheduler.java | 4 +- .../server/resourcemanager/TestRMRestart.java | 9 +- .../TestWorkPreservingRMRestart.java | 30 + .../recovery/RMStateStoreTestBase.java | 14 +- .../attempt/TestRMAppAttemptTransitions.java | 20 +- .../rmcontainer/TestRMContainerImpl.java | 42 + .../capacity/TestCapacityScheduler.java | 67 ++ .../scheduler/fair/FairSchedulerTestBase.java | 25 +- .../scheduler/fair/TestFairScheduler.java | 146 +++- .../security/TestAMRMTokens.java | 81 +- .../TestRMWebServicesDelegationTokens.java | 775 +++++++++++++++++++ .../src/site/apt/ResourceManagerRest.apt.vm | 220 ++++++ .../src/site/apt/YarnCommands.apt.vm | 9 +- 249 files changed, 9962 insertions(+), 1859 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/341695e7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java ----------------------------------------------------------------------
