Merge remote-tracking branch 'origin/trunk' into HDFS-EC
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bdc01015 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bdc01015 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bdc01015 Branch: refs/heads/HDFS-EC Commit: bdc01015c1a1cac7f58baffbd9812bd1c770e375 Parents: 041e97d 7caa3bc Author: Vinayakumar B <vinayakuma...@intel.com> Authored: Wed Dec 3 10:51:14 2014 +0530 Committer: Vinayakumar B <vinayakuma...@intel.com> Committed: Wed Dec 3 10:51:14 2014 +0530 ---------------------------------------------------------------------- dev-support/smart-apply-patch.sh | 47 + hadoop-common-project/hadoop-common/CHANGES.txt | 54 +- .../hadoop-common/src/main/bin/hadoop | 5 + .../src/main/bin/hadoop-functions.sh | 6 + .../hadoop-common/src/main/bin/hadoop.cmd | 7 + .../hadoop-common/src/main/conf/hadoop-env.sh | 112 +- .../apache/hadoop/conf/ReconfigurableBase.java | 2 +- .../hadoop/crypto/key/JavaKeyStoreProvider.java | 3 + .../org/apache/hadoop/crypto/key/KeyShell.java | 14 +- .../org/apache/hadoop/http/HttpServer2.java | 2 +- .../apache/hadoop/io/WritableComparator.java | 1 + .../metrics2/impl/MetricsSourceAdapter.java | 5 +- .../hadoop/net/unix/DomainSocketWatcher.java | 15 + .../hadoop/security/ShellBasedIdMapping.java | 344 ++++- .../hadoop/security/UserGroupInformation.java | 17 +- .../ZKDelegationTokenSecretManager.java | 126 +- .../hadoop/util/ApplicationClassLoader.java | 11 +- .../src/site/apt/CommandsManual.apt.vm | 3 + .../hadoop-common/src/site/apt/Metrics.apt.vm | 139 ++ .../markdown/filesystem/fsdatainputstream.md | 62 +- .../apache/hadoop/conf/TestReconfiguration.java | 5 +- .../crypto/key/TestKeyProviderFactory.java | 11 + .../metrics2/impl/TestMetricsSourceAdapter.java | 51 + .../security/TestShellBasedIdMapping.java | 61 + .../TestZKDelegationTokenSecretManager.java | 275 +++- .../hadoop/util/TestApplicationClassLoader.java | 26 +- hadoop-common-project/hadoop-kms/pom.xml | 4 +- .../hadoop/crypto/key/kms/server/KMSACLs.java | 31 +- .../crypto/key/kms/server/KMSConfiguration.java | 1 + .../kms/server/KeyAuthorizationKeyProvider.java | 139 +- .../hadoop-kms/src/main/libexec/kms-config.sh | 12 +- .../hadoop-kms/src/main/sbin/kms.sh | 17 +- .../hadoop-kms/src/main/tomcat/ssl-server.xml | 135 -- .../src/main/tomcat/ssl-server.xml.conf | 136 ++ .../hadoop-kms/src/site/apt/index.apt.vm | 38 +- .../hadoop/crypto/key/kms/server/TestKMS.java | 121 +- .../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 3 +- .../hdfs/nfs/nfs3/TestRpcProgramNfs3.java | 2 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 109 +- .../dev-support/findbugsExcludeFile.xml | 9 + .../bkjournal/TestBookKeeperHACheckpoints.java | 2 +- .../hadoop-hdfs/src/main/bin/hdfs | 28 +- .../hadoop-hdfs/src/main/bin/hdfs-config.sh | 7 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 10 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 17 + .../hadoop/hdfs/DFSInotifyEventInputStream.java | 65 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 320 +++-- .../org/apache/hadoop/hdfs/DFSOutputStream.java | 6 +- .../apache/hadoop/hdfs/inotify/EventBatch.java | 41 + .../hadoop/hdfs/inotify/EventBatchList.java | 63 + .../apache/hadoop/hdfs/inotify/EventsList.java | 63 - .../hadoop/hdfs/protocol/ClientProtocol.java | 13 +- .../hdfs/protocol/LastBlockWithStatus.java | 46 + .../hadoop/hdfs/protocol/LocatedBlocks.java | 9 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 20 +- .../ClientNamenodeProtocolTranslatorPB.java | 15 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 341 ++--- .../hadoop/hdfs/server/balancer/Dispatcher.java | 17 +- .../hdfs/server/blockmanagement/BlockInfo.java | 5 +- .../server/blockmanagement/BlockManager.java | 57 +- .../hdfs/server/blockmanagement/BlocksMap.java | 6 +- .../server/blockmanagement/DatanodeManager.java | 30 +- .../blockmanagement/DatanodeStorageInfo.java | 14 +- .../hdfs/server/common/HdfsServerConstants.java | 7 +- .../hadoop/hdfs/server/common/JspHelper.java | 5 +- .../hadoop/hdfs/server/datanode/DataNode.java | 140 +- .../hdfs/server/datanode/DataNodeMXBean.java | 7 + .../hdfs/server/datanode/DataXceiver.java | 168 ++- .../server/datanode/SecureDataNodeStarter.java | 45 +- .../server/datanode/fsdataset/FsDatasetSpi.java | 7 + .../datanode/fsdataset/impl/BlockPoolSlice.java | 4 + .../datanode/fsdataset/impl/FsDatasetImpl.java | 134 +- .../datanode/fsdataset/impl/FsVolumeImpl.java | 4 + .../impl/RamDiskAsyncLazyPersistService.java | 2 +- .../server/datanode/web/DatanodeHttpServer.java | 174 +++ .../datanode/web/SimpleHttpProxyHandler.java | 161 +++ .../hdfs/server/datanode/web/URLDispatcher.java | 59 + .../web/resources/DatanodeWebHdfsMethods.java | 477 ------- .../datanode/web/resources/OpenEntity.java | 93 -- .../web/webhdfs/DataNodeUGIProvider.java | 70 + .../datanode/web/webhdfs/ExceptionHandler.java | 115 ++ .../server/datanode/web/webhdfs/HdfsWriter.java | 82 ++ .../datanode/web/webhdfs/ParameterParser.java | 125 ++ .../datanode/web/webhdfs/WebHdfsHandler.java | 256 ++++ .../apache/hadoop/hdfs/server/mover/Mover.java | 29 +- .../hdfs/server/namenode/FSDirConcatOp.java | 233 +++ .../hdfs/server/namenode/FSDirMkdirOp.java | 238 ++++ .../hdfs/server/namenode/FSDirRenameOp.java | 691 +++++++++ .../hdfs/server/namenode/FSDirSnapshotOp.java | 222 +++ .../server/namenode/FSDirStatAndListingOp.java | 480 +++++++ .../hdfs/server/namenode/FSDirectory.java | 1229 +++------------- .../hadoop/hdfs/server/namenode/FSEditLog.java | 4 +- .../hdfs/server/namenode/FSEditLogLoader.java | 29 +- .../hdfs/server/namenode/FSEditLogOp.java | 297 +++- .../hadoop/hdfs/server/namenode/FSImage.java | 5 - .../hdfs/server/namenode/FSImageFormat.java | 11 +- .../server/namenode/FSImageFormatPBINode.java | 4 +- .../server/namenode/FSImageSerialization.java | 2 +- .../hdfs/server/namenode/FSNamesystem.java | 1340 ++++-------------- .../hadoop/hdfs/server/namenode/INodeFile.java | 1 - .../namenode/InotifyFSEditLogOpTranslator.java | 74 +- .../hdfs/server/namenode/LeaseManager.java | 82 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 349 ++++- .../hdfs/server/namenode/ha/EditLogTailer.java | 12 +- .../server/namenode/ha/StandbyCheckpointer.java | 12 +- .../server/namenode/top/TopAuditLogger.java | 74 + .../hdfs/server/namenode/top/TopConf.java | 61 + .../server/namenode/top/metrics/TopMetrics.java | 265 ++++ .../namenode/top/window/RollingWindow.java | 189 +++ .../top/window/RollingWindowManager.java | 265 ++++ .../server/protocol/DatanodeRegistration.java | 1 + .../hdfs/shortcircuit/ShortCircuitCache.java | 19 +- .../hdfs/shortcircuit/ShortCircuitReplica.java | 12 +- .../hadoop/hdfs/web/resources/IntegerParam.java | 3 +- .../hadoop/hdfs/web/resources/LongParam.java | 3 +- .../hadoop/hdfs/web/resources/ShortParam.java | 3 +- .../src/main/proto/ClientNamenodeProtocol.proto | 1 + .../hadoop-hdfs/src/main/proto/inotify.proto | 10 +- .../src/main/resources/hdfs-default.xml | 28 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 26 +- .../hdfs/TestDFSInotifyEventInputStream.java | 209 +-- .../apache/hadoop/hdfs/TestEncryptionZones.java | 6 +- .../hadoop/hdfs/TestEncryptionZonesWithHA.java | 2 +- .../apache/hadoop/hdfs/TestRenameWhileOpen.java | 2 +- .../hadoop/hdfs/TestReservedRawPaths.java | 2 +- .../server/blockmanagement/TestBlockInfo.java | 4 +- .../TestComputeInvalidateWork.java | 167 ++- .../blockmanagement/TestDatanodeDescriptor.java | 8 +- .../blockmanagement/TestReplicationPolicy.java | 10 + .../server/datanode/SimulatedFSDataset.java | 7 + .../server/datanode/TestBlockReplacement.java | 101 +- .../server/datanode/TestDataNodeMetrics.java | 18 + .../resources/TestDatanodeWebHdfsMethods.java | 54 - .../web/webhdfs/TestParameterParser.java | 55 + .../hadoop/hdfs/server/mover/TestMover.java | 48 +- .../hdfs/server/namenode/NameNodeAdapter.java | 6 +- .../hdfs/server/namenode/TestAuditLogger.java | 29 +- .../hdfs/server/namenode/TestDeadDatanode.java | 32 +- .../namenode/TestDecommissioningStatus.java | 45 +- .../TestDefaultBlockPlacementPolicy.java | 22 +- .../hdfs/server/namenode/TestEditLog.java | 11 +- .../hdfs/server/namenode/TestEditLogRace.java | 43 +- .../namenode/TestFSPermissionChecker.java | 2 +- .../hdfs/server/namenode/TestFsLimits.java | 4 +- .../hdfs/server/namenode/TestINodeFile.java | 20 +- .../hdfs/server/namenode/TestLeaseManager.java | 25 + .../hdfs/server/namenode/TestMetaSave.java | 23 +- .../server/namenode/TestNameNodeMXBean.java | 4 +- .../server/namenode/TestNameNodeRecovery.java | 3 +- .../server/namenode/TestNamenodeRetryCache.java | 129 +- .../namenode/ha/TestEditLogsDuringFailover.java | 2 +- .../namenode/ha/TestRetryCacheWithHA.java | 3 +- .../namenode/ha/TestStandbyCheckpoints.java | 23 +- .../namenode/metrics/TestNameNodeMetrics.java | 61 + .../namenode/top/window/TestRollingWindow.java | 84 ++ .../top/window/TestRollingWindowManager.java | 93 ++ .../hdfs/web/TestFSMainOperationsWebHdfs.java | 2 - .../hdfs/web/TestWebHdfsFileSystemContract.java | 4 +- hadoop-mapreduce-project/CHANGES.txt | 25 +- hadoop-mapreduce-project/bin/mapred | 4 +- hadoop-mapreduce-project/conf/mapred-env.sh | 8 +- .../jobhistory/JobHistoryEventHandler.java | 13 +- .../hadoop/mapreduce/v2/app/MRAppMaster.java | 3 +- .../v2/app/client/MRClientService.java | 24 +- .../authorize/ClientHSPolicyProvider.java | 17 +- .../jobhistory/TestJobHistoryEventHandler.java | 2 +- .../mapreduce/v2/app/TestMRClientService.java | 16 + .../hadoop-mapreduce-client-common/pom.xml | 1 + .../mapreduce/v2/api/HSAdminProtocol.java | 34 + .../v2/api/HSAdminRefreshProtocol.java | 59 + .../v2/api/HSAdminRefreshProtocolPB.java | 33 + .../mapreduce/v2/jobhistory/JHAdminConfig.java | 4 +- .../src/main/proto/HSAdminRefreshProtocol.proto | 100 ++ .../java/org/apache/hadoop/mapred/Merger.java | 11 +- .../hadoop/mapreduce/lib/db/TestDbClasses.java | 8 +- .../mapreduce/task/reduce/TestMerger.java | 2 + .../hadoop-mapreduce-client-hs/pom.xml | 30 - .../hadoop/mapreduce/v2/hs/CompletedJob.java | 14 +- .../hadoop/mapreduce/v2/hs/HSProxies.java | 4 +- .../mapreduce/v2/hs/HistoryClientService.java | 22 +- .../hadoop/mapreduce/v2/hs/client/HSAdmin.java | 18 +- .../v2/hs/protocol/HSAdminProtocol.java | 38 - .../v2/hs/protocol/HSAdminRefreshProtocol.java | 63 - ...inRefreshProtocolClientSideTranslatorPB.java | 3 +- .../hs/protocolPB/HSAdminRefreshProtocolPB.java | 33 - ...inRefreshProtocolServerSideTranslatorPB.java | 3 +- .../mapreduce/v2/hs/server/HSAdminServer.java | 14 +- .../src/main/proto/HSAdminRefreshProtocol.proto | 100 -- .../mapreduce/v2/hs/TestJobHistoryEntities.java | 30 +- .../mapreduce/v2/hs/TestJobHistoryServer.java | 19 + .../v2/hs/server/TestHSAdminServer.java | 20 + ...33-2-0-SUCCEEDED-default-1416424779349.jhist | 29 + .../mapred/TestMRTimelineEventHandling.java | 58 +- .../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 6 +- .../org/apache/hadoop/tools/HadoopArchives.java | 17 +- .../apache/hadoop/tools/TestHadoopArchives.java | 77 +- hadoop-yarn-project/CHANGES.txt | 77 +- .../hadoop-yarn/bin/start-yarn.sh | 4 +- .../hadoop-yarn/bin/stop-yarn.sh | 4 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 14 +- .../hadoop-yarn/bin/yarn-daemons.sh | 6 +- .../hadoop-yarn/conf/yarn-env.sh | 19 +- .../hadoop-yarn/hadoop-yarn-api/pom.xml | 1 + .../hadoop/yarn/api/ClientSCMProtocol.java | 90 ++ .../hadoop/yarn/api/ClientSCMProtocolPB.java | 28 + .../ReleaseSharedCacheResourceRequest.java | 67 + .../ReleaseSharedCacheResourceResponse.java | 37 + .../UseSharedCacheResourceRequest.java | 70 + .../UseSharedCacheResourceResponse.java | 55 + .../hadoop/yarn/conf/YarnConfiguration.java | 15 + .../src/main/proto/client_SCM_protocol.proto | 30 + .../src/main/proto/yarn_service_protos.proto | 21 + .../distributedshell/ApplicationMaster.java | 47 +- .../distributedshell/TestDistributedShell.java | 2 +- .../hadoop/yarn/client/ProtocolHATestBase.java | 6 +- .../client/ClientSCMProtocolPBClientImpl.java | 93 ++ .../service/ClientSCMProtocolPBServiceImpl.java | 78 + ...ReleaseSharedCacheResourceRequestPBImpl.java | 122 ++ ...eleaseSharedCacheResourceResponsePBImpl.java | 53 + .../pb/UseSharedCacheResourceRequestPBImpl.java | 120 ++ .../UseSharedCacheResourceResponsePBImpl.java | 79 ++ .../client/api/impl/TimelineClientImpl.java | 89 +- .../logaggregation/AggregatedLogFormat.java | 4 + .../server/security/ApplicationACLsManager.java | 7 + .../yarn/webapp/log/AggregatedLogsBlock.java | 10 +- .../src/main/resources/yarn-default.xml | 34 +- .../client/api/impl/TestTimelineClient.java | 58 +- .../ApplicationHistoryClientService.java | 8 + .../server/timeline/LeveldbTimelineStore.java | 233 +-- .../TestApplicationHistoryServer.java | 14 + .../timeline/TestLeveldbTimelineStore.java | 81 +- .../container/ContainerImpl.java | 7 + .../nodemanager/metrics/NodeManagerMetrics.java | 9 +- .../metrics/TestNodeManagerMetrics.java | 1 + .../hadoop-yarn-server-resourcemanager/pom.xml | 4 + .../server/resourcemanager/ClusterMetrics.java | 13 +- .../resourcemanager/RMActiveServiceContext.java | 455 ++++++ .../server/resourcemanager/RMAppManager.java | 13 +- .../server/resourcemanager/RMContextImpl.java | 235 ++- .../server/resourcemanager/ResourceManager.java | 57 +- .../recovery/FileSystemRMStateStore.java | 74 +- .../recovery/LeveldbRMStateStore.java | 691 +++++++++ .../recovery/MemoryRMStateStore.java | 86 +- .../recovery/NullRMStateStore.java | 2 +- .../resourcemanager/recovery/RMStateStore.java | 294 ++-- .../recovery/RMStateStoreAppAttemptEvent.java | 8 +- .../recovery/RMStateStoreAppEvent.java | 8 +- .../recovery/RMStateStoreEventType.java | 3 +- .../recovery/RMStateStoreRemoveAppEvent.java | 8 +- .../recovery/RMStateUpdateAppAttemptEvent.java | 9 +- .../recovery/RMStateUpdateAppEvent.java | 8 +- .../recovery/ZKRMStateStore.java | 54 +- .../records/ApplicationAttemptStateData.java | 36 +- .../recovery/records/ApplicationStateData.java | 25 +- .../pb/ApplicationAttemptStateDataPBImpl.java | 60 +- .../reservation/AbstractReservationSystem.java | 103 +- .../reservation/CapacityOverTimePolicy.java | 14 +- .../reservation/CapacityReservationSystem.java | 113 +- .../reservation/NoOverCommitPolicy.java | 4 +- .../resourcemanager/reservation/Planner.java | 3 +- .../ReservationSchedulerConfiguration.java | 159 +++ .../reservation/SharingPolicy.java | 3 +- .../reservation/SimpleCapacityReplanner.java | 14 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 16 +- .../rmapp/attempt/RMAppAttemptImpl.java | 40 +- .../scheduler/AbstractYarnScheduler.java | 104 +- .../scheduler/SchedulerNode.java | 5 +- .../scheduler/capacity/CapacityScheduler.java | 15 +- .../CapacitySchedulerConfiguration.java | 37 +- .../fair/AllocationFileLoaderService.java | 6 + .../resourcemanager/scheduler/fair/FSQueue.java | 22 +- .../scheduler/fair/FairScheduler.java | 21 +- .../scheduler/fair/QueuePlacementRule.java | 30 +- .../scheduler/fifo/FifoScheduler.java | 16 +- .../security/QueueACLsManager.java | 7 + .../security/RMAuthenticationHandler.java | 157 -- .../server/resourcemanager/webapp/AppBlock.java | 50 +- .../resourcemanager/webapp/AppsBlock.java | 7 +- .../webapp/CapacitySchedulerPage.java | 4 +- .../webapp/DefaultSchedulerPage.java | 4 +- .../webapp/FairSchedulerAppsBlock.java | 9 +- .../webapp/MetricsOverviewTable.java | 9 +- .../resourcemanager/webapp/NodesPage.java | 9 +- .../server/resourcemanager/webapp/RMWebApp.java | 7 - .../resourcemanager/webapp/RMWebServices.java | 2 +- .../webapp/dao/ClusterMetricsInfo.java | 3 +- .../webapp/dao/UserMetricsInfo.java | 3 +- .../resourcemanager/TestClusterMetrics.java | 60 + .../resourcemanager/TestFifoScheduler.java | 1 + .../yarn/server/resourcemanager/TestRMHA.java | 66 +- .../server/resourcemanager/TestRMRestart.java | 84 +- .../TestWorkPreservingRMRestart.java | 8 + .../applicationsmanager/TestAMRestart.java | 8 +- .../recovery/RMStateStoreTestBase.java | 102 +- .../recovery/TestLeveldbRMStateStore.java | 139 ++ .../recovery/TestZKRMStateStore.java | 94 ++ .../reservation/ReservationSystemTestUtil.java | 91 +- .../reservation/TestCapacityOverTimePolicy.java | 21 +- .../TestCapacityReservationSystem.java | 33 +- .../reservation/TestGreedyReservationAgent.java | 30 +- .../reservation/TestNoOverCommitPolicy.java | 15 +- .../TestSimpleCapacityReplanner.java | 8 +- .../rmapp/TestRMAppTransitions.java | 31 +- .../attempt/TestRMAppAttemptTransitions.java | 4 +- .../scheduler/TestAbstractYarnScheduler.java | 213 +++ .../capacity/TestContainerAllocation.java | 4 +- .../scheduler/fair/PeriodGroupsMapping.java | 44 + .../fair/TestAllocationFileLoaderService.java | 24 + .../scheduler/fair/TestFairScheduler.java | 132 +- .../fair/TestQueuePlacementPolicy.java | 49 +- .../resourcemanager/webapp/TestAppPage.java | 90 ++ .../webapp/TestRMWebServices.java | 8 - .../webapp/TestRMWebServicesApps.java | 7 - .../TestRMWebServicesAppsModification.java | 7 - .../webapp/TestRMWebServicesCapacitySched.java | 7 - .../TestRMWebServicesDelegationTokens.java | 7 - .../webapp/TestRMWebServicesFairScheduler.java | 7 - .../webapp/TestRMWebServicesNodeLabels.java | 3 - .../webapp/TestRMWebServicesNodes.java | 7 - .../ClientProtocolService.java | 192 +++ .../sharedcachemanager/SharedCacheManager.java | 7 + .../metrics/ClientSCMMetrics.java | 113 ++ .../TestClientSCMProtocolService.java | 278 ++++ .../hadoop/yarn/server/MiniYARNCluster.java | 24 +- .../src/site/apt/DockerContainerExecutor.apt.vm | 18 +- .../src/site/apt/FairScheduler.apt.vm | 14 +- .../src/site/apt/TimelineServer.apt.vm | 4 +- 327 files changed, 14868 insertions(+), 6138 deletions(-) ----------------------------------------------------------------------
