Merge branch 'trunk' into HDFS-7240
Conflicts:
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
hadoop-hdfs-project/hadoop-hdfs/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4675655
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4675655
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4675655
Branch: refs/heads/HDFS-7240
Commit: f46756559acc2cf2249d8c5209af4343d62a4271
Parents: 7c88653 78af6cd
Author: Anu Engineer <[email protected]>
Authored: Tue Oct 10 15:35:26 2017 -0700
Committer: Anu Engineer <[email protected]>
Committed: Tue Oct 10 15:35:26 2017 -0700
----------------------------------------------------------------------
.../crypto/key/kms/KMSClientProvider.java | 39 +-
.../key/kms/LoadBalancingKMSClientProvider.java | 81 +-
.../fs/CommonConfigurationKeysPublic.java | 29 +
.../java/org/apache/hadoop/fs/FileStatus.java | 27 +-
.../apache/hadoop/fs/protocolPB/PBHelper.java | 14 +-
.../org/apache/hadoop/fs/viewfs/InodeTree.java | 3 +-
.../io/serializer/SerializationFactory.java | 18 +-
.../org/apache/hadoop/ipc/FairCallQueue.java | 8 +-
.../hadoop/security/http/CrossOriginFilter.java | 27 +-
.../DelegationTokenAuthenticationHandler.java | 5 +-
.../web/DelegationTokenAuthenticator.java | 1 +
.../delegation/web/DelegationTokenManager.java | 15 +-
.../hadoop/io/erasurecode/jni_xor_decoder.c | 3 +-
.../hadoop-common/src/main/proto/FSProtos.proto | 7 +-
.../services/org.apache.hadoop.fs.FileSystem | 1 -
.../src/main/resources/core-default.xml | 46 +-
.../src/site/markdown/HttpAuthentication.md | 2 +-
.../release/3.0.0-beta1/CHANGES.3.0.0-beta1.md | 646 ++++++
.../3.0.0-beta1/RELEASENOTES.3.0.0-beta1.md | 377 ++++
.../conf/TestCommonConfigurationFields.java | 4 +
.../kms/TestLoadBalancingKMSClientProvider.java | 313 ++-
.../io/serializer/TestSerializationFactory.java | 13 +
.../security/http/TestCrossOriginFilter.java | 79 +
...tionTokenAuthenticationHandlerWithMocks.java | 72 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 6 +-
.../org/apache/hadoop/nfs/nfs3/FileHandle.java | 51 +-
.../hadoop/nfs/nfs3/request/WRITE3Request.java | 4 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 7 +-
.../hadoop/hdfs/protocol/HdfsFileStatus.java | 12 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 5 +
.../src/main/proto/hdfs.proto | 1 +
.../hadoop/fs/http/client/HttpFSFileSystem.java | 17 +-
.../src/main/native/libhdfs/jni_helper.c | 12 +-
.../hadoop/hdfs/nfs/mount/RpcProgramMountd.java | 81 +-
.../hadoop/hdfs/nfs/nfs3/DFSClientCache.java | 174 +-
.../apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java | 46 +
.../hadoop/hdfs/nfs/nfs3/OpenFileCtx.java | 19 +-
.../hadoop/hdfs/nfs/nfs3/OpenFileCtxCache.java | 6 +-
.../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 362 ++--
.../apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java | 8 +-
.../hadoop/hdfs/nfs/nfs3/WriteManager.java | 24 +-
.../nfs/nfs3/TestClientAccessPrivilege.java | 3 +-
.../hdfs/nfs/nfs3/TestDFSClientCache.java | 13 +-
.../hadoop/hdfs/nfs/nfs3/TestExportsTable.java | 161 +-
.../hadoop/hdfs/nfs/nfs3/TestReaddir.java | 19 +-
.../hdfs/nfs/nfs3/TestRpcProgramNfs3.java | 66 +-
.../hdfs/nfs/nfs3/TestViewfsWithNfs3.java | 330 +++
.../apache/hadoop/hdfs/nfs/nfs3/TestWrites.java | 9 +-
.../dev-support/findbugsExcludeFile.xml | 3 +
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 10 +
.../hadoop-hdfs/src/main/bin/hdfs | 9 +
.../hadoop-hdfs/src/main/bin/hdfs.cmd | 14 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 162 ++
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 38 +
...tNamenodeProtocolServerSideTranslatorPB.java | 1 +
.../hdfs/protocolPB/RouterAdminProtocolPB.java | 44 +
...uterAdminProtocolServerSideTranslatorPB.java | 151 ++
.../RouterAdminProtocolTranslatorPB.java | 150 ++
.../server/blockmanagement/BlockManager.java | 10 +
.../blockmanagement/BlockPlacementPolicy.java | 2 +-
.../BlockPlacementPolicyRackFaultTolerant.java | 49 +-
.../hdfs/server/blockmanagement/BlocksMap.java | 40 +-
.../datanode/web/webhdfs/ParameterParser.java | 3 +-
.../federation/metrics/FederationMBean.java | 211 ++
.../federation/metrics/FederationMetrics.java | 692 ++++++
.../federation/metrics/FederationRPCMBean.java | 90 +
.../metrics/FederationRPCMetrics.java | 239 ++
.../FederationRPCPerformanceMonitor.java | 211 ++
.../federation/metrics/NamenodeBeanMetrics.java | 624 ++++++
.../federation/metrics/StateStoreMBean.java | 45 +
.../federation/metrics/StateStoreMetrics.java | 144 ++
.../server/federation/metrics/package-info.java | 27 +
.../resolver/ActiveNamenodeResolver.java | 117 +
.../resolver/FederationNamenodeContext.java | 87 +
.../FederationNamenodeServiceState.java | 46 +
.../resolver/FederationNamespaceInfo.java | 73 +
.../resolver/FileSubclusterResolver.java | 75 +
.../resolver/MembershipNamenodeResolver.java | 317 +++
.../federation/resolver/MountTableManager.java | 80 +
.../federation/resolver/MountTableResolver.java | 544 +++++
.../resolver/NamenodePriorityComparator.java | 63 +
.../resolver/NamenodeStatusReport.java | 388 ++++
.../federation/resolver/PathLocation.java | 212 ++
.../federation/resolver/RemoteLocation.java | 76 +
.../resolver/order/DestinationOrder.java | 29 +
.../federation/resolver/order/package-info.java | 29 +
.../federation/resolver/package-info.java | 41 +
.../federation/router/ConnectionContext.java | 104 +
.../federation/router/ConnectionManager.java | 431 ++++
.../federation/router/ConnectionPool.java | 337 +++
.../federation/router/ConnectionPoolId.java | 117 +
.../server/federation/router/DFSRouter.java | 76 +
.../federation/router/FederationUtil.java | 169 ++
.../router/NamenodeHeartbeatService.java | 361 +++
.../federation/router/PeriodicService.java | 198 ++
.../router/RemoteLocationContext.java | 68 +
.../server/federation/router/RemoteMethod.java | 164 ++
.../server/federation/router/RemoteParam.java | 71 +
.../hdfs/server/federation/router/Router.java | 527 +++++
.../federation/router/RouterAdminServer.java | 183 ++
.../server/federation/router/RouterClient.java | 76 +
.../federation/router/RouterHttpServer.java | 124 ++
.../server/federation/router/RouterMetrics.java | 73 +
.../federation/router/RouterMetricsService.java | 108 +
.../federation/router/RouterRpcClient.java | 932 ++++++++
.../federation/router/RouterRpcMonitor.java | 95 +
.../federation/router/RouterRpcServer.java | 2046 ++++++++++++++++++
.../server/federation/router/package-info.java | 31 +
.../federation/store/CachedRecordStore.java | 245 +++
.../federation/store/MembershipStore.java | 126 ++
.../federation/store/MountTableStore.java | 49 +
.../server/federation/store/RecordStore.java | 100 +
.../federation/store/StateStoreCache.java | 36 +
.../store/StateStoreCacheUpdateService.java | 67 +
.../StateStoreConnectionMonitorService.java | 67 +
.../federation/store/StateStoreService.java | 441 ++++
.../store/StateStoreUnavailableException.java | 33 +
.../federation/store/StateStoreUtils.java | 109 +
.../store/driver/StateStoreDriver.java | 202 ++
.../driver/StateStoreRecordOperations.java | 163 ++
.../store/driver/StateStoreSerializer.java | 119 +
.../store/driver/impl/StateStoreBaseImpl.java | 86 +
.../driver/impl/StateStoreFileBaseImpl.java | 429 ++++
.../store/driver/impl/StateStoreFileImpl.java | 161 ++
.../driver/impl/StateStoreFileSystemImpl.java | 178 ++
.../driver/impl/StateStoreSerializableImpl.java | 98 +
.../driver/impl/StateStoreSerializerPBImpl.java | 115 +
.../driver/impl/StateStoreZooKeeperImpl.java | 324 +++
.../store/driver/impl/package-info.java | 39 +
.../federation/store/driver/package-info.java | 37 +
.../store/impl/MembershipStoreImpl.java | 311 +++
.../store/impl/MountTableStoreImpl.java | 116 +
.../federation/store/impl/package-info.java | 31 +
.../server/federation/store/package-info.java | 62 +
.../protocol/AddMountTableEntryRequest.java | 47 +
.../protocol/AddMountTableEntryResponse.java | 42 +
.../protocol/GetMountTableEntriesRequest.java | 49 +
.../protocol/GetMountTableEntriesResponse.java | 53 +
.../GetNamenodeRegistrationsRequest.java | 52 +
.../GetNamenodeRegistrationsResponse.java | 55 +
.../store/protocol/GetNamespaceInfoRequest.java | 30 +
.../protocol/GetNamespaceInfoResponse.java | 52 +
.../protocol/NamenodeHeartbeatRequest.java | 52 +
.../protocol/NamenodeHeartbeatResponse.java | 49 +
.../protocol/RemoveMountTableEntryRequest.java | 49 +
.../protocol/RemoveMountTableEntryResponse.java | 42 +
.../protocol/UpdateMountTableEntryRequest.java | 51 +
.../protocol/UpdateMountTableEntryResponse.java | 43 +
.../UpdateNamenodeRegistrationRequest.java | 72 +
.../UpdateNamenodeRegistrationResponse.java | 51 +
.../pb/AddMountTableEntryRequestPBImpl.java | 84 +
.../pb/AddMountTableEntryResponsePBImpl.java | 76 +
.../impl/pb/FederationProtocolPBTranslator.java | 145 ++
.../pb/GetMountTableEntriesRequestPBImpl.java | 76 +
.../pb/GetMountTableEntriesResponsePBImpl.java | 104 +
.../GetNamenodeRegistrationsRequestPBImpl.java | 87 +
.../GetNamenodeRegistrationsResponsePBImpl.java | 99 +
.../impl/pb/GetNamespaceInfoRequestPBImpl.java | 60 +
.../impl/pb/GetNamespaceInfoResponsePBImpl.java | 95 +
.../impl/pb/NamenodeHeartbeatRequestPBImpl.java | 93 +
.../pb/NamenodeHeartbeatResponsePBImpl.java | 71 +
.../pb/RemoveMountTableEntryRequestPBImpl.java | 76 +
.../pb/RemoveMountTableEntryResponsePBImpl.java | 76 +
.../pb/UpdateMountTableEntryRequestPBImpl.java | 96 +
.../pb/UpdateMountTableEntryResponsePBImpl.java | 76 +
...UpdateNamenodeRegistrationRequestPBImpl.java | 95 +
...pdateNamenodeRegistrationResponsePBImpl.java | 73 +
.../store/protocol/impl/pb/package-info.java | 29 +
.../federation/store/protocol/package-info.java | 31 +
.../federation/store/records/BaseRecord.java | 207 ++
.../store/records/MembershipState.java | 329 +++
.../store/records/MembershipStats.java | 126 ++
.../federation/store/records/MountTable.java | 324 +++
.../server/federation/store/records/Query.java | 66 +
.../federation/store/records/QueryResult.java | 56 +
.../records/impl/pb/MembershipStatePBImpl.java | 335 +++
.../records/impl/pb/MembershipStatsPBImpl.java | 191 ++
.../store/records/impl/pb/MountTablePBImpl.java | 213 ++
.../store/records/impl/pb/PBRecord.java | 47 +
.../store/records/impl/pb/package-info.java | 29 +
.../federation/store/records/package-info.java | 36 +
.../server/namenode/FSDirStatAndListingOp.java | 7 +
.../hdfs/server/namenode/FSNamesystem.java | 12 +
.../hadoop/hdfs/server/namenode/NameNode.java | 16 +
.../namenode/NameNodeFormatException.java | 37 +
.../namenode/metrics/ECBlockGroupsMBean.java | 5 +
.../namenode/metrics/ReplicatedBlocksMBean.java | 5 +
.../hadoop/hdfs/tools/StoragePolicyAdmin.java | 61 +-
.../hdfs/tools/federation/RouterAdmin.java | 341 +++
.../hdfs/tools/federation/package-info.java | 28 +
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 4 +
.../src/main/proto/FederationProtocol.proto | 166 ++
.../src/main/proto/RouterProtocol.proto | 47 +
.../src/main/resources/hdfs-default.xml | 313 +++
.../main/webapps/router/federationhealth.html | 371 ++++
.../src/main/webapps/router/federationhealth.js | 313 +++
.../src/main/webapps/router/index.html | 24 +
.../src/site/markdown/HDFSCommands.md | 2 +-
.../src/site/markdown/HDFSRouterFederation.md | 311 +++
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 2 +
.../site/resources/images/routerfederation.png | Bin 0 -> 24961 bytes
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 8 +
.../hadoop/hdfs/TestDFSStripedInputStream.java | 13 +
.../hadoop/hdfs/TestDistributedFileSystem.java | 19 +
.../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 19 +-
.../hdfs/TestErasureCodingMultipleRacks.java | 107 +
.../hdfs/TestSafeModeWithStripedFile.java | 29 +-
...feModeWithStripedFileWithRandomECPolicy.java | 49 +
.../hdfs/server/datanode/DataNodeTestUtils.java | 32 +-
.../datanode/InternalDataNodeTestUtils.java | 27 +
.../server/datanode/TestBlockReplacement.java | 2 +-
.../server/federation/FederationTestUtils.java | 236 ++
.../hdfs/server/federation/MockResolver.java | 310 +++
.../server/federation/RouterConfigBuilder.java | 132 ++
.../server/federation/RouterDFSCluster.java | 1000 +++++++++
.../server/federation/StateStoreDFSCluster.java | 148 ++
.../metrics/TestFederationMetrics.java | 237 ++
.../federation/metrics/TestMetricsBase.java | 151 ++
.../resolver/TestMountTableResolver.java | 396 ++++
.../resolver/TestNamenodeResolver.java | 284 +++
.../router/TestNamenodeHeartbeat.java | 168 ++
.../server/federation/router/TestRouter.java | 150 ++
.../federation/router/TestRouterAdmin.java | 261 +++
.../server/federation/router/TestRouterRpc.java | 869 ++++++++
.../router/TestRouterRpcMultiDestination.java | 216 ++
.../store/FederationStateStoreTestUtils.java | 269 +++
.../federation/store/TestStateStoreBase.java | 81 +
.../store/TestStateStoreMembershipState.java | 463 ++++
.../store/TestStateStoreMountTable.java | 250 +++
.../store/driver/TestStateStoreDriverBase.java | 591 +++++
.../store/driver/TestStateStoreFile.java | 64 +
.../store/driver/TestStateStoreFileSystem.java | 88 +
.../store/driver/TestStateStoreZK.java | 105 +
.../store/records/TestMembershipState.java | 129 ++
.../store/records/TestMountTable.java | 176 ++
.../hadoop/hdfs/server/mover/TestMover.java | 6 +-
.../hdfs/server/namenode/TestClusterId.java | 34 +
.../server/namenode/TestNameNodeMXBean.java | 107 +-
.../hdfs/tools/TestStoragePolicyCommands.java | 11 +-
.../tools/TestViewFSStoragePolicyCommands.java | 80 +
.../tools/TestWebHDFSStoragePolicyCommands.java | 42 +
hadoop-project/pom.xml | 2 +-
hadoop-project/src/site/markdown/index.md.vm | 20 +-
hadoop-project/src/site/site.xml | 1 +
.../src/site/markdown/HadoopArchiveLogs.md | 6 +-
.../hadoop/fs/azure/NativeAzureFileSystem.java | 225 +-
.../hadoop-azure/src/site/markdown/index.md | 18 +-
.../fs/azure/ITestWasbRemoteCallHelper.java | 2 +-
.../TestNativeAzureFileSystemAuthorization.java | 451 +++-
.../ITestAzureFileSystemInstrumentation.java | 6 +-
.../src/main/data/2jobs2min-rumen-jh.json | 4 +-
.../hadoop/yarn/sls/appmaster/AMSimulator.java | 4 +-
.../src/site/markdown/SchedulerLoadSimulator.md | 20 +-
hadoop-yarn-project/hadoop-yarn/README | 2 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 4 +
hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 6 +
.../yarn/api/records/ApplicationReport.java | 2 +-
.../records/ApplicationSubmissionContext.java | 2 +-
.../yarn/api/records/ProfileCapability.java | 5 +-
.../yarn/api/records/ResourceRequest.java | 16 +
.../hadoop/yarn/conf/YarnConfiguration.java | 63 +-
.../yarn/exceptions/YarnRuntimeException.java | 2 +-
.../src/main/proto/yarn_service_protos.proto | 2 +-
.../hadoop/yarn/client/api/NMTokenCache.java | 2 +-
.../hadoop/yarn/client/api/YarnClient.java | 2 +-
.../hadoop/yarn/client/cli/SchedConfCLI.java | 254 +++
.../api/impl/TestDistributedScheduling.java | 649 ------
.../yarn/client/api/impl/TestNMClient.java | 265 ++-
.../TestOpportunisticContainerAllocation.java | 784 -------
...TestOpportunisticContainerAllocationE2E.java | 784 +++++++
.../yarn/client/cli/TestSchedConfCLI.java | 158 ++
.../src/test/resources/resource-profiles.json | 8 -
.../yarn/YarnUncaughtExceptionHandler.java | 2 +-
.../GetAllResourceProfilesResponsePBImpl.java | 6 +-
.../pb/GetResourceProfileResponsePBImpl.java | 9 +-
.../impl/pb/ProfileCapabilityPBImpl.java | 7 +-
.../java/org/apache/hadoop/yarn/util/Apps.java | 2 +-
.../hadoop/yarn/util/TrackingUriPlugin.java | 2 +-
.../hadoop/yarn/util/YarnVersionInfo.java | 12 +-
.../hadoop/yarn/webapp/dao/QueueConfigInfo.java | 57 +
.../yarn/webapp/dao/SchedConfUpdateInfo.java | 85 +
.../hadoop/yarn/webapp/dao/package-info.java | 27 +
.../yarn/webapp/util/YarnWebServiceUtils.java | 14 +
.../src/main/resources/yarn-default.xml | 70 +-
.../hadoop/yarn/api/TestProfileCapability.java | 109 +
.../registry/server/services/package-info.java | 2 +-
.../hadoop/yarn/server/api/ServerRMProxy.java | 7 +-
.../server/api/protocolrecords/RemoteNode.java | 35 +
.../impl/pb/RemoteNodePBImpl.java | 19 +
.../policies/FederationPolicyUtils.java | 4 +-
.../OpportunisticContainerAllocator.java | 353 ++-
.../OpportunisticContainerContext.java | 68 +-
.../BaseContainerTokenSecretManager.java | 24 +-
.../security/BaseNMTokenSecretManager.java | 26 +-
.../server/sharedcache/SharedCacheUtil.java | 12 +-
.../hadoop/yarn/server/utils/BuilderUtils.java | 1 +
.../yarn/server/webapp/AppAttemptBlock.java | 7 +-
.../hadoop/yarn/server/webapp/AppBlock.java | 6 +-
.../hadoop/yarn/server/webapp/AppsBlock.java | 6 +-
.../yarn/server/webapp/ContainerBlock.java | 7 +-
.../server/webapp/ErrorsAndWarningsBlock.java | 14 +-
.../yarn_server_common_service_protos.proto | 1 +
.../yarn/server/MockResourceManagerFacade.java | 2 +-
.../TestOpportunisticContainerAllocator.java | 599 +++++
.../nodemanager/DefaultContainerExecutor.java | 2 +-
.../containermanager/ContainerManagerImpl.java | 6 +-
.../launcher/ContainerLaunch.java | 22 +-
.../linux/resources/CGroupsHandlerImpl.java | 6 +-
.../TestLinuxContainerExecutorWithMocks.java | 3 -
.../linux/resources/TestCGroupsHandlerImpl.java | 6 +-
.../server/resourcemanager/AdminService.java | 24 +-
...pportunisticContainerAllocatorAMService.java | 8 +-
.../yarn/server/resourcemanager/RMContext.java | 2 +
.../server/resourcemanager/RMContextImpl.java | 37 +-
.../server/resourcemanager/ResourceManager.java | 35 +-
.../RMStateVersionIncompatibleException.java | 2 +-
.../recovery/ZKRMStateStore.java | 5 +-
.../reservation/InMemoryPlan.java | 6 +
.../PeriodicRLESparseResourceAllocation.java | 3 +-
.../RLESparseResourceAllocation.java | 6 +-
.../reservation/planning/IterativePlanner.java | 27 +-
.../reservation/planning/PlanningAlgorithm.java | 79 +-
.../reservation/planning/StageAllocator.java | 3 +-
.../planning/StageAllocatorGreedy.java | 4 +-
.../planning/StageAllocatorGreedyRLE.java | 7 +-
.../planning/StageAllocatorLowCostAligned.java | 6 +-
.../resource/ResourceProfilesManagerImpl.java | 41 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 20 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 23 +-
.../scheduler/AbstractYarnScheduler.java | 48 +-
.../ConfigurationMutationACLPolicy.java | 47 +
.../ConfigurationMutationACLPolicyFactory.java | 49 +
.../DefaultConfigurationMutationACLPolicy.java | 45 +
.../scheduler/MutableConfScheduler.java | 53 +
.../scheduler/MutableConfigurationProvider.java | 66 +
.../scheduler/SchedulerNode.java | 2 +-
.../scheduler/capacity/CapacityScheduler.java | 66 +-
.../capacity/CapacitySchedulerContext.java | 6 +
.../capacity/CapacitySchedulerQueueManager.java | 11 +-
.../capacity/conf/CSConfigurationProvider.java | 47 +
.../conf/FileBasedCSConfigurationProvider.java | 67 +
.../conf/InMemoryConfigurationStore.java | 95 +
.../conf/LeveldbConfigurationStore.java | 322 +++
.../conf/MutableCSConfigurationProvider.java | 284 +++
...ueueAdminConfigurationMutationACLPolicy.java | 110 +
.../capacity/conf/YarnConfigurationStore.java | 178 ++
.../conf/YarnConfigurationStoreFactory.java | 46 +
.../capacity/conf/ZKConfigurationStore.java | 235 ++
.../scheduler/capacity/conf/package-info.java | 29 +
.../scheduler/fair/AllocationConfiguration.java | 44 +-
.../fair/AllocationFileLoaderService.java | 47 +-
.../scheduler/fair/ConfigurableResource.java | 72 +
.../scheduler/fair/FSLeafQueue.java | 6 +-
.../scheduler/fair/FSParentQueue.java | 6 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 33 +-
.../scheduler/fair/FairScheduler.java | 3 +-
.../fair/FairSchedulerConfiguration.java | 54 +-
.../scheduler/fair/QueueManager.java | 3 +-
.../webapp/CapacitySchedulerPage.java | 3 +-
.../resourcemanager/webapp/RMWebServices.java | 66 +-
.../org/apache/hadoop/test/YarnTestDriver.java | 2 +-
.../yarn/server/resourcemanager/MockNM.java | 41 +-
.../yarn/server/resourcemanager/MockRM.java | 9 +
.../TestApplicationMasterService.java | 35 -
.../resourcemanager/TestClientRMService.java | 8 +
...pportunisticContainerAllocatorAMService.java | 2 +
.../resourcemanager/TestRMAdminService.java | 23 +
.../reservation/BaseSharingPolicyTest.java | 8 +-
.../reservation/TestCapacityOverTimePolicy.java | 29 +-
.../reservation/TestInMemoryPlan.java | 69 +-
.../planning/TestAlignedPlanner.java | 40 +-
.../planning/TestGreedyReservationAgent.java | 69 +-
.../planning/TestReservationAgents.java | 213 ++
.../resource/MockResourceProfileManager.java | 79 +
.../resource/TestResourceProfiles.java | 48 +-
.../attempt/TestRMAppAttemptTransitions.java | 20 +-
.../TestConfigurationMutationACLPolicies.java | 172 ++
.../capacity/TestCapacityScheduler.java | 4 +-
...CapacitySchedulerWithMultiResourceTypes.java | 141 ++
.../capacity/TestContainerAllocation.java | 6 +-
.../scheduler/capacity/TestUtils.java | 42 +-
.../conf/ConfigurationStoreBaseTest.java | 92 +
.../conf/TestInMemoryConfigurationStore.java | 30 +
.../conf/TestLeveldbConfigurationStore.java | 211 ++
.../TestMutableCSConfigurationProvider.java | 102 +
.../capacity/conf/TestZKConfigurationStore.java | 412 ++++
.../fair/TestAllocationFileLoaderService.java | 22 +-
.../fair/TestConfigurableResource.java | 68 +
.../scheduler/fair/TestFSLeafQueue.java | 2 +-
.../scheduler/fair/TestFSParentQueue.java | 3 +
.../scheduler/fair/TestFairScheduler.java | 15 +-
.../fair/TestFairSchedulerConfiguration.java | 81 +-
.../fair/TestMaxRunningAppsEnforcer.java | 5 +-
.../scheduler/fair/TestQueueManager.java | 5 +-
.../TestRMWebServicesConfigurationMutation.java | 507 +++++
.../webapp/dao/TestFairSchedulerQueueInfo.java | 3 +
.../resources/profiles/sample-profiles-1.json | 8 -
.../resources/profiles/sample-profiles-2.json | 8 -
.../clientrm/FederationClientInterceptor.java | 6 +-
.../webapp/FederationInterceptorREST.java | 24 +-
.../router/webapp/RouterWebServiceUtil.java | 8 +-
.../MockDefaultRequestInterceptorREST.java | 2 +-
.../server/ContainerTokenIdentifierForTest.java | 6 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 11 +-
.../server/NMTokenIdentifierNewForTest.java | 6 +-
.../server/TestContainerManagerSecurity.java | 6 +-
.../hadoop/yarn/server/TestDiskFailures.java | 6 +-
.../common/HBaseTimelineStorageUtils.java | 2 +-
.../yarn/server/webproxy/AppReportFetcher.java | 3 -
.../server/webproxy/WebAppProxyServlet.java | 2 +-
.../webproxy/amfilter/AmFilterInitializer.java | 38 +-
.../server/webproxy/amfilter/AmIpFilter.java | 67 +-
.../server/webproxy/amfilter/TestAmFilter.java | 8 +-
.../amfilter/TestAmFilterInitializer.java | 14 +-
.../src/site/markdown/CapacityScheduler.md | 25 +-
.../src/site/markdown/FairScheduler.md | 4 +-
.../src/site/markdown/ResourceManagerRest.md | 188 ++
.../src/site/markdown/SharedCache.md | 168 ++
.../src/site/markdown/TimelineServer.md | 32 +-
.../src/site/markdown/YarnCommands.md | 13 +
.../hadoop-yarn/hadoop-yarn-ui/README.md | 6 +-
.../main/webapp/app/components/nodes-heatmap.js | 1 -
.../main/webapp/app/components/tree-selector.js | 4 +-
.../main/webapp/app/controllers/application.js | 9 +-
.../src/main/webapp/app/index.html | 2 +-
.../app/models/yarn-queue/capacity-queue.js | 8 +-
.../webapp/app/models/yarn-queue/fair-queue.js | 8 +-
.../webapp/app/models/yarn-queue/fifo-queue.js | 8 +-
.../app/serializers/yarn-queue/fair-queue.js | 5 +-
.../main/webapp/app/templates/application.hbs | 2 +-
.../yarn-queue/capacity-queue-conf-table.hbs | 4 +-
.../yarn-queue/fair-queue-conf-table.hbs | 2 +-
.../components/yarn-queue/fair-queue.hbs | 2 +-
.../yarn-queue/fifo-queue-conf-table.hbs | 4 +-
.../components/yarn-queue/fifo-queue.hbs | 2 +-
.../src/main/webapp/app/utils/converter.js | 6 +
.../src/main/webapp/config/default-config.js | 2 +-
437 files changed, 40413 insertions(+), 2986 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4675655/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4675655/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
----------------------------------------------------------------------
diff --cc
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
index adc195e,6677716..36f75dd
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
@@@ -99,8 -99,9 +99,12 @@@ public class TestCommonConfigurationFie
// S3A properties are in a different subtree.
xmlPrefixToSkipCompare.add("fs.s3a.");
+ // O3 properties are in a different subtree.
+ xmlPrefixToSkipCompare.add("fs.o3.");
++
+ //ftp properties are in a different subtree.
+ // - org.apache.hadoop.fs.ftp.FTPFileSystem.
+ xmlPrefixToSkipCompare.add("fs.ftp.impl");
// WASB properties are in a different subtree.
// - org.apache.hadoop.fs.azure.NativeAzureFileSystem
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4675655/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4675655/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4675655/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
index d5db8b3,4b958b5..c730d9d
--- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
@@@ -18,12 -15,9 +18,15 @@@
<Package name="org.apache.hadoop.hdfs.qjournal.protocol" />
</Match>
<Match>
+ <Package name="org.apache.hadoop.ozone.protocol.proto" />
+ </Match>
+ <Match>
+ <Package name ="org.apache.hadoop.cblock.protocol.proto" />
+ </Match>
+ <Match>
+ <Package name="org.apache.hadoop.hdfs.federation.protocol.proto" />
+ </Match>
+ <Match>
<Bug pattern="EI_EXPOSE_REP" />
</Match>
<Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4675655/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 72883da,65eea31..a437ab4
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@@ -204,27 -204,10 +204,32 @@@ http://maven.apache.org/xsd/maven-4.0.0
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
+ <groupId>org.jscsi</groupId>
+ <artifactId>target</artifactId>
+ <version>2.5.3</version>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.jctools</groupId>
+ <artifactId>jctools-core</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.xerial</groupId>
+ <artifactId>sqlite-jdbc</artifactId>
+ <version>3.8.7</version>
+ </dependency>
++ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
@@@ -354,9 -339,8 +362,11 @@@
<include>QJournalProtocol.proto</include>
<include>editlog.proto</include>
<include>fsimage.proto</include>
+ <include>StorageContainerDatanodeProtocol.proto</include>
+ <include>CBlockServiceProtocol.proto</include>
+ <include>CBlockClientServerProtocol.proto</include>
+ <include>FederationProtocol.proto</include>
+ <include>RouterProtocol.proto</include>
</includes>
</source>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4675655/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4675655/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4675655/hadoop-project/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4675655/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
