YARN-2913. Fair scheduler should have ability to set MaxResourceDefault for each queue. (Siqi Li via mingma)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/934d96a3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/934d96a3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/934d96a3 Branch: refs/heads/HADOOP-11890 Commit: 934d96a334598fcf0e5aba2043ff539469025f69 Parents: f8adeb7 Author: Ming Ma <min...@apache.org> Authored: Fri Oct 23 08:36:33 2015 -0700 Committer: Ming Ma <min...@apache.org> Committed: Fri Oct 23 08:36:33 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../scheduler/fair/AllocationConfiguration.java | 26 +++++++++++++++++--- .../fair/AllocationFileLoaderService.java | 15 ++++++++--- .../fair/TestAllocationFileLoaderService.java | 18 +++++++++++++- .../src/site/markdown/FairScheduler.md | 3 +++ 5 files changed, 56 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/934d96a3/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 53bf85a..125ff94 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -531,6 +531,9 @@ Release 2.8.0 - UNRELEASED YARN-4243. Add retry on establishing Zookeeper conenction in EmbeddedElectorService#serviceInit. (Xuan Gong via junping_du) + YARN-2913. Fair scheduler should have ability to set MaxResourceDefault for + each queue. (Siqi Li via mingma) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/934d96a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index 0ea7314..bf4eae8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -29,6 +29,8 @@ import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; import com.google.common.annotations.VisibleForTesting; @@ -36,7 +38,8 @@ import com.google.common.annotations.VisibleForTesting; public class AllocationConfiguration extends ReservationSchedulerConfiguration { private static final AccessControlList EVERYBODY_ACL = new AccessControlList("*"); private static final AccessControlList NOBODY_ACL = new AccessControlList(" "); - + private static final ResourceCalculator RESOURCE_CALCULATOR = + new DefaultResourceCalculator(); // Minimum resource allocation for each queue private final Map<String, Resource> minQueueResources; // Maximum amount of resources per queue @@ -53,6 +56,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { final Map<String, Integer> userMaxApps; private final int userMaxAppsDefault; private final int queueMaxAppsDefault; + private final Resource queueMaxResourcesDefault; // Maximum resource share for each leaf queue that can be used to run AMs final Map<String, Float> queueMaxAMShares; @@ -99,7 +103,8 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { Map<String, Integer> queueMaxApps, Map<String, Integer> userMaxApps, Map<String, ResourceWeights> queueWeights, Map<String, Float> queueMaxAMShares, int userMaxAppsDefault, - int queueMaxAppsDefault, float queueMaxAMShareDefault, + int queueMaxAppsDefault, Resource queueMaxResourcesDefault, + float queueMaxAMShareDefault, Map<String, SchedulingPolicy> schedulingPolicies, SchedulingPolicy defaultSchedulingPolicy, Map<String, Long> minSharePreemptionTimeouts, @@ -117,6 +122,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { this.queueMaxAMShares = queueMaxAMShares; this.queueWeights = queueWeights; this.userMaxAppsDefault = userMaxAppsDefault; + this.queueMaxResourcesDefault = queueMaxResourcesDefault; this.queueMaxAppsDefault = queueMaxAppsDefault; this.queueMaxAMShareDefault = queueMaxAMShareDefault; this.defaultSchedulingPolicy = defaultSchedulingPolicy; @@ -140,6 +146,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { queueMaxAMShares = new HashMap<String, Float>(); userMaxAppsDefault = Integer.MAX_VALUE; queueMaxAppsDefault = Integer.MAX_VALUE; + queueMaxResourcesDefault = Resources.unbounded(); queueMaxAMShareDefault = 0.5f; queueAcls = new HashMap<String, Map<QueueACL, AccessControlList>>(); minSharePreemptionTimeouts = new HashMap<String, Long>(); @@ -243,7 +250,18 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { public Resource getMaxResources(String queueName) { Resource maxQueueResource = maxQueueResources.get(queueName); - return (maxQueueResource == null) ? Resources.unbounded() : maxQueueResource; + if (maxQueueResource == null) { + Resource minQueueResource = minQueueResources.get(queueName); + if (minQueueResource != null && + Resources.greaterThan(RESOURCE_CALCULATOR, Resources.unbounded(), + minQueueResource, queueMaxResourcesDefault)) { + return minQueueResource; + } else { + return queueMaxResourcesDefault; + } + } else { + return maxQueueResource; + } } public boolean hasAccess(String queueName, QueueACL acl, @@ -337,4 +355,4 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration { public void setAverageCapacity(int avgCapacity) { globalReservationQueueConfig.setAverageCapacity(avgCapacity); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/934d96a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index dab6d9f..9049525 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -202,7 +202,8 @@ public class AllocationFileLoaderService extends AbstractService { * @throws SAXException if config file is malformed. */ public synchronized void reloadAllocations() throws IOException, - ParserConfigurationException, SAXException, AllocationConfigurationException { + ParserConfigurationException, SAXException, + AllocationConfigurationException { if (allocFile == null) { return; } @@ -225,6 +226,7 @@ public class AllocationFileLoaderService extends AbstractService { Set<String> reservableQueues = new HashSet<String>(); int userMaxAppsDefault = Integer.MAX_VALUE; int queueMaxAppsDefault = Integer.MAX_VALUE; + Resource queueMaxResourcesDefault = Resources.unbounded(); float queueMaxAMShareDefault = 0.5f; long defaultFairSharePreemptionTimeout = Long.MAX_VALUE; long defaultMinSharePreemptionTimeout = Long.MAX_VALUE; @@ -282,6 +284,11 @@ public class AllocationFileLoaderService extends AbstractService { userMaxApps.put(userName, val); } } + } else if ("queueMaxResourcesDefault".equals(element.getTagName())) { + String text = ((Text)element.getFirstChild()).getData().trim(); + Resource val = + FairSchedulerConfiguration.parseResourceConfigValue(text); + queueMaxResourcesDefault = val; } else if ("userMaxAppsDefault".equals(element.getTagName())) { String text = ((Text)element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); @@ -398,9 +405,9 @@ public class AllocationFileLoaderService extends AbstractService { AllocationConfiguration info = new AllocationConfiguration(minQueueResources, maxQueueResources, queueMaxApps, userMaxApps, queueWeights, queueMaxAMShares, userMaxAppsDefault, queueMaxAppsDefault, - queueMaxAMShareDefault, queuePolicies, defaultSchedPolicy, - minSharePreemptionTimeouts, fairSharePreemptionTimeouts, - fairSharePreemptionThresholds, queueAcls, + queueMaxResourcesDefault, queueMaxAMShareDefault, queuePolicies, + defaultSchedPolicy, minSharePreemptionTimeouts, + fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls, newPlacementPolicy, configuredQueues, globalReservationQueueConfig, reservableQueues); http://git-wip-us.apache.org/repos/asf/hadoop/blob/934d96a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index 202eb09..7a909ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -153,15 +153,18 @@ public class TestAllocationFileLoaderService { // Give queue A a minimum of 1024 M out.println("<queue name=\"queueA\">"); out.println("<minResources>1024mb,0vcores</minResources>"); + out.println("<maxResources>2048mb,10vcores</maxResources>"); out.println("</queue>"); // Give queue B a minimum of 2048 M out.println("<queue name=\"queueB\">"); out.println("<minResources>2048mb,0vcores</minResources>"); + out.println("<maxResources>5120mb,110vcores</maxResources>"); out.println("<aclAdministerApps>alice,bob admins</aclAdministerApps>"); out.println("<schedulingPolicy>fair</schedulingPolicy>"); out.println("</queue>"); // Give queue C no minimum out.println("<queue name=\"queueC\">"); + out.println("<minResources>5120mb,0vcores</minResources>"); out.println("<aclSubmitApps>alice,bob admins</aclSubmitApps>"); out.println("</queue>"); // Give queue D a limit of 3 running apps and 0.4f maxAMShare @@ -190,6 +193,8 @@ public class TestAllocationFileLoaderService { out.println("</queue>"); // Set default limit of apps per queue to 15 out.println("<queueMaxAppsDefault>15</queueMaxAppsDefault>"); + // Set default limit of max resource per queue to 4G and 100 cores + out.println("<queueMaxResourcesDefault>4096mb,100vcores</queueMaxResourcesDefault>"); // Set default limit of apps per user to 5 out.println("<userMaxAppsDefault>5</userMaxAppsDefault>"); // Set default limit of AMResourceShare to 0.5f @@ -222,11 +227,22 @@ public class TestAllocationFileLoaderService { assertEquals(Resources.createResource(0), queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME)); + assertEquals(Resources.createResource(2048, 10), + queueConf.getMaxResources("root.queueA")); + assertEquals(Resources.createResource(5120, 110), + queueConf.getMaxResources("root.queueB")); + assertEquals(Resources.createResource(5120, 0), + queueConf.getMaxResources("root.queueC")); + assertEquals(Resources.createResource(4096, 100), + queueConf.getMaxResources("root.queueD")); + assertEquals(Resources.createResource(4096, 100), + queueConf.getMaxResources("root.queueE")); + assertEquals(Resources.createResource(1024, 0), queueConf.getMinResources("root.queueA")); assertEquals(Resources.createResource(2048, 0), queueConf.getMinResources("root.queueB")); - assertEquals(Resources.createResource(0), + assertEquals(Resources.createResource(5120, 0), queueConf.getMinResources("root.queueC")); assertEquals(Resources.createResource(0), queueConf.getMinResources("root.queueD")); http://git-wip-us.apache.org/repos/asf/hadoop/blob/934d96a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md index a58b3d3..26c6d22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md @@ -128,6 +128,8 @@ The allocation file must be in XML format. The format contains five types of ele * **A queueMaxAppsDefault element**: which sets the default running app limit for queues; overriden by maxRunningApps element in each queue. +* **A queueMaxResourcesDefault element**: which sets the default max resource limit for queue; overriden by maxResources element in each queue. + * **A queueMaxAMShareDefault element**: which sets the default AM resource limit for queue; overriden by maxAMShare element in each queue. * **A defaultQueueSchedulingPolicy element**: which sets the default scheduling policy for queues; overriden by the schedulingPolicy element in each queue if specified. Defaults to "fair". @@ -167,6 +169,7 @@ The allocation file must be in XML format. The format contains five types of ele </queue> <queueMaxAMShareDefault>0.5</queueMaxAMShareDefault> + <queueMaxResourcesDefault>40000 mb,0vcores</queueMaxResourcesDefault> <!-- Queue 'secondary_group_queue' is a parent queue and may have user queues under it -->
