Repository: hadoop Updated Branches: refs/heads/trunk 60971b819 -> 042ef2fa7
YARN-7789. Should fail RM if 3rd resource type is configured but RM uses DefaultResourceCalculator. (Zian Chen via wangda) Change-Id: I1f6a451fe16758def6f47c046a9b8a67ad7255af Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/042ef2fa Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/042ef2fa Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/042ef2fa Branch: refs/heads/trunk Commit: 042ef2fa7bcc22e3ca4eb8205c34d83e594bc7de Parents: 60971b8 Author: Wangda Tan <wan...@apache.org> Authored: Wed Feb 14 23:11:10 2018 +0800 Committer: Wangda Tan <wan...@apache.org> Committed: Wed Feb 14 23:11:10 2018 +0800 ---------------------------------------------------------------------- .../scheduler/capacity/CapacityScheduler.java | 8 +++++ ...CapacitySchedulerWithMultiResourceTypes.java | 37 ++++++++++++++++++++ 2 files changed, 45 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/042ef2fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index d3aa5cb..cd9d1373 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -338,6 +338,14 @@ public class CapacityScheduler extends this.minimumAllocation = super.getMinimumAllocation(); initMaximumResourceCapability(super.getMaximumAllocation()); this.calculator = this.conf.getResourceCalculator(); + if (this.calculator instanceof DefaultResourceCalculator + && ResourceUtils.getNumberOfKnownResourceTypes() > 2) { + throw new YarnRuntimeException("RM uses DefaultResourceCalculator which" + + " used only memory as resource-type but invalid resource-types" + + " specified " + ResourceUtils.getResourceTypes() + ". Use" + + " DomainantResourceCalculator instead to make effective use of" + + " these resource-types"); + } this.usePortForNodeName = this.conf.getUsePortForNodeName(); this.applications = new ConcurrentHashMap<>(); this.labelManager = rmContext.getNodeLabelManager(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/042ef2fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java index 2da2cdd..ea29f7f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java @@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; @@ -38,6 +39,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.junit.Assert; @@ -286,4 +288,39 @@ public class TestCapacitySchedulerWithMultiResourceTypes { rm.close(); } + + @Test + public void testDefaultResourceCalculatorWithThirdResourceTypes() throws Exception { + + CapacitySchedulerConfiguration csconf = + new CapacitySchedulerConfiguration(); + csconf.setResourceComparator(DefaultResourceCalculator.class); + + YarnConfiguration conf = new YarnConfiguration(csconf); + + String[] res1 = {"resource1", "M"}; + String[] res2 = {"resource2", "G"}; + String[] res3 = {"resource3", "H"}; + + String[][] test = {res1, res2, res3}; + + String resSt = ""; + for (String[] resources : test) { + resSt += (resources[0] + ","); + } + resSt = resSt.substring(0, resSt.length() - 1); + conf.set(YarnConfiguration.RESOURCE_TYPES, resSt); + + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + + boolean exception = false; + try { + MockRM rm = new MockRM(conf); + } catch (YarnRuntimeException e) { + exception = true; + } + + Assert.assertTrue("Should have exception in CS", exception); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
