This is an automated email from the ASF dual-hosted git repository. rohithsharmaks pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push: new ab98f91 YARN-8917. Absolute (maximum) capacity of level3+ queues is wrongly calculated for absolute resource. Contributed by Tao Yang. ab98f91 is described below commit ab98f91638f38358cb71ace4d1570e01961c3f29 Author: Rohith Sharma K S <rohithsharm...@apache.org> AuthorDate: Mon Aug 26 21:06:15 2019 +0530 YARN-8917. Absolute (maximum) capacity of level3+ queues is wrongly calculated for absolute resource. Contributed by Tao Yang. --- .../scheduler/capacity/ParentQueue.java | 8 ++-- .../TestAbsoluteResourceConfiguration.java | 49 ++++++++++++++++++++++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 0aefa69..443079f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -1104,14 +1104,14 @@ public class ParentQueue extends AbstractCSQueue { // 3. Update absolute capacity as a float based on parent's minResource and // cluster resource. childQueue.getQueueCapacities().setAbsoluteCapacity(label, - (float) childQueue.getQueueCapacities().getCapacity() - / getQueueCapacities().getAbsoluteCapacity(label)); + childQueue.getQueueCapacities().getCapacity(label) + * getQueueCapacities().getAbsoluteCapacity(label)); // 4. Update absolute max-capacity as a float based on parent's maxResource // and cluster resource. childQueue.getQueueCapacities().setAbsoluteMaximumCapacity(label, - (float) childQueue.getQueueCapacities().getMaximumCapacity(label) - / getQueueCapacities().getAbsoluteMaximumCapacity(label)); + childQueue.getQueueCapacities().getMaximumCapacity(label) + * getQueueCapacities().getAbsoluteMaximumCapacity(label)); // Re-visit max applications for a queue based on absolute capacity if // needed. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java index 298e1ab..fe4c880 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java @@ -33,6 +33,7 @@ import org.junit.Test; public class TestAbsoluteResourceConfiguration { private static final int GB = 1024; + private static final float DELTA = 0.001f; private static final String QUEUEA = "queueA"; private static final String QUEUEB = "queueB"; @@ -544,6 +545,10 @@ public class TestAbsoluteResourceConfiguration { QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEA is not correct", QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource()); + Assert.assertEquals("Absolute capacity for QUEUEA is not correct", + 0.4, qA.getAbsoluteCapacity(), DELTA); + Assert.assertEquals("Absolute Max capacity for QUEUEA is not correct", + 0.8, qA.getAbsoluteMaximumCapacity(), DELTA); ParentQueue qB = (ParentQueue) cs.getQueue(QUEUEB); Assert.assertNotNull(qB); @@ -555,6 +560,10 @@ public class TestAbsoluteResourceConfiguration { QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEB is not correct", QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource()); + Assert.assertEquals("Absolute capacity for QUEUEB is not correct", + 0.2, qB.getAbsoluteCapacity(), DELTA); + Assert.assertEquals("Absolute Max capacity for QUEUEB is not correct", + 0.6, qB.getAbsoluteMaximumCapacity(), DELTA); LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC); Assert.assertNotNull(qC); @@ -566,18 +575,30 @@ public class TestAbsoluteResourceConfiguration { QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEC is not correct", QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); + Assert.assertEquals("Absolute capacity for QUEUEC is not correct", + 0.2, qC.getAbsoluteCapacity(), DELTA); + Assert.assertEquals("Absolute Max capacity for QUEUEC is not correct", + 0.6, qC.getAbsoluteMaximumCapacity(), DELTA); LeafQueue qA1 = (LeafQueue) cs.getQueue(QUEUEA1); Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct", QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct", QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource()); + Assert.assertEquals("Absolute capacity for QUEUEA1 is not correct", + 0.2, qA1.getAbsoluteCapacity(), DELTA); + Assert.assertEquals("Absolute Max capacity for QUEUEA1 is not correct", + 0.8, qA1.getAbsoluteMaximumCapacity(), DELTA); LeafQueue qA2 = (LeafQueue) cs.getQueue(QUEUEA2); Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct", QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct", QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource()); + Assert.assertEquals("Absolute capacity for QUEUEA2 is not correct", + 0.2, qA2.getAbsoluteCapacity(), DELTA); + Assert.assertEquals("Absolute Max capacity for QUEUEA2 is not correct", + 0.8, qA2.getAbsoluteMaximumCapacity(), DELTA); LeafQueue qB1 = (LeafQueue) cs.getQueue(QUEUEB1); Assert.assertEquals("Min resource configured for QUEUEB1 is not correct", @@ -588,6 +609,10 @@ public class TestAbsoluteResourceConfiguration { QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct", QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource()); + Assert.assertEquals("Absolute capacity for QUEUEB1 is not correct", + 0.16, qB1.getAbsoluteCapacity(), DELTA); + Assert.assertEquals("Absolute Max capacity for QUEUEB1 is not correct", + 0.6, qB1.getAbsoluteMaximumCapacity(), DELTA); // add new NM. rm.registerNode("127.0.0.3:1234", 125 * GB, 20); @@ -599,31 +624,55 @@ public class TestAbsoluteResourceConfiguration { QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEA is not correct", QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource()); + Assert.assertEquals("Absolute capacity for QUEUEA is not correct", + 0.266, qA.getAbsoluteCapacity(), DELTA); + Assert.assertEquals("Absolute Max capacity for QUEUEA is not correct", + 0.533, qA.getAbsoluteMaximumCapacity(), DELTA); Assert.assertEquals("Effective Min resource for QUEUEB is not correct", QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEB is not correct", QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource()); + Assert.assertEquals("Absolute capacity for QUEUEB is not correct", + 0.133, qB.getAbsoluteCapacity(), DELTA); + Assert.assertEquals("Absolute Max capacity for QUEUEB is not correct", + 0.4, qB.getAbsoluteMaximumCapacity(), DELTA); Assert.assertEquals("Effective Min resource for QUEUEC is not correct", QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEC is not correct", QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); + Assert.assertEquals("Absolute capacity for QUEUEC is not correct", + 0.133, qC.getAbsoluteCapacity(), DELTA); + Assert.assertEquals("Absolute Max capacity for QUEUEC is not correct", + 0.4, qC.getAbsoluteMaximumCapacity(), DELTA); Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct", QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct", QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource()); + Assert.assertEquals("Absolute capacity for QUEUEB1 is not correct", + 0.106, qB1.getAbsoluteCapacity(), DELTA); + Assert.assertEquals("Absolute Max capacity for QUEUEB1 is not correct", + 0.4, qB1.getAbsoluteMaximumCapacity(), DELTA); Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct", QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct", QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource()); + Assert.assertEquals("Absolute capacity for QUEUEA1 is not correct", + 0.133, qA1.getAbsoluteCapacity(), DELTA); + Assert.assertEquals("Absolute Max capacity for QUEUEA1 is not correct", + 0.533, qA1.getAbsoluteMaximumCapacity(), DELTA); Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct", QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource()); Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct", QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource()); + Assert.assertEquals("Absolute capacity for QUEUEA2 is not correct", + 0.133, qA2.getAbsoluteCapacity(), DELTA); + Assert.assertEquals("Absolute Max capacity for QUEUEA2 is not correct", + 0.533, qA2.getAbsoluteMaximumCapacity(), DELTA); rm.stop(); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org