This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new 7c4ef42 YARN-10599. fs2cs should generate new 'auto-queue-creation-v2.enabled' properties for all parents. Contributed by Peter Bacsko 7c4ef42 is described below commit 7c4ef42837955a531400005f824b27ca64c11cbd Author: Szilard Nemeth <snem...@apache.org> AuthorDate: Wed Jan 27 15:49:26 2021 +0100 YARN-10599. fs2cs should generate new 'auto-queue-creation-v2.enabled' properties for all parents. Contributed by Peter Bacsko --- .../weightconversion/WeightToWeightConverter.java | 12 +++++++-- .../fair/converter/TestFSQueueConverter.java | 29 ++++++++++++++++++++ .../TestWeightToWeightConverter.java | 31 ++++++++++++++++++++-- .../weightconversion/WeightConverterTestBase.java | 10 ++++--- 4 files changed, 75 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java index e5ce0b7..6dcd05c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.C import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; public class WeightToWeightConverter @@ -33,14 +34,21 @@ public class WeightToWeightConverter Configuration csConfig) { List<FSQueue> children = queue.getChildQueues(); - children.forEach(fsQueue -> csConfig.set( - getProperty(fsQueue), getWeightString(fsQueue))); + if (queue instanceof FSParentQueue || !children.isEmpty()) { + children.forEach(fsQueue -> csConfig.set( + getProperty(fsQueue), getWeightString(fsQueue))); + csConfig.setBoolean(getAutoCreateV2EnabledProperty(queue), true); + } } private String getProperty(FSQueue queue) { return PREFIX + queue.getName() + ".capacity"; } + private String getAutoCreateV2EnabledProperty(FSQueue queue) { + return PREFIX + queue.getName() + ".auto-queue-creation-v2.enabled"; + } + private String getWeightString(FSQueue queue) { return Float.toString(queue.getWeight()) + "w"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java index 7807092..e648c1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java @@ -335,6 +335,35 @@ public class TestFSQueueConverter { } @Test + public void testAutoCreateV2FlagsInWeightMode() { + converter = builder.withPercentages(false).build(); + + converter.convertQueueHierarchy(rootQueue); + + assertTrue("root autocreate v2 flag", + csConfig.getBoolean( + PREFIX + "root.auto-queue-creation-v2.enabled", false)); + assertTrue("root.admins autocreate v2 flag", + csConfig.getBoolean( + PREFIX + "root.admins.auto-queue-creation-v2.enabled", false)); + assertTrue("root.users autocreate v2 flag", + csConfig.getBoolean( + PREFIX + "root.users.auto-queue-creation-v2.enabled", false)); + assertTrue("root.misc autocreate v2 flag", + csConfig.getBoolean( + PREFIX + "root.misc.auto-queue-creation-v2.enabled", false)); + + Set<String> leafs = Sets.difference(ALL_QUEUES, + Sets.newHashSet("root", + "root.default", + "root.admins", + "root.users", + "root.misc")); + assertNoValueForQueues(leafs, "auto-queue-creation-v2.enabled", + csConfig); + } + + @Test public void testZeroSumCapacityValidation() { converter = builder.withPercentages(true).build(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java index c605a84..5f9b2d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java @@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.w import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; @@ -41,7 +44,7 @@ public class TestWeightToWeightConverter extends WeightConverterTestBase { FSQueue root = createFSQueues(); converter.convertWeightsForChildQueues(root, config); - assertEquals("Converted items", 0, + assertEquals("Converted items", 1, config.getPropsWithPrefix(PREFIX).size()); } @@ -52,6 +55,8 @@ public class TestWeightToWeightConverter extends WeightConverterTestBase { assertEquals("root.a weight", "1.0w", config.get(PREFIX + "root.a.capacity")); + assertEquals("Number of properties", 2, + config.getPropsWithPrefix(PREFIX).size()); } @Test @@ -60,7 +65,7 @@ public class TestWeightToWeightConverter extends WeightConverterTestBase { converter.convertWeightsForChildQueues(root, config); - assertEquals("Number of properties", 3, + assertEquals("Number of properties", 4, config.getPropsWithPrefix(PREFIX).size()); assertEquals("root.a weight", "1.0w", config.get(PREFIX + "root.a.capacity")); @@ -69,4 +74,26 @@ public class TestWeightToWeightConverter extends WeightConverterTestBase { assertEquals("root.c weight", "3.0w", config.get(PREFIX + "root.c.capacity")); } + + @Test + public void testAutoCreateV2FlagOnParent() { + FSQueue root = createFSQueues(1); + converter.convertWeightsForChildQueues(root, config); + + assertTrue("root autocreate v2 enabled", + config.getBoolean(PREFIX + "root.auto-queue-creation-v2.enabled", + false)); + } + + @Test + public void testAutoCreateV2FlagOnParentWithoutChildren() { + FSQueue root = createParent(new ArrayList<>()); + converter.convertWeightsForChildQueues(root, config); + + assertEquals("Number of properties", 1, + config.getPropsWithPrefix(PREFIX).size()); + assertTrue("root autocreate v2 enabled", + config.getBoolean(PREFIX + "root.auto-queue-creation-v2.enabled", + false)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightConverterTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightConverterTestBase.java index 7ea6104..6dbdc12 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightConverterTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightConverterTestBase.java @@ -24,6 +24,7 @@ import static org.mockito.Mockito.when; import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; import org.apache.hadoop.yarn.util.resource.Resources; @@ -44,12 +45,15 @@ public abstract class WeightConverterTestBase { queues.add(queue); } - FSQueue root = mock(FSQueue.class); + return createParent(queues); + } + + protected FSParentQueue createParent(List<FSQueue> children) { + FSParentQueue root = mock(FSParentQueue.class); when(root.getWeight()).thenReturn(1.0f); when(root.getName()).thenReturn("root"); when(root.getMinShare()).thenReturn(Resources.none()); - when(root.getChildQueues()).thenReturn(queues); - + when(root.getChildQueues()).thenReturn(children); return root; } } \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org