This is an automated email from the ASF dual-hosted git repository. snemeth pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push: new 770a5631 YARN-10528. maxAMShare should only be accepted for leaf queues, not parent queues. Contributed by Siddharth Ahuja 770a5631 is described below commit 770a56315e1e6af48a735bdd617d209cd9158064 Author: Szilard Nemeth <snem...@apache.org> AuthorDate: Fri Jan 8 12:51:06 2021 +0100 YARN-10528. maxAMShare should only be accepted for leaf queues, not parent queues. Contributed by Siddharth Ahuja --- .../fair/allocation/AllocationFileQueueParser.java | 24 ++++++-- .../fair/TestAllocationFileLoaderService.java | 71 ++++++++++++++++++++++ 2 files changed, 91 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java index ec7e4a4..ebd37fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java @@ -124,6 +124,7 @@ public class AllocationFileQueueParser { NodeList fields = element.getChildNodes(); boolean isLeaf = true; boolean isReservable = false; + boolean isMaxAMShareSet = false; for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); @@ -155,6 +156,7 @@ public class AllocationFileQueueParser { float val = Float.parseFloat(text); val = Math.min(val, 1.0f); builder.queueMaxAMShares(queueName, val); + isMaxAMShareSet = true; } else if (WEIGHT.equals(field.getTagName())) { String text = getTrimmedTextData(field); double val = Double.parseDouble(text); @@ -220,10 +222,11 @@ public class AllocationFileQueueParser { builder.configuredQueues(FSQueueType.LEAF, queueName); } else { if (isReservable) { - throw new AllocationConfigurationException("The configuration settings" - + " for " + queueName + " are invalid. A queue element that " - + "contains child queue elements or that has the type='parent' " - + "attribute cannot also include a reservation element."); + throw new AllocationConfigurationException( + getErrorString(queueName, RESERVATION)); + } else if (isMaxAMShareSet) { + throw new AllocationConfigurationException( + getErrorString(queueName, MAX_AMSHARE)); } builder.configuredQueues(FSQueueType.PARENT, queueName); } @@ -243,6 +246,19 @@ public class AllocationFileQueueParser { builder.getMaxQueueResources(), queueName); } + /** + * Set up the error string based on the supplied parent queueName and element. + * @param parentQueueName the parent queue name. + * @param element the element that should not be present for the parent queue. + * @return the error string. + */ + private String getErrorString(String parentQueueName, String element) { + return "The configuration settings" + + " for " + parentQueueName + " are invalid. A queue element that " + + "contains child queue elements or that has the type='parent' " + + "attribute cannot also include a " + element + " element."; + } + private String getTrimmedTextData(Element element) { return ((Text) element.getFirstChild()).getData().trim(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index 4a7461d..cc28255 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -671,6 +671,77 @@ public class TestAllocationFileLoaderService { allocLoader.reloadAllocations(); } + /** + * Verify that a parent queue (type = parent) cannot have a maxAMShare element + * as dynamic queues won't be able to inherit this setting. + */ + @Test + public void testParentTagWitMaxAMShare() throws Exception { + Configuration conf = new Configuration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println("<?xml version=\"1.0\"?>"); + out.println("<allocations>"); + out.println("<queue name=\"parent\" type=\"parent\">"); + out.println("<maxAMShare>0.75</maxAMShare>"); + out.println("</queue>"); + out.println("</allocations>"); + out.close(); + + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); + try { + allocLoader.reloadAllocations(); + fail("Expect allocation parsing to fail as maxAMShare cannot be set for" + + " a parent queue."); + } catch (AllocationConfigurationException ex) { + assertEquals(ex.getMessage(), "The configuration settings for root.parent" + + " are invalid. A queue element that contains child queue elements" + + " or that has the type='parent' attribute cannot also include a" + + " maxAMShare element."); + } + } + + /** + * Verify that a parent queue that is not explicitly tagged with "type" + * as "parent" but has a child queue (implicit parent) cannot have a + * maxAMShare element. + */ + @Test + public void testParentWithMaxAMShare() throws Exception { + Configuration conf = new Configuration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println("<?xml version=\"1.0\"?>"); + out.println("<allocations>"); + out.println("<queue name=\"parent\">"); + out.println("<maxAMShare>0.76</maxAMShare>"); + out.println(" <queue name=\"child\">"); + out.println(" </queue>"); + out.println("</queue>"); + out.println("</allocations>"); + out.close(); + + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); + try { + allocLoader.reloadAllocations(); + fail("Expect allocation parsing to fail as maxAMShare cannot be set for" + + " a parent queue."); + } catch (AllocationConfigurationException ex) { + assertEquals(ex.getMessage(), "The configuration settings for root.parent" + + " are invalid. A queue element that contains child queue elements" + + " or that has the type='parent' attribute cannot also include a" + + " maxAMShare element."); + } + } + @Test public void testParentTagWithReservation() throws Exception { Configuration conf = new Configuration(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org