YARN-5272. Handle queue names consistently in FairScheduler. (Wilfred Spiegelenburg via rchiang)
(cherry picked from commit 2df1b1e72b18422606e38a9fbe311988637cb9a5) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/66a501be Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/66a501be Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/66a501be Branch: refs/heads/YARN-5355-branch-2 Commit: 66a501be5e3243df89332ea2bacc4683085d5ca2 Parents: 9d31c0b Author: Ray Chiang <rchi...@apache.org> Authored: Fri Jul 15 14:35:50 2016 -0700 Committer: Ray Chiang <rchi...@apache.org> Committed: Fri Jul 15 14:39:28 2016 -0700 ---------------------------------------------------------------------- .../fair/AllocationFileLoaderService.java | 4 ++- .../scheduler/fair/QueueManager.java | 6 ++++- .../fair/TestAllocationFileLoaderService.java | 28 ++++++++++++++++++++ .../scheduler/fair/TestFairScheduler.java | 11 ++++++++ .../scheduler/fair/TestQueueManager.java | 3 +++ 5 files changed, 50 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/66a501be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index 661caa7..fab536d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -53,6 +53,7 @@ import org.w3c.dom.NodeList; import org.w3c.dom.Text; import org.xml.sax.SAXException; +import com.google.common.base.CharMatcher; import com.google.common.annotations.VisibleForTesting; @Public @@ -445,7 +446,8 @@ public class AllocationFileLoaderService extends AbstractService { Set<String> reservableQueues, Set<String> nonPreemptableQueues) throws AllocationConfigurationException { - String queueName = element.getAttribute("name").trim(); + String queueName = CharMatcher.WHITESPACE.trimFrom( + element.getAttribute("name")); if (queueName.contains(".")) { throw new AllocationConfigurationException("Bad fair scheduler config " http://git-wip-us.apache.org/repos/asf/hadoop/blob/66a501be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java index aeadcf6..8d308dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java @@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.xml.sax.SAXException; +import com.google.common.base.CharMatcher; import com.google.common.annotations.VisibleForTesting; /** * Maintains a list of queues as well as scheduling parameters for each queue, @@ -457,6 +458,9 @@ public class QueueManager { */ @VisibleForTesting boolean isQueueNameValid(String node) { - return !node.isEmpty() && node.equals(node.trim()); + // use the same white space trim as in QueueMetrics() otherwise things fail + // guava uses a different definition for whitespace than java. + return !node.isEmpty() && + node.equals(CharMatcher.WHITESPACE.trimFrom(node)); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/66a501be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index cc91ef9..11d4981 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -20,8 +20,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import static org.junit.Assert.*; import java.io.File; +import java.io.FileOutputStream; import java.io.FileWriter; +import java.io.OutputStreamWriter; import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -581,6 +584,31 @@ public class TestAllocationFileLoaderService { } /** + * Verify that you can't have the queue name with just a non breaking + * whitespace in the allocations file. + */ + @Test (expected = AllocationConfigurationException.class) + public void testQueueNameContainingNBWhitespace() throws Exception { + Configuration conf = new Configuration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + PrintWriter out = new PrintWriter(new OutputStreamWriter( + new FileOutputStream(ALLOC_FILE), StandardCharsets.UTF_8)); + out.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"); + out.println("<allocations>"); + out.println("<queue name=\"\u00a0\">"); + out.println("</queue>"); + out.println("</allocations>"); + out.close(); + + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); + allocLoader.reloadAllocations(); + } + + /** * Verify that defaultQueueSchedulingMode can't accept FIFO as a value. */ @Test (expected = AllocationConfigurationException.class) http://git-wip-us.apache.org/repos/asf/hadoop/blob/66a501be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 61d9322..eb56c26 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -4362,6 +4362,17 @@ public class TestFairScheduler extends FairSchedulerTestBase { assertEquals(1, scheduler.getQueueManager().getLeafQueue("B.C", true) .getNumRunnableApps()); assertNotNull(scheduler.getSchedulerApp(appAttemptId3)); + + // submit app with queue name "A\u00a0" (non-breaking space) + ApplicationAttemptId appAttemptId4 = createAppAttemptId(4, 1); + AppAddedSchedulerEvent appAddedEvent4 = new AppAddedSchedulerEvent( + appAttemptId4.getApplicationId(), "A\u00a0", "user1"); + scheduler.handle(appAddedEvent4); + // submission rejected + assertEquals(3, scheduler.getQueueManager().getLeafQueues().size()); + assertNull(scheduler.getSchedulerApplications().get(appAttemptId4. + getApplicationId())); + assertNull(scheduler.getSchedulerApp(appAttemptId4)); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/66a501be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java index a9b27a1..33d4419 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java @@ -131,6 +131,9 @@ public class TestQueueManager { assertFalse(queueManager.isQueueNameValid(" a")); assertFalse(queueManager.isQueueNameValid("a ")); assertFalse(queueManager.isQueueNameValid(" a ")); + assertFalse(queueManager.isQueueNameValid("\u00a0")); + assertFalse(queueManager.isQueueNameValid("a\u00a0")); + assertFalse(queueManager.isQueueNameValid("\u00a0a\u00a0")); assertTrue(queueManager.isQueueNameValid("a b")); assertTrue(queueManager.isQueueNameValid("a")); } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
