Author: sandy
Date: Fri Jul 4 15:17:23 2014
New Revision: 1607874
URL: http://svn.apache.org/r1607874
Log:
YARN-2250. FairScheduler.findLowestCommonAncestorQueue returns null when queues
not identical (Krisztian Horvath via Sandy Ryza)
Modified:
hadoop/common/branches/branch-2.5/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Modified: hadoop/common/branches/branch-2.5/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.5/hadoop-yarn-project/CHANGES.txt?rev=1607874&r1=1607873&r2=1607874&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.5/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.5/hadoop-yarn-project/CHANGES.txt Fri Jul
4 15:17:23 2014
@@ -316,6 +316,9 @@ Release 2.5.0 - UNRELEASED
YARN-2232. Fixed ResourceManager to allow DelegationToken owners to be able
to cancel their own tokens in secure mode. (Varun Vasudev via vinodkv)
+ YARN-2250. FairScheduler.findLowestCommonAncestorQueue returns null when
+ queues not identical (Krisztian Horvath via Sandy Ryza)
+
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1607874&r1=1607873&r2=1607874&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
(original)
+++
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
Fri Jul 4 15:17:23 2014
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-import com.google.common.base.Preconditions;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -93,6 +92,7 @@ import org.apache.hadoop.yarn.util.resou
import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
/**
* A scheduler that schedules resources between a set of queues. The scheduler
@@ -1468,8 +1468,9 @@ public class FairScheduler extends
maxRunningEnforcer.updateRunnabilityOnAppRemoval(attempt, oldQueue);
}
}
-
- private FSQueue findLowestCommonAncestorQueue(FSQueue queue1, FSQueue
queue2) {
+
+ @VisibleForTesting
+ FSQueue findLowestCommonAncestorQueue(FSQueue queue1, FSQueue queue2) {
// Because queue names include ancestors, separated by periods, we can find
// the lowest common ancestors by going from the start of the names until
// there's a character that doesn't match.
@@ -1481,7 +1482,7 @@ public class FairScheduler extends
for (int i = 0; i < Math.max(name1.length(), name2.length()); i++) {
if (name1.length() <= i || name2.length() <= i ||
name1.charAt(i) != name2.charAt(i)) {
- return queueMgr.getQueue(name1.substring(lastPeriodIndex));
+ return queueMgr.getQueue(name1.substring(0, lastPeriodIndex));
} else if (name1.charAt(i) == '.') {
lastPeriodIndex = i;
}
Modified:
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1607874&r1=1607873&r2=1607874&view=diff
==============================================================================
---
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
(original)
+++
hadoop/common/branches/branch-2.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Fri Jul 4 15:17:23 2014
@@ -25,6 +25,8 @@ import static org.junit.Assert.assertNul
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.io.File;
import java.io.FileWriter;
@@ -3077,4 +3079,71 @@ public class TestFairScheduler extends F
createSchedulingRequest(1024, 1, "queue1", "user1", 3);
scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
}
+
+ @Test
+ public void testLowestCommonAncestorForNonRootParent() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ FSLeafQueue aQueue = mock(FSLeafQueue.class);
+ FSLeafQueue bQueue = mock(FSLeafQueue.class);
+ when(aQueue.getName()).thenReturn("root.queue1.a");
+ when(bQueue.getName()).thenReturn("root.queue1.b");
+
+ QueueManager queueManager = scheduler.getQueueManager();
+ FSParentQueue queue1 = queueManager.getParentQueue("queue1", true);
+ queue1.addChildQueue(aQueue);
+ queue1.addChildQueue(bQueue);
+
+ FSQueue ancestorQueue =
+ scheduler.findLowestCommonAncestorQueue(aQueue, bQueue);
+ assertEquals(ancestorQueue, queue1);
+ }
+
+ @Test
+ public void testLowestCommonAncestorRootParent() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ FSLeafQueue aQueue = mock(FSLeafQueue.class);
+ FSLeafQueue bQueue = mock(FSLeafQueue.class);
+ when(aQueue.getName()).thenReturn("root.a");
+ when(bQueue.getName()).thenReturn("root.b");
+
+ QueueManager queueManager = scheduler.getQueueManager();
+ FSParentQueue queue1 = queueManager.getParentQueue("root", false);
+ queue1.addChildQueue(aQueue);
+ queue1.addChildQueue(bQueue);
+
+ FSQueue ancestorQueue =
+ scheduler.findLowestCommonAncestorQueue(aQueue, bQueue);
+ assertEquals(ancestorQueue, queue1);
+ }
+
+ @Test
+ public void testLowestCommonAncestorDeeperHierarchy() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ FSQueue aQueue = mock(FSLeafQueue.class);
+ FSQueue bQueue = mock(FSLeafQueue.class);
+ FSQueue a1Queue = mock(FSLeafQueue.class);
+ FSQueue b1Queue = mock(FSLeafQueue.class);
+ when(a1Queue.getName()).thenReturn("root.queue1.a.a1");
+ when(b1Queue.getName()).thenReturn("root.queue1.b.b1");
+ when(aQueue.getChildQueues()).thenReturn(Arrays.asList(a1Queue));
+ when(bQueue.getChildQueues()).thenReturn(Arrays.asList(b1Queue));
+
+ QueueManager queueManager = scheduler.getQueueManager();
+ FSParentQueue queue1 = queueManager.getParentQueue("queue1", true);
+ queue1.addChildQueue(aQueue);
+ queue1.addChildQueue(bQueue);
+
+ FSQueue ancestorQueue =
+ scheduler.findLowestCommonAncestorQueue(a1Queue, b1Queue);
+ assertEquals(ancestorQueue, queue1);
+ }
}
