Author: tucu Date: Thu Oct 4 23:00:41 2012 New Revision: 1394321 URL: http://svn.apache.org/viewvc?rev=1394321&view=rev Log: YARN-23. FairScheduler: FSQueueSchedulable#updateDemand() - potential redundant aggregation. (kkambatl via tucu)
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSQueueSchedulable.java Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueSchedulable.java Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1394321&r1=1394320&r2=1394321&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Thu Oct 4 23:00:41 2012 @@ -43,6 +43,9 @@ Release 2.0.3-alpha - Unreleased YARN-116. Add the ability to change the RM include/exclude file without a restart. (xieguiming and Harsh J via sseth) + YARN-23. FairScheduler: FSQueueSchedulable#updateDemand() - potential + redundant aggregation. (kkambatl via tucu) + OPTIMIZATIONS BUG FIXES Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueSchedulable.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueSchedulable.java?rev=1394321&r1=1394320&r2=1394321&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueSchedulable.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueSchedulable.java Thu Oct 4 23:00:41 2012 @@ -65,6 +65,17 @@ public class FSQueueSchedulable extends long lastTimeAtMinShare; long lastTimeAtHalfFairShare; + // Constructor for tests + protected FSQueueSchedulable(FairScheduler scheduler, FSQueue fsQueue, + QueueManager qMgr, QueueMetrics metrics, long minShare, long fairShare) { + this.scheduler = scheduler; + this.queueMgr = qMgr; + this.queue = fsQueue; + this.metrics = metrics; + this.lastTimeAtMinShare = minShare; + this.lastTimeAtHalfFairShare = fairShare; + } + public FSQueueSchedulable(FairScheduler scheduler, FSQueue queue) { this.scheduler = scheduler; this.queue = queue; @@ -93,19 +104,27 @@ public class FSQueueSchedulable extends */ @Override public void updateDemand() { + // Compute demand by iterating through apps in the queue + // Limit demand to maxResources + Resource maxRes = queueMgr.getMaxResources(queue.getName()); demand = Resources.createResource(0); for (AppSchedulable sched: appScheds) { sched.updateDemand(); Resource toAdd = sched.getDemand(); - LOG.debug("Counting resource from " + sched.getName() + " " + toAdd.toString()); - LOG.debug("Total resource consumption for " + this.getName() + " now " + demand.toString()); + if (LOG.isDebugEnabled()) { + LOG.debug("Counting resource from " + sched.getName() + " " + toAdd + + "; Total resource consumption for " + this.getName() + " now " + + demand); + } demand = Resources.add(demand, toAdd); - + if (Resources.greaterThanOrEqual(demand, maxRes)) { + demand = maxRes; + break; + } } - // if demand exceeds the cap for this queue, limit to the max - Resource maxRes = queueMgr.getMaxResources(queue.getName()); - if(Resources.greaterThan(demand, maxRes)) { - demand = maxRes; + if (LOG.isDebugEnabled()) { + LOG.debug("The updated demand for " + this.getName() + " is " + demand + + "; the max is " + maxRes); } } Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSQueueSchedulable.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSQueueSchedulable.java?rev=1394321&view=auto ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSQueueSchedulable.java (added) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSQueueSchedulable.java Thu Oct 4 23:00:41 2012 @@ -0,0 +1,42 @@ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +public class TestFSQueueSchedulable { + private FSQueueSchedulable schedulable = null; + private Resource maxResource = Resources.createResource(10); + + @Before + public void setup() { + String queueName = "testFSQueue"; + FSQueue mockQueue = mock(FSQueue.class); + when(mockQueue.getName()).thenReturn(queueName); + + QueueManager mockMgr = mock(QueueManager.class); + when(mockMgr.getMaxResources(queueName)).thenReturn(maxResource); + + schedulable = new FSQueueSchedulable(null, mockQueue, mockMgr, null, 0, 0); + } + + @Test + public void testUpdateDemand() { + AppSchedulable app = mock(AppSchedulable.class); + Mockito.when(app.getDemand()).thenReturn(maxResource); + + schedulable.addApp(app); + schedulable.addApp(app); + + schedulable.updateDemand(); + + assertTrue("Demand is greater than max allowed ", + Resources.equals(schedulable.getDemand(), maxResource)); + } +}