YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbfe86de Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbfe86de Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbfe86de Branch: refs/heads/YARN-5734 Commit: fbfe86deea5f2aa857cd13fee913b7becee57f93 Parents: 815d535 Author: Karthik Kambatla <ka...@apache.org> Authored: Sun Feb 26 20:36:33 2017 -0800 Committer: Karthik Kambatla <ka...@apache.org> Committed: Sun Feb 26 20:36:33 2017 -0800 ---------------------------------------------------------------------- .../scheduler/fair/FSLeafQueue.java | 21 +++++------------ .../scheduler/fair/TestFSAppStarvation.java | 24 ++++++++++++++++---- 2 files changed, 26 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfe86de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index d0e0961..aad2916 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -331,20 +331,22 @@ public class FSLeafQueue extends FSQueue { public void updateDemand() { // Compute demand by iterating through apps in the queue // Limit demand to maxResources - demand = Resources.createResource(0); + Resource tmpDemand = Resources.createResource(0); readLock.lock(); try { for (FSAppAttempt sched : runnableApps) { - updateDemandForApp(sched); + sched.updateDemand(); + Resources.addTo(tmpDemand, sched.getDemand()); } for (FSAppAttempt sched : nonRunnableApps) { - updateDemandForApp(sched); + sched.updateDemand(); + Resources.addTo(tmpDemand, sched.getDemand()); } } finally { readLock.unlock(); } // Cap demand to maxShare to limit allocation to maxShare - demand = Resources.componentwiseMin(demand, maxShare); + demand = Resources.componentwiseMin(tmpDemand, maxShare); if (LOG.isDebugEnabled()) { LOG.debug("The updated demand for " + getName() + " is " + demand + "; the max is " + maxShare); @@ -352,17 +354,6 @@ public class FSLeafQueue extends FSQueue { + getFairShare()); } } - - private void updateDemandForApp(FSAppAttempt sched) { - sched.updateDemand(); - Resource toAdd = sched.getDemand(); - if (LOG.isDebugEnabled()) { - LOG.debug("Counting resource from " + sched.getName() + " " + toAdd - + "; Total resource demand for " + getName() + " now " - + demand); - } - demand = Resources.add(demand, toAdd); - } @Override public Resource assignContainer(FSSchedulerNode node) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfe86de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java index 2eacc9e..0712b4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java @@ -96,6 +96,14 @@ public class TestFSAppStarvation extends FairSchedulerTestBase { public void testPreemptionEnabled() throws Exception { setupClusterAndSubmitJobs(); + // Wait for apps to be processed by MockPreemptionThread + for (int i = 0; i < 6000; ++i) { + if (preemptionThread.uniqueAppsAdded() >= 3) { + break; + } + Thread.sleep(10); + } + assertNotNull("FSContext does not have an FSStarvedApps instance", scheduler.getContext().getStarvedApps()); assertEquals("Expecting 3 starved applications, one each for the " @@ -113,8 +121,19 @@ public class TestFSAppStarvation extends FairSchedulerTestBase { clock.tickMsec( FairSchedulerWithMockPreemption.DELAY_FOR_NEXT_STARVATION_CHECK_MS); scheduler.update(); + + // Wait for apps to be processed by MockPreemptionThread + for (int i = 0; i < 6000; ++i) { + if(preemptionThread.totalAppsAdded() > + preemptionThread.uniqueAppsAdded()) { + break; + } + Thread.sleep(10); + } + assertTrue("Each app is marked as starved exactly once", - preemptionThread.totalAppsAdded() > preemptionThread.uniqueAppsAdded()); + preemptionThread.totalAppsAdded() > + preemptionThread.uniqueAppsAdded()); } /* @@ -154,9 +173,6 @@ public class TestFSAppStarvation extends FairSchedulerTestBase { // Scheduler update to populate starved apps scheduler.update(); - - // Wait for apps to be processed by MockPreemptionThread - Thread.yield(); } /** --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
