subject:"hadoop git commit\: YARN\-6172. FSLeafQueue demand update needs to be atomic. \(Miklos Szegedi via kasha\)"

[13/50] [abbrv] hadoop git commit: YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via kasha)

2017-03-03 Thread aengineer

YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via 
kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbfe86de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbfe86de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbfe86de

Branch: refs/heads/HDFS-7240
Commit: fbfe86deea5f2aa857cd13fee913b7becee57f93
Parents: 815d535
Author: Karthik Kambatla 
Authored: Sun Feb 26 20:36:33 2017 -0800
Committer: Karthik Kambatla 
Committed: Sun Feb 26 20:36:33 2017 -0800

--
 .../scheduler/fair/FSLeafQueue.java | 21 +
 .../scheduler/fair/TestFSAppStarvation.java | 24 
 2 files changed, 26 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfe86de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index d0e0961..aad2916 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -331,20 +331,22 @@ public class FSLeafQueue extends FSQueue {
   public void updateDemand() {
 // Compute demand by iterating through apps in the queue
 // Limit demand to maxResources
-demand = Resources.createResource(0);
+Resource tmpDemand = Resources.createResource(0);
 readLock.lock();
 try {
   for (FSAppAttempt sched : runnableApps) {
-updateDemandForApp(sched);
+sched.updateDemand();
+Resources.addTo(tmpDemand, sched.getDemand());
   }
   for (FSAppAttempt sched : nonRunnableApps) {
-updateDemandForApp(sched);
+sched.updateDemand();
+Resources.addTo(tmpDemand, sched.getDemand());
   }
 } finally {
   readLock.unlock();
 }
 // Cap demand to maxShare to limit allocation to maxShare
-demand = Resources.componentwiseMin(demand, maxShare);
+demand = Resources.componentwiseMin(tmpDemand, maxShare);
 if (LOG.isDebugEnabled()) {
   LOG.debug("The updated demand for " + getName() + " is " + demand
   + "; the max is " + maxShare);
@@ -352,17 +354,6 @@ public class FSLeafQueue extends FSQueue {
   + getFairShare());
 }
   }
-  
-  private void updateDemandForApp(FSAppAttempt sched) {
-sched.updateDemand();
-Resource toAdd = sched.getDemand();
-if (LOG.isDebugEnabled()) {
-  LOG.debug("Counting resource from " + sched.getName() + " " + toAdd
-  + "; Total resource demand for " + getName() + " now "
-  + demand);
-}
-demand = Resources.add(demand, toAdd);
-  }
 
   @Override
   public Resource assignContainer(FSSchedulerNode node) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfe86de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
index 2eacc9e..0712b4c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
@@ -96,6 +96,14 @@ public class TestFSAppStarvation extends 
FairSchedulerTestBase {
   public void testPreemptionEnabled() throws Exception {
 setupClusterAndSubmitJobs();
 
+// Wait for apps to be processed by MockPree

[19/50] [abbrv] hadoop git commit: YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via kasha)

2017-03-02 Thread jhung

YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via 
kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbfe86de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbfe86de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbfe86de

Branch: refs/heads/YARN-5734
Commit: fbfe86deea5f2aa857cd13fee913b7becee57f93
Parents: 815d535
Author: Karthik Kambatla 
Authored: Sun Feb 26 20:36:33 2017 -0800
Committer: Karthik Kambatla 
Committed: Sun Feb 26 20:36:33 2017 -0800

--
 .../scheduler/fair/FSLeafQueue.java | 21 +
 .../scheduler/fair/TestFSAppStarvation.java | 24 
 2 files changed, 26 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfe86de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index d0e0961..aad2916 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -331,20 +331,22 @@ public class FSLeafQueue extends FSQueue {
   public void updateDemand() {
 // Compute demand by iterating through apps in the queue
 // Limit demand to maxResources
-demand = Resources.createResource(0);
+Resource tmpDemand = Resources.createResource(0);
 readLock.lock();
 try {
   for (FSAppAttempt sched : runnableApps) {
-updateDemandForApp(sched);
+sched.updateDemand();
+Resources.addTo(tmpDemand, sched.getDemand());
   }
   for (FSAppAttempt sched : nonRunnableApps) {
-updateDemandForApp(sched);
+sched.updateDemand();
+Resources.addTo(tmpDemand, sched.getDemand());
   }
 } finally {
   readLock.unlock();
 }
 // Cap demand to maxShare to limit allocation to maxShare
-demand = Resources.componentwiseMin(demand, maxShare);
+demand = Resources.componentwiseMin(tmpDemand, maxShare);
 if (LOG.isDebugEnabled()) {
   LOG.debug("The updated demand for " + getName() + " is " + demand
   + "; the max is " + maxShare);
@@ -352,17 +354,6 @@ public class FSLeafQueue extends FSQueue {
   + getFairShare());
 }
   }
-  
-  private void updateDemandForApp(FSAppAttempt sched) {
-sched.updateDemand();
-Resource toAdd = sched.getDemand();
-if (LOG.isDebugEnabled()) {
-  LOG.debug("Counting resource from " + sched.getName() + " " + toAdd
-  + "; Total resource demand for " + getName() + " now "
-  + demand);
-}
-demand = Resources.add(demand, toAdd);
-  }
 
   @Override
   public Resource assignContainer(FSSchedulerNode node) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfe86de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
index 2eacc9e..0712b4c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
@@ -96,6 +96,14 @@ public class TestFSAppStarvation extends 
FairSchedulerTestBase {
   public void testPreemptionEnabled() throws Exception {
 setupClusterAndSubmitJobs();
 
+// Wait for apps to be processed by MockPree

[07/38] hadoop git commit: YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via kasha)

2017-03-02 Thread jhung

YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via 
kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbfe86de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbfe86de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbfe86de

Branch: refs/heads/YARN-5734
Commit: fbfe86deea5f2aa857cd13fee913b7becee57f93
Parents: 815d535
Author: Karthik Kambatla 
Authored: Sun Feb 26 20:36:33 2017 -0800
Committer: Karthik Kambatla 
Committed: Sun Feb 26 20:36:33 2017 -0800

--
 .../scheduler/fair/FSLeafQueue.java | 21 +
 .../scheduler/fair/TestFSAppStarvation.java | 24 
 2 files changed, 26 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfe86de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index d0e0961..aad2916 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -331,20 +331,22 @@ public class FSLeafQueue extends FSQueue {
   public void updateDemand() {
 // Compute demand by iterating through apps in the queue
 // Limit demand to maxResources
-demand = Resources.createResource(0);
+Resource tmpDemand = Resources.createResource(0);
 readLock.lock();
 try {
   for (FSAppAttempt sched : runnableApps) {
-updateDemandForApp(sched);
+sched.updateDemand();
+Resources.addTo(tmpDemand, sched.getDemand());
   }
   for (FSAppAttempt sched : nonRunnableApps) {
-updateDemandForApp(sched);
+sched.updateDemand();
+Resources.addTo(tmpDemand, sched.getDemand());
   }
 } finally {
   readLock.unlock();
 }
 // Cap demand to maxShare to limit allocation to maxShare
-demand = Resources.componentwiseMin(demand, maxShare);
+demand = Resources.componentwiseMin(tmpDemand, maxShare);
 if (LOG.isDebugEnabled()) {
   LOG.debug("The updated demand for " + getName() + " is " + demand
   + "; the max is " + maxShare);
@@ -352,17 +354,6 @@ public class FSLeafQueue extends FSQueue {
   + getFairShare());
 }
   }
-  
-  private void updateDemandForApp(FSAppAttempt sched) {
-sched.updateDemand();
-Resource toAdd = sched.getDemand();
-if (LOG.isDebugEnabled()) {
-  LOG.debug("Counting resource from " + sched.getName() + " " + toAdd
-  + "; Total resource demand for " + getName() + " now "
-  + demand);
-}
-demand = Resources.add(demand, toAdd);
-  }
 
   @Override
   public Resource assignContainer(FSSchedulerNode node) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfe86de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
index 2eacc9e..0712b4c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
@@ -96,6 +96,14 @@ public class TestFSAppStarvation extends 
FairSchedulerTestBase {
   public void testPreemptionEnabled() throws Exception {
 setupClusterAndSubmitJobs();
 
+// Wait for apps to be processed by MockPree

[27/31] hadoop git commit: YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via kasha)

2017-02-27 Thread stevel

YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via 
kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbfe86de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbfe86de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbfe86de

Branch: refs/heads/HADOOP-13345
Commit: fbfe86deea5f2aa857cd13fee913b7becee57f93
Parents: 815d535
Author: Karthik Kambatla 
Authored: Sun Feb 26 20:36:33 2017 -0800
Committer: Karthik Kambatla 
Committed: Sun Feb 26 20:36:33 2017 -0800

--
 .../scheduler/fair/FSLeafQueue.java | 21 +
 .../scheduler/fair/TestFSAppStarvation.java | 24 
 2 files changed, 26 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfe86de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index d0e0961..aad2916 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -331,20 +331,22 @@ public class FSLeafQueue extends FSQueue {
   public void updateDemand() {
 // Compute demand by iterating through apps in the queue
 // Limit demand to maxResources
-demand = Resources.createResource(0);
+Resource tmpDemand = Resources.createResource(0);
 readLock.lock();
 try {
   for (FSAppAttempt sched : runnableApps) {
-updateDemandForApp(sched);
+sched.updateDemand();
+Resources.addTo(tmpDemand, sched.getDemand());
   }
   for (FSAppAttempt sched : nonRunnableApps) {
-updateDemandForApp(sched);
+sched.updateDemand();
+Resources.addTo(tmpDemand, sched.getDemand());
   }
 } finally {
   readLock.unlock();
 }
 // Cap demand to maxShare to limit allocation to maxShare
-demand = Resources.componentwiseMin(demand, maxShare);
+demand = Resources.componentwiseMin(tmpDemand, maxShare);
 if (LOG.isDebugEnabled()) {
   LOG.debug("The updated demand for " + getName() + " is " + demand
   + "; the max is " + maxShare);
@@ -352,17 +354,6 @@ public class FSLeafQueue extends FSQueue {
   + getFairShare());
 }
   }
-  
-  private void updateDemandForApp(FSAppAttempt sched) {
-sched.updateDemand();
-Resource toAdd = sched.getDemand();
-if (LOG.isDebugEnabled()) {
-  LOG.debug("Counting resource from " + sched.getName() + " " + toAdd
-  + "; Total resource demand for " + getName() + " now "
-  + demand);
-}
-demand = Resources.add(demand, toAdd);
-  }
 
   @Override
   public Resource assignContainer(FSSchedulerNode node) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfe86de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
index 2eacc9e..0712b4c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
@@ -96,6 +96,14 @@ public class TestFSAppStarvation extends 
FairSchedulerTestBase {
   public void testPreemptionEnabled() throws Exception {
 setupClusterAndSubmitJobs();
 
+// Wait for apps to be processed by MockP

hadoop git commit: YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via kasha)

2017-02-26 Thread kasha

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 a95d3e1fe -> c24325bbc


YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via 
kasha)

(cherry picked from commit fbfe86deea5f2aa857cd13fee913b7becee57f93)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c24325bb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c24325bb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c24325bb

Branch: refs/heads/branch-2
Commit: c24325bbcbc8e10d1f8b1dd2e0f2ce0d2d61b232
Parents: a95d3e1
Author: Karthik Kambatla 
Authored: Sun Feb 26 20:36:33 2017 -0800
Committer: Karthik Kambatla 
Committed: Sun Feb 26 20:37:18 2017 -0800

--
 .../scheduler/fair/FSLeafQueue.java | 21 +
 .../scheduler/fair/TestFSAppStarvation.java | 24 
 2 files changed, 26 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c24325bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 3692526..0d61bd6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -331,20 +331,22 @@ public class FSLeafQueue extends FSQueue {
   public void updateDemand() {
 // Compute demand by iterating through apps in the queue
 // Limit demand to maxResources
-demand = Resources.createResource(0);
+Resource tmpDemand = Resources.createResource(0);
 readLock.lock();
 try {
   for (FSAppAttempt sched : runnableApps) {
-updateDemandForApp(sched);
+sched.updateDemand();
+Resources.addTo(tmpDemand, sched.getDemand());
   }
   for (FSAppAttempt sched : nonRunnableApps) {
-updateDemandForApp(sched);
+sched.updateDemand();
+Resources.addTo(tmpDemand, sched.getDemand());
   }
 } finally {
   readLock.unlock();
 }
 // Cap demand to maxShare to limit allocation to maxShare
-demand = Resources.componentwiseMin(demand, maxShare);
+demand = Resources.componentwiseMin(tmpDemand, maxShare);
 if (LOG.isDebugEnabled()) {
   LOG.debug("The updated demand for " + getName() + " is " + demand
   + "; the max is " + maxShare);
@@ -352,17 +354,6 @@ public class FSLeafQueue extends FSQueue {
   + getFairShare());
 }
   }
-  
-  private void updateDemandForApp(FSAppAttempt sched) {
-sched.updateDemand();
-Resource toAdd = sched.getDemand();
-if (LOG.isDebugEnabled()) {
-  LOG.debug("Counting resource from " + sched.getName() + " " + toAdd
-  + "; Total resource demand for " + getName() + " now "
-  + demand);
-}
-demand = Resources.add(demand, toAdd);
-  }
 
   @Override
   public Resource assignContainer(FSSchedulerNode node) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c24325bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
index 2eacc9e..0712b4c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
@@ -96,6 +96,14 @@ public class TestFSAppStarvation extends 
FairSchedulerT

hadoop git commit: YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via kasha)

2017-02-26 Thread kasha

Repository: hadoop
Updated Branches:
  refs/heads/trunk 815d53506 -> fbfe86dee


YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via 
kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbfe86de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbfe86de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbfe86de

Branch: refs/heads/trunk
Commit: fbfe86deea5f2aa857cd13fee913b7becee57f93
Parents: 815d535
Author: Karthik Kambatla 
Authored: Sun Feb 26 20:36:33 2017 -0800
Committer: Karthik Kambatla 
Committed: Sun Feb 26 20:36:33 2017 -0800

--
 .../scheduler/fair/FSLeafQueue.java | 21 +
 .../scheduler/fair/TestFSAppStarvation.java | 24 
 2 files changed, 26 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfe86de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index d0e0961..aad2916 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -331,20 +331,22 @@ public class FSLeafQueue extends FSQueue {
   public void updateDemand() {
 // Compute demand by iterating through apps in the queue
 // Limit demand to maxResources
-demand = Resources.createResource(0);
+Resource tmpDemand = Resources.createResource(0);
 readLock.lock();
 try {
   for (FSAppAttempt sched : runnableApps) {
-updateDemandForApp(sched);
+sched.updateDemand();
+Resources.addTo(tmpDemand, sched.getDemand());
   }
   for (FSAppAttempt sched : nonRunnableApps) {
-updateDemandForApp(sched);
+sched.updateDemand();
+Resources.addTo(tmpDemand, sched.getDemand());
   }
 } finally {
   readLock.unlock();
 }
 // Cap demand to maxShare to limit allocation to maxShare
-demand = Resources.componentwiseMin(demand, maxShare);
+demand = Resources.componentwiseMin(tmpDemand, maxShare);
 if (LOG.isDebugEnabled()) {
   LOG.debug("The updated demand for " + getName() + " is " + demand
   + "; the max is " + maxShare);
@@ -352,17 +354,6 @@ public class FSLeafQueue extends FSQueue {
   + getFairShare());
 }
   }
-  
-  private void updateDemandForApp(FSAppAttempt sched) {
-sched.updateDemand();
-Resource toAdd = sched.getDemand();
-if (LOG.isDebugEnabled()) {
-  LOG.debug("Counting resource from " + sched.getName() + " " + toAdd
-  + "; Total resource demand for " + getName() + " now "
-  + demand);
-}
-demand = Resources.add(demand, toAdd);
-  }
 
   @Override
   public Resource assignContainer(FSSchedulerNode node) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbfe86de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
index 2eacc9e..0712b4c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
@@ -96,6 +96,14 @@ public class TestFSAppStarvation extends 
FairSchedulerTestBase {
   public void testPreemptionEnabled() throws Exception {
 set

[13/50] [abbrv] hadoop git commit: YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via kasha)

[19/50] [abbrv] hadoop git commit: YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via kasha)

[07/38] hadoop git commit: YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via kasha)

[27/31] hadoop git commit: YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via kasha)

hadoop git commit: YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via kasha)

hadoop git commit: YARN-6172. FSLeafQueue demand update needs to be atomic. (Miklos Szegedi via kasha)

6 matches

Site Navigation

Mail list logo

Footer information