Repository: hadoop Updated Branches: refs/heads/branch-2 312e50f9c -> 99b825569
YARN-3258. FairScheduler: Need to add more logging to investigate allocations. Contributed by Anubhav Dhoot. (cherry picked from commit b5a22e983832d4843b5df1d07858988e8bbf37e3) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99b82556 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99b82556 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99b82556 Branch: refs/heads/branch-2 Commit: 99b8255693f92db8c6cd6666e8c976e7f4263dbc Parents: 312e50f Author: Tsuyoshi Ozawa <oz...@apache.org> Authored: Tue Mar 31 17:42:44 2015 +0900 Committer: Tsuyoshi Ozawa <oz...@apache.org> Committed: Tue Mar 31 17:43:07 2015 +0900 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../server/resourcemanager/scheduler/fair/FSAppAttempt.java | 4 ++++ .../server/resourcemanager/scheduler/fair/FSLeafQueue.java | 8 +++++++- .../yarn/server/resourcemanager/scheduler/fair/FSQueue.java | 8 ++++++++ 4 files changed, 22 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/99b82556/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index d2850ad..b76ef16 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -38,6 +38,9 @@ Release 2.8.0 - UNRELEASED YARN-2495. Allow admin specify labels from each NM (Distributed configuration for node label). (Naganarasimha G R via wangda) + YARN-3258. FairScheduler: Need to add more logging to investigate + allocations. (Anubhav Dhoot via ozawa) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/99b82556/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index dfde5ab..46617ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -570,6 +570,10 @@ public class FSAppAttempt extends SchedulerApplicationAttempt // Check the AM resource usage for the leaf queue if (getLiveContainers().size() == 0 && !getUnmanagedAM()) { if (!getQueue().canRunAppAM(getAMResource())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping allocation because maxAMShare limit would " + + "be exceeded"); + } return Resources.none(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/99b82556/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index 3c97535..c49a323 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -284,6 +284,8 @@ public class FSLeafQueue extends FSQueue { if (LOG.isDebugEnabled()) { LOG.debug("The updated demand for " + getName() + " is " + demand + "; the max is " + maxRes); + LOG.debug("The updated fairshare for " + getName() + " is " + + getFairShare()); } } @@ -304,7 +306,7 @@ public class FSLeafQueue extends FSQueue { Resource assigned = Resources.none(); if (LOG.isDebugEnabled()) { LOG.debug("Node " + node.getNodeName() + " offered to queue: " + - getName()); + getName() + " fairShare: " + getFairShare()); } if (!assignContainerPreCheck(node)) { @@ -330,6 +332,10 @@ public class FSLeafQueue extends FSQueue { assigned = sched.assignContainer(node); if (!assigned.equals(Resources.none())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Assigned container in queue:" + getName() + " " + + "container:" + assigned); + } break; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/99b82556/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index 1562bf6..ee47d9a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -23,6 +23,8 @@ import java.util.Collection; import java.util.List; import java.util.Set; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.UserGroupInformation; @@ -41,6 +43,9 @@ import org.apache.hadoop.yarn.util.resource.Resources; @Private @Unstable public abstract class FSQueue implements Queue, Schedulable { + private static final Log LOG = LogFactory.getLog( + FSQueue.class.getName()); + private Resource fairShare = Resources.createResource(0, 0); private Resource steadyFairShare = Resources.createResource(0, 0); private final String name; @@ -164,6 +169,9 @@ public abstract class FSQueue implements Queue, Schedulable { public void setFairShare(Resource fairShare) { this.fairShare = fairShare; metrics.setFairShare(fairShare); + if (LOG.isDebugEnabled()) { + LOG.debug("The updated fairShare for " + getName() + " is " + fairShare); + } } /** Get the steady fair share assigned to this Schedulable. */
