subject:"hadoop git commit\: YARN\-6546. SLS is slow while loading 10k queues. \(Yufei Gu via Haibo Chen\)"

[17/50] hadoop git commit: YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)

2017-10-23 Thread aengineer

YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46eb1033
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46eb1033
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46eb1033

Branch: refs/heads/HDFS-7240
Commit: 46eb1033a86ca53e7b94202567aef8af2417bdf8
Parents: f27a4ad
Author: Haibo Chen 
Authored: Tue Oct 17 16:03:46 2017 -0700
Committer: Haibo Chen 
Committed: Tue Oct 17 16:04:19 2017 -0700

--
 .../org/apache/hadoop/yarn/sls/SLSRunner.java   |  1 +
 .../sls/scheduler/CapacitySchedulerMetrics.java |  5 --
 .../sls/scheduler/FairSchedulerMetrics.java | 17 +
 .../sls/scheduler/FifoSchedulerMetrics.java |  5 +-
 .../sls/scheduler/SLSCapacityScheduler.java | 19 --
 .../yarn/sls/scheduler/SLSFairScheduler.java| 19 --
 .../yarn/sls/scheduler/SchedulerMetrics.java| 66 +---
 7 files changed, 36 insertions(+), 96 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 59f9c17..dfdf7c9 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -731,6 +731,7 @@ public class SLSRunner extends Configured implements Tool {
 }
 
 queueAppNumMap.put(queueName, appNum);
+wrapper.getSchedulerMetrics().trackQueue(queueName);
   }
 
   private void runNewAM(String jobType, String user,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
index a73f48c..89b44c6 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
@@ -28,9 +28,4 @@ public class CapacitySchedulerMetrics extends 
SchedulerMetrics {
   public CapacitySchedulerMetrics() {
 super();
   }
-
-  @Override
-  public void trackQueue(String queueName) {
-trackedQueues.add(queueName);
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
index 7b306f0..a5aee74 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
@@ -167,8 +167,9 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
   }
 
   @Override
-  public void trackQueue(String queueName) {
-trackedQueues.add(queueName);
+  protected void registerQueueMetrics(String queueName) {
+super.registerQueueMetrics(queueName);
+
 FairScheduler fair = (FairScheduler) scheduler;
 final FSQueue queue = fair.getQueueManager().getQueue(queueName);
 registerQueueMetrics(queue, Metric.DEMAND);
@@ -209,16 +210,4 @@ public class FairSchedulerMetrics extends SchedulerMetrics 
{
   }
 );
   }
-
-  @Override
-  public void untrackQueue(String queueName) {
-trackedQueues.remove(queueName);
-
-for (Metric metric: Metric.values()) {
-  metrics.remove("variable.queue." + queueName + "." +
-  metric.value + ".memory");
-  metrics.remove("variable.queue." + queueName + "." +
-  metric.value + ".vcores");
-}
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
 
b/hadoop-tools/hadoop-sls/src/ma

[33/50] [abbrv] hadoop git commit: YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)

2017-10-19 Thread haibochen

YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46eb1033
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46eb1033
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46eb1033

Branch: refs/heads/YARN-1011
Commit: 46eb1033a86ca53e7b94202567aef8af2417bdf8
Parents: f27a4ad
Author: Haibo Chen 
Authored: Tue Oct 17 16:03:46 2017 -0700
Committer: Haibo Chen 
Committed: Tue Oct 17 16:04:19 2017 -0700

--
 .../org/apache/hadoop/yarn/sls/SLSRunner.java   |  1 +
 .../sls/scheduler/CapacitySchedulerMetrics.java |  5 --
 .../sls/scheduler/FairSchedulerMetrics.java | 17 +
 .../sls/scheduler/FifoSchedulerMetrics.java |  5 +-
 .../sls/scheduler/SLSCapacityScheduler.java | 19 --
 .../yarn/sls/scheduler/SLSFairScheduler.java| 19 --
 .../yarn/sls/scheduler/SchedulerMetrics.java| 66 +---
 7 files changed, 36 insertions(+), 96 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 59f9c17..dfdf7c9 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -731,6 +731,7 @@ public class SLSRunner extends Configured implements Tool {
 }
 
 queueAppNumMap.put(queueName, appNum);
+wrapper.getSchedulerMetrics().trackQueue(queueName);
   }
 
   private void runNewAM(String jobType, String user,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
index a73f48c..89b44c6 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
@@ -28,9 +28,4 @@ public class CapacitySchedulerMetrics extends 
SchedulerMetrics {
   public CapacitySchedulerMetrics() {
 super();
   }
-
-  @Override
-  public void trackQueue(String queueName) {
-trackedQueues.add(queueName);
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
index 7b306f0..a5aee74 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
@@ -167,8 +167,9 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
   }
 
   @Override
-  public void trackQueue(String queueName) {
-trackedQueues.add(queueName);
+  protected void registerQueueMetrics(String queueName) {
+super.registerQueueMetrics(queueName);
+
 FairScheduler fair = (FairScheduler) scheduler;
 final FSQueue queue = fair.getQueueManager().getQueue(queueName);
 registerQueueMetrics(queue, Metric.DEMAND);
@@ -209,16 +210,4 @@ public class FairSchedulerMetrics extends SchedulerMetrics 
{
   }
 );
   }
-
-  @Override
-  public void untrackQueue(String queueName) {
-trackedQueues.remove(queueName);
-
-for (Metric metric: Metric.values()) {
-  metrics.remove("variable.queue." + queueName + "." +
-  metric.value + ".memory");
-  metrics.remove("variable.queue." + queueName + "." +
-  metric.value + ".vcores");
-}
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
 
b/hadoop-tools/hadoop-sls/src/ma

[03/50] [abbrv] hadoop git commit: YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)

2017-10-19 Thread templedf

YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)

(cherry picked from commit 46eb1033a86ca53e7b94202567aef8af2417bdf8)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/175c4472
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/175c4472
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/175c4472

Branch: refs/heads/resource-types
Commit: 175c4472d5b42bb972cd7da8469c9aa71b9bb0d1
Parents: 6959db9
Author: Haibo Chen 
Authored: Tue Oct 17 16:03:46 2017 -0700
Committer: Haibo Chen 
Committed: Tue Oct 17 16:04:49 2017 -0700

--
 .../org/apache/hadoop/yarn/sls/SLSRunner.java   |  1 +
 .../sls/scheduler/CapacitySchedulerMetrics.java |  5 --
 .../sls/scheduler/FairSchedulerMetrics.java | 17 +
 .../sls/scheduler/FifoSchedulerMetrics.java |  5 +-
 .../sls/scheduler/SLSCapacityScheduler.java | 19 --
 .../yarn/sls/scheduler/SLSFairScheduler.java| 19 --
 .../yarn/sls/scheduler/SchedulerMetrics.java| 66 +---
 7 files changed, 36 insertions(+), 96 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/175c4472/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 59f9c17..dfdf7c9 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -731,6 +731,7 @@ public class SLSRunner extends Configured implements Tool {
 }
 
 queueAppNumMap.put(queueName, appNum);
+wrapper.getSchedulerMetrics().trackQueue(queueName);
   }
 
   private void runNewAM(String jobType, String user,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/175c4472/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
index a73f48c..89b44c6 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
@@ -28,9 +28,4 @@ public class CapacitySchedulerMetrics extends 
SchedulerMetrics {
   public CapacitySchedulerMetrics() {
 super();
   }
-
-  @Override
-  public void trackQueue(String queueName) {
-trackedQueues.add(queueName);
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/175c4472/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
index 7b306f0..a5aee74 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
@@ -167,8 +167,9 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
   }
 
   @Override
-  public void trackQueue(String queueName) {
-trackedQueues.add(queueName);
+  protected void registerQueueMetrics(String queueName) {
+super.registerQueueMetrics(queueName);
+
 FairScheduler fair = (FairScheduler) scheduler;
 final FSQueue queue = fair.getQueueManager().getQueue(queueName);
 registerQueueMetrics(queue, Metric.DEMAND);
@@ -209,16 +210,4 @@ public class FairSchedulerMetrics extends SchedulerMetrics 
{
   }
 );
   }
-
-  @Override
-  public void untrackQueue(String queueName) {
-trackedQueues.remove(queueName);
-
-for (Metric metric: Metric.values()) {
-  metrics.remove("variable.queue." + queueName + "." +
-  metric.value + ".memory");
-  metrics.remove("variable.queue." + queueName + "." +
-  metric.value + ".vcores");
-}
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/175c4472/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn

hadoop git commit: YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)

2017-10-17 Thread haibochen

Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 6959db9c2 -> 175c4472d


YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)

(cherry picked from commit 46eb1033a86ca53e7b94202567aef8af2417bdf8)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/175c4472
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/175c4472
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/175c4472

Branch: refs/heads/branch-3.0
Commit: 175c4472d5b42bb972cd7da8469c9aa71b9bb0d1
Parents: 6959db9
Author: Haibo Chen 
Authored: Tue Oct 17 16:03:46 2017 -0700
Committer: Haibo Chen 
Committed: Tue Oct 17 16:04:49 2017 -0700

--
 .../org/apache/hadoop/yarn/sls/SLSRunner.java   |  1 +
 .../sls/scheduler/CapacitySchedulerMetrics.java |  5 --
 .../sls/scheduler/FairSchedulerMetrics.java | 17 +
 .../sls/scheduler/FifoSchedulerMetrics.java |  5 +-
 .../sls/scheduler/SLSCapacityScheduler.java | 19 --
 .../yarn/sls/scheduler/SLSFairScheduler.java| 19 --
 .../yarn/sls/scheduler/SchedulerMetrics.java| 66 +---
 7 files changed, 36 insertions(+), 96 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/175c4472/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 59f9c17..dfdf7c9 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -731,6 +731,7 @@ public class SLSRunner extends Configured implements Tool {
 }
 
 queueAppNumMap.put(queueName, appNum);
+wrapper.getSchedulerMetrics().trackQueue(queueName);
   }
 
   private void runNewAM(String jobType, String user,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/175c4472/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
index a73f48c..89b44c6 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
@@ -28,9 +28,4 @@ public class CapacitySchedulerMetrics extends 
SchedulerMetrics {
   public CapacitySchedulerMetrics() {
 super();
   }
-
-  @Override
-  public void trackQueue(String queueName) {
-trackedQueues.add(queueName);
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/175c4472/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
index 7b306f0..a5aee74 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
@@ -167,8 +167,9 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
   }
 
   @Override
-  public void trackQueue(String queueName) {
-trackedQueues.add(queueName);
+  protected void registerQueueMetrics(String queueName) {
+super.registerQueueMetrics(queueName);
+
 FairScheduler fair = (FairScheduler) scheduler;
 final FSQueue queue = fair.getQueueManager().getQueue(queueName);
 registerQueueMetrics(queue, Metric.DEMAND);
@@ -209,16 +210,4 @@ public class FairSchedulerMetrics extends SchedulerMetrics 
{
   }
 );
   }
-
-  @Override
-  public void untrackQueue(String queueName) {
-trackedQueues.remove(queueName);
-
-for (Metric metric: Metric.values()) {
-  metrics.remove("variable.queue." + queueName + "." +
-  metric.value + ".memory");
-  metrics.remove("variable.queue." + queueName + "." +
-  metric.value + ".vcores");
-}
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/175c4472/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
---

hadoop git commit: YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)

2017-10-17 Thread haibochen

Repository: hadoop
Updated Branches:
  refs/heads/trunk f27a4ad03 -> 46eb1033a


YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46eb1033
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46eb1033
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46eb1033

Branch: refs/heads/trunk
Commit: 46eb1033a86ca53e7b94202567aef8af2417bdf8
Parents: f27a4ad
Author: Haibo Chen 
Authored: Tue Oct 17 16:03:46 2017 -0700
Committer: Haibo Chen 
Committed: Tue Oct 17 16:04:19 2017 -0700

--
 .../org/apache/hadoop/yarn/sls/SLSRunner.java   |  1 +
 .../sls/scheduler/CapacitySchedulerMetrics.java |  5 --
 .../sls/scheduler/FairSchedulerMetrics.java | 17 +
 .../sls/scheduler/FifoSchedulerMetrics.java |  5 +-
 .../sls/scheduler/SLSCapacityScheduler.java | 19 --
 .../yarn/sls/scheduler/SLSFairScheduler.java| 19 --
 .../yarn/sls/scheduler/SchedulerMetrics.java| 66 +---
 7 files changed, 36 insertions(+), 96 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 59f9c17..dfdf7c9 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -731,6 +731,7 @@ public class SLSRunner extends Configured implements Tool {
 }
 
 queueAppNumMap.put(queueName, appNum);
+wrapper.getSchedulerMetrics().trackQueue(queueName);
   }
 
   private void runNewAM(String jobType, String user,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
index a73f48c..89b44c6 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java
@@ -28,9 +28,4 @@ public class CapacitySchedulerMetrics extends 
SchedulerMetrics {
   public CapacitySchedulerMetrics() {
 super();
   }
-
-  @Override
-  public void trackQueue(String queueName) {
-trackedQueues.add(queueName);
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
index 7b306f0..a5aee74 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java
@@ -167,8 +167,9 @@ public class FairSchedulerMetrics extends SchedulerMetrics {
   }
 
   @Override
-  public void trackQueue(String queueName) {
-trackedQueues.add(queueName);
+  protected void registerQueueMetrics(String queueName) {
+super.registerQueueMetrics(queueName);
+
 FairScheduler fair = (FairScheduler) scheduler;
 final FSQueue queue = fair.getQueueManager().getQueue(queueName);
 registerQueueMetrics(queue, Metric.DEMAND);
@@ -209,16 +210,4 @@ public class FairSchedulerMetrics extends SchedulerMetrics 
{
   }
 );
   }
-
-  @Override
-  public void untrackQueue(String queueName) {
-trackedQueues.remove(queueName);
-
-for (Metric metric: Metric.values()) {
-  metrics.remove("variable.queue." + queueName + "." +
-  metric.value + ".memory");
-  metrics.remove("variable.queue." + queueName + "." +
-  metric.value + ".vcores");
-}
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46eb1033/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/ya

[17/50] hadoop git commit: YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)

[33/50] [abbrv] hadoop git commit: YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)

[03/50] [abbrv] hadoop git commit: YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)

hadoop git commit: YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)

hadoop git commit: YARN-6546. SLS is slow while loading 10k queues. (Yufei Gu via Haibo Chen)

5 matches

Site Navigation

Mail list logo

Footer information