Repository: hadoop Updated Branches: refs/heads/branch-3.0 0e31e014f -> f12c78120
YARN-8212. Pending backlog for async allocation threads should be configurable. Contributed by Tao Yang. (Cherry picked from commit 2d319e37937c1e20c6a7dc4477ef88defd1f8464) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f12c7812 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f12c7812 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f12c7812 Branch: refs/heads/branch-3.0 Commit: f12c78120edf8679407a43857f5caa310bc556ca Parents: 0e31e01 Author: Weiwei Yang <w...@apache.org> Authored: Tue May 1 09:47:10 2018 +0800 Committer: Weiwei Yang <w...@apache.org> Committed: Tue May 1 10:19:08 2018 +0800 ---------------------------------------------------------------------- .../scheduler/capacity/CapacityScheduler.java | 9 ++++++++- .../scheduler/capacity/CapacitySchedulerConfiguration.java | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f12c7812/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 5a29e5d..3587f30 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -245,6 +245,7 @@ public class CapacityScheduler extends CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".scheduling-interval-ms"; private static final long DEFAULT_ASYNC_SCHEDULER_INTERVAL = 5; + private long asyncMaxPendingBacklogs; public CapacityScheduler() { super(CapacityScheduler.class.getName()); @@ -354,6 +355,11 @@ public class CapacityScheduler extends asyncSchedulerThreads.add(new AsyncScheduleThread(this)); } resourceCommitterService = new ResourceCommitterService(this); + asyncMaxPendingBacklogs = this.conf.getInt( + CapacitySchedulerConfiguration. + SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS, + CapacitySchedulerConfiguration. + DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS); } // Setup how many containers we can allocate for each round @@ -541,7 +547,8 @@ public class CapacityScheduler extends Thread.sleep(100); } else { // Don't run schedule if we have some pending backlogs already - if (cs.getAsyncSchedulingPendingBacklogs() > 100) { + if (cs.getAsyncSchedulingPendingBacklogs() + > cs.asyncMaxPendingBacklogs) { Thread.sleep(1); } else{ schedule(cs); http://git-wip-us.apache.org/repos/asf/hadoop/blob/f12c7812/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 1e22e0b..c05b948 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -240,6 +240,14 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".maximum-threads"; @Private + public static final String SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS = + SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".maximum-pending-backlogs"; + + @Private + public static final Integer + DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS = 100; + + @Private public static final boolean DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE = false; @Private --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
