Repository: hadoop Updated Branches: refs/heads/trunk 680be58aa -> c0e492e50
YARN-5616. Clean up WeightAdjuster. (Yufei Gu via kasha) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0e492e5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0e492e5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0e492e5 Branch: refs/heads/trunk Commit: c0e492e50fa98d423c8a61c94d2d1f9553558b6d Parents: 680be58 Author: Karthik Kambatla <ka...@apache.org> Authored: Tue Sep 6 22:40:20 2016 -0700 Committer: Karthik Kambatla <ka...@apache.org> Committed: Tue Sep 6 22:40:20 2016 -0700 ---------------------------------------------------------------------- .../scheduler/fair/FairScheduler.java | 5 -- .../scheduler/fair/NewAppWeightBooster.java | 60 -------------------- .../scheduler/fair/WeightAdjuster.java | 36 ------------ 3 files changed, 101 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0e492e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index ac8dbe1..15e2318 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -186,7 +186,6 @@ public class FairScheduler extends // an app can be reserved on protected boolean sizeBasedWeight; // Give larger weights to larger jobs - protected WeightAdjuster weightAdjuster; // Can be null for no weight adjuster protected boolean continuousSchedulingEnabled; // Continuous Scheduling enabled or not protected int continuousSchedulingSleepMs; // Sleep time for each pass in continuous scheduling private Comparator<FSSchedulerNode> nodeAvailableResourceComparator = @@ -563,10 +562,6 @@ public class FairScheduler extends weight = Math.log1p(app.getDemand().getMemorySize()) / Math.log(2); } weight *= app.getPriority().getPriority(); - if (weightAdjuster != null) { - // Run weight through the user-supplied weightAdjuster - weight = weightAdjuster.adjustWeight(app, weight); - } ResourceWeights resourceWeights = app.getResourceWeights(); resourceWeights.setWeight((float)weight); return resourceWeights; http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0e492e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java deleted file mode 100644 index fb32e56..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; - -/** - * A {@link WeightAdjuster} implementation that gives a weight boost to new jobs - * for a certain amount of time -- by default, a 3x weight boost for 60 seconds. - * This can be used to make shorter jobs finish faster, emulating Shortest Job - * First scheduling while not starving long jobs. - */ -@Private -@Unstable -public class NewAppWeightBooster extends Configured implements WeightAdjuster { - private static final float DEFAULT_FACTOR = 3; - private static final long DEFAULT_DURATION = 5 * 60 * 1000; - - private float factor; - private long duration; - - public void setConf(Configuration conf) { - if (conf != null) { - factor = conf.getFloat("mapred.newjobweightbooster.factor", - DEFAULT_FACTOR); - duration = conf.getLong("mapred.newjobweightbooster.duration", - DEFAULT_DURATION); - } - super.setConf(conf); - } - - public double adjustWeight(FSAppAttempt app, double curWeight) { - long start = app.getStartTime(); - long now = System.currentTimeMillis(); - if (now - start < duration) { - return curWeight * factor; - } else { - return curWeight; - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0e492e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java deleted file mode 100644 index 67364ed..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.conf.Configurable; - -/** - * A pluggable object for altering the weights of apps in the fair scheduler, - * which is used for example by {@link NewAppWeightBooster} to give higher - * weight to new jobs so that short jobs finish faster. - * - * May implement {@link Configurable} to access configuration parameters. - */ -@Private -@Unstable -public interface WeightAdjuster { - public double adjustWeight(FSAppAttempt app, double curWeight); -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
