YARN-5703. ReservationAgents are not correctly configured. Contributed by Manikandan R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f5b031d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f5b031d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f5b031d Branch: refs/heads/HDFS-7240 Commit: 5f5b031d1f20cb7f621db41979e963eaa42cf52f Parents: 4d33683 Author: Naganarasimha <naganarasimha...@apache.org> Authored: Mon Feb 27 20:38:29 2017 +0530 Committer: Naganarasimha <naganarasimha...@apache.org> Committed: Mon Feb 27 20:38:29 2017 +0530 ---------------------------------------------------------------------- .../reservation/AbstractReservationSystem.java | 8 ++++++-- .../planning/AlignedPlannerWithGreedy.java | 16 +++++++++------- .../planning/GreedyReservationAgent.java | 19 ++++++------------- .../reservation/planning/PlanningAlgorithm.java | 4 ++++ .../reservation/planning/ReservationAgent.java | 8 ++++++++ .../planning/TryManyReservationAgents.java | 5 ++++- .../reservation/planning/TestAlignedPlanner.java | 3 +++ .../planning/TestGreedyReservationAgent.java | 4 ++-- 8 files changed, 42 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java index 8769ca1..5ef4912 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java @@ -451,12 +451,16 @@ public abstract class AbstractReservationSystem extends AbstractService try { Class<?> agentClazz = conf.getClassByName(agentClassName); if (ReservationAgent.class.isAssignableFrom(agentClazz)) { - return (ReservationAgent) ReflectionUtils.newInstance(agentClazz, conf); + ReservationAgent resevertionAgent = + (ReservationAgent) agentClazz.newInstance(); + resevertionAgent.init(conf); + return resevertionAgent; } else { throw new YarnRuntimeException("Class: " + agentClassName + " not instance of " + ReservationAgent.class.getCanonicalName()); } - } catch (ClassNotFoundException e) { + } catch (ClassNotFoundException | InstantiationException + | IllegalAccessException e) { throw new YarnRuntimeException("Could not instantiate Agent: " + agentClassName + " for queue: " + queueName, e); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java index b23cf1e..00c2333 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning; import java.util.LinkedList; import java.util.List; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan; @@ -35,22 +36,25 @@ import org.slf4j.LoggerFactory; public class AlignedPlannerWithGreedy implements ReservationAgent { // Default smoothness factor - private static final int DEFAULT_SMOOTHNESS_FACTOR = 10; + public static final int DEFAULT_SMOOTHNESS_FACTOR = 10; + public static final String SMOOTHNESS_FACTOR = + "yarn.resourcemanager.reservation-system.smoothness-factor"; // Log private static final Logger LOG = LoggerFactory .getLogger(AlignedPlannerWithGreedy.class); // Smoothness factor - private final ReservationAgent planner; + private ReservationAgent planner; // Constructor public AlignedPlannerWithGreedy() { - this(DEFAULT_SMOOTHNESS_FACTOR); } - // Constructor - public AlignedPlannerWithGreedy(int smoothnessFactor) { + @Override + public void init(Configuration conf) { + int smoothnessFactor = + conf.getInt(SMOOTHNESS_FACTOR, DEFAULT_SMOOTHNESS_FACTOR); // List of algorithms List<ReservationAgent> listAlg = new LinkedList<ReservationAgent>(); @@ -71,7 +75,6 @@ public class AlignedPlannerWithGreedy implements ReservationAgent { // 1. Attempt to execute algAligned // 2. If failed, fall back to algGreedy planner = new TryManyReservationAgents(listAlg); - } @Override @@ -119,5 +122,4 @@ public class AlignedPlannerWithGreedy implements ReservationAgent { return planner.deleteReservation(reservationId, user, plan); } - } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java index 915a834..1559b97 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java @@ -46,25 +46,19 @@ public class GreedyReservationAgent implements ReservationAgent { .getLogger(GreedyReservationAgent.class); // Greedy planner - private final ReservationAgent planner; - + private ReservationAgent planner; public final static String GREEDY_FAVOR_EARLY_ALLOCATION = "yarn.resourcemanager.reservation-system.favor-early-allocation"; - public final static boolean DEFAULT_GREEDY_FAVOR_EARLY_ALLOCATION = true; - - private final boolean allocateLeft; + private boolean allocateLeft; public GreedyReservationAgent() { - this(new Configuration()); } - public GreedyReservationAgent(Configuration yarnConfiguration) { - - allocateLeft = - yarnConfiguration.getBoolean(GREEDY_FAVOR_EARLY_ALLOCATION, - DEFAULT_GREEDY_FAVOR_EARLY_ALLOCATION); - + @Override + public void init(Configuration conf) { + allocateLeft = conf.getBoolean(GREEDY_FAVOR_EARLY_ALLOCATION, + DEFAULT_GREEDY_FAVOR_EARLY_ALLOCATION); if (allocateLeft) { LOG.info("Initializing the GreedyReservationAgent to favor \"early\"" + " (left) allocations (controlled by parameter: " @@ -78,7 +72,6 @@ public class GreedyReservationAgent implements ReservationAgent { planner = new IterativePlanner(new StageEarliestStartByJobArrival(), new StageAllocatorGreedyRLE(allocateLeft), allocateLeft); - } public boolean isAllocateLeft(){ http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java index e1b508d..199bfa5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning; import java.util.Map; import java.util.Map.Entry; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; @@ -206,4 +207,7 @@ public abstract class PlanningAlgorithm implements ReservationAgent { } + @Override + public void init(Configuration conf) { + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java index bdea2f4..52e7055 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java @@ -17,6 +17,7 @@ *******************************************************************************/ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan; @@ -70,4 +71,11 @@ public interface ReservationAgent { public boolean deleteReservation(ReservationId reservationId, String user, Plan plan) throws PlanningException; + /** + * Init configuration. + * + * @param conf Configuration + */ + void init(Configuration conf); + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java index 1d37ce5..ab2e1e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning; import java.util.LinkedList; import java.util.List; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan; @@ -110,5 +111,7 @@ public class TryManyReservationAgents implements ReservationAgent { return plan.deleteReservation(reservationId); } - + @Override + public void init(Configuration conf) { + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java index 4b01eb9..2645366 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java @@ -726,8 +726,11 @@ public class TestAlignedPlanner { QueueMetrics queueMetrics = mock(QueueMetrics.class); RMContext context = ReservationSystemTestUtil.createMockRMContext(); + conf.setInt(AlignedPlannerWithGreedy.SMOOTHNESS_FACTOR, + AlignedPlannerWithGreedy.DEFAULT_SMOOTHNESS_FACTOR); // Set planning agent agent = new AlignedPlannerWithGreedy(); + agent.init(conf); // Create Plan plan = http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java index ec11ffe..6d1cfa8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java @@ -110,8 +110,8 @@ public class TestGreedyReservationAgent { // setting conf to conf.setBoolean(GreedyReservationAgent.GREEDY_FAVOR_EARLY_ALLOCATION, allocateLeft); - - agent = new GreedyReservationAgent(conf); + agent = new GreedyReservationAgent(); + agent.init(conf); QueueMetrics queueMetrics = mock(QueueMetrics.class); RMContext context = ReservationSystemTestUtil.createMockRMContext(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
