[
https://issues.apache.org/jira/browse/STORM-898?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15063483#comment-15063483
]
ASF GitHub Bot commented on STORM-898:
--------------------------------------
Github user d2r commented on a diff in the pull request:
https://github.com/apache/storm/pull/921#discussion_r47995347
--- Diff:
storm-core/test/jvm/backtype/storm/scheduler/resource/TestResourceAwareScheduler.java
---
@@ -0,0 +1,1166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package backtype.storm.scheduler.resource;
+
+import backtype.storm.Config;
+import backtype.storm.scheduler.Cluster;
+import backtype.storm.scheduler.ExecutorDetails;
+import backtype.storm.scheduler.INimbus;
+import backtype.storm.scheduler.IScheduler;
+import backtype.storm.scheduler.SchedulerAssignment;
+import backtype.storm.scheduler.SchedulerAssignmentImpl;
+import backtype.storm.scheduler.SupervisorDetails;
+import backtype.storm.scheduler.Topologies;
+import backtype.storm.scheduler.TopologyDetails;
+import backtype.storm.scheduler.WorkerSlot;
+import backtype.storm.utils.Time;
+import backtype.storm.utils.Utils;
+import backtype.storm.validation.ConfigValidation;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+public class TestResourceAwareScheduler {
+
+ private static final int NUM_SUPS = 20;
+ private static final int NUM_WORKERS_PER_SUP = 4;
+ private final String TOPOLOGY_SUBMITTER = "jerry";
+
+ private static final Logger LOG =
LoggerFactory.getLogger(TestResourceAwareScheduler.class);
+
+ @Test
+ public void TestReadInResourceAwareSchedulerUserPools() {
+
+ Map fromFile =
Utils.findAndReadConfigFile("user-resource-pools.yaml", false);
+ LOG.info("fromFile: {}", fromFile);
+ ConfigValidation.validateFields(fromFile);
+ }
+
+ @Test
+ public void TestTopologySortedInCorrectOrder() {
+ INimbus iNimbus = new
TestUtilsForResourceAwareScheduler.INimbusTest();
+ Map<String, Number> resourceMap = new HashMap<String, Number>();
+ resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 100.0);
+ resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1024.0);
+ Map<String, SupervisorDetails> supMap =
TestUtilsForResourceAwareScheduler.genSupervisors(20, 4, resourceMap);
+ Config config = new Config();
+ config.putAll(Utils.readDefaultConfig());
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY,
backtype.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy.class.getName());
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY,
backtype.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy.class.getName());
+ config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY,
backtype.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class.getName());
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, TOPOLOGY_SUBMITTER);
+
+ Map<String, Map<String, Number>> resourceUserPool = new
HashMap<String, Map<String, Number>>();
+ resourceUserPool.put("jerry", new HashMap<String, Number>());
+ resourceUserPool.get("jerry").put("cpu", 1000);
+ resourceUserPool.get("jerry").put("memory", 8192.0);
+
+ resourceUserPool.put("bobby", new HashMap<String, Number>());
+ resourceUserPool.get("bobby").put("cpu", 10000.0);
+ resourceUserPool.get("bobby").put("memory", 32768);
+
+ resourceUserPool.put("derek", new HashMap<String, Number>());
+ resourceUserPool.get("derek").put("cpu", 5000.0);
+ resourceUserPool.get("derek").put("memory", 16384.0);
+
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_USER_POOLS,
resourceUserPool);
+
+ TopologyDetails topo1 =
TestUtilsForResourceAwareScheduler.getTopology("topo-1", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 2, 20);
+ TopologyDetails topo2 =
TestUtilsForResourceAwareScheduler.getTopology("topo-2", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 8, 30);
+ TopologyDetails topo3 =
TestUtilsForResourceAwareScheduler.getTopology("topo-3", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 16, 30);
+ TopologyDetails topo4 =
TestUtilsForResourceAwareScheduler.getTopology("topo-4", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 16, 20);
+ TopologyDetails topo5 =
TestUtilsForResourceAwareScheduler.getTopology("topo-5", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 24, 30);
+
+ Map<String, TopologyDetails> topoMap = new HashMap<String,
TopologyDetails>();
+ topoMap.put(topo1.getId(), topo1);
+ topoMap.put(topo2.getId(), topo2);
+ topoMap.put(topo3.getId(), topo3);
+ topoMap.put(topo4.getId(), topo4);
+ topoMap.put(topo5.getId(), topo5);
+
+ Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String,
SchedulerAssignmentImpl>(), config);
+
+ Topologies topologies = new Topologies(topoMap);
+
+ ResourceAwareScheduler rs = new ResourceAwareScheduler();
+
+ rs.prepare(config);
+ rs.schedule(topologies, cluster);
+
+ Set<TopologyDetails> queue =
rs.getUser("jerry").getTopologiesPending();
+ Assert.assertEquals("check size", queue.size(), 0);
+
+ queue = rs.getUser("jerry").getTopologiesRunning();
+
+ Iterator<TopologyDetails> itr = queue.iterator();
+
+ TopologyDetails topo = itr.next();
+ LOG.info("{} - {}", topo.getName(), queue);
+ Assert.assertEquals("check order", topo.getName(), "topo-4");
+
+ topo = itr.next();
+ LOG.info("{} - {}", topo.getName(), queue);
+ Assert.assertEquals("check order", topo.getName(), "topo-1");
+
+ topo = itr.next();
+ LOG.info("{} - {}", topo.getName(), queue);
+ Assert.assertEquals("check order", topo.getName(), "topo-5");
+
+ topo = itr.next();
+ LOG.info("{} - {}", topo.getName(), queue);
+ Assert.assertEquals("check order", topo.getName(), "topo-3");
+
+ topo = itr.next();
+ LOG.info("{} - {}", topo.getName(), queue);
+ Assert.assertEquals("check order", topo.getName(), "topo-2");
+
+ TopologyDetails topo6 =
TestUtilsForResourceAwareScheduler.getTopology("topo-6", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 30, 10);
+ topoMap.put(topo6.getId(), topo6);
+
+ topologies = new Topologies(topoMap);
+ rs.prepare(config);
+ rs.schedule(topologies, cluster);
+
+ queue = rs.getUser("jerry").getTopologiesRunning();
+ itr = queue.iterator();
+
+ topo = itr.next();
+ Assert.assertEquals("check order", topo.getName(), "topo-6");
+
+ topo = itr.next();
+ Assert.assertEquals("check order", topo.getName(), "topo-4");
+
+ topo = itr.next();
+ Assert.assertEquals("check order", topo.getName(), "topo-1");
+
+ topo = itr.next();
+ Assert.assertEquals("check order", topo.getName(), "topo-5");
+
+ topo = itr.next();
+ Assert.assertEquals("check order", topo.getName(), "topo-3");
+
+ topo = itr.next();
+ Assert.assertEquals("check order", topo.getName(), "topo-2");
+
+ queue = rs.getUser("jerry").getTopologiesPending();
+ Assert.assertEquals("check size", queue.size(), 0);
+ }
+
+ @Test
+ public void TestMultipleUsers() {
+ INimbus iNimbus = new
TestUtilsForResourceAwareScheduler.INimbusTest();
+ Map<String, Number> resourceMap = new HashMap<String, Number>();
+ resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 1000.0);
+ resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1024.0 * 10);
+ Map<String, SupervisorDetails> supMap =
TestUtilsForResourceAwareScheduler.genSupervisors(20, 4, resourceMap);
+ Config config = new Config();
+ config.putAll(Utils.readDefaultConfig());
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY,
backtype.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy.class.getName());
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY,
backtype.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy.class.getName());
+ config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY,
backtype.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class.getName());
+ Map<String, Map<String, Number>> resourceUserPool = new
HashMap<String, Map<String, Number>>();
+ resourceUserPool.put("jerry", new HashMap<String, Number>());
+ resourceUserPool.get("jerry").put("cpu", 1000);
+ resourceUserPool.get("jerry").put("memory", 8192.0);
+
+ resourceUserPool.put("bobby", new HashMap<String, Number>());
+ resourceUserPool.get("bobby").put("cpu", 10000.0);
+ resourceUserPool.get("bobby").put("memory", 32768);
+
+ resourceUserPool.put("derek", new HashMap<String, Number>());
+ resourceUserPool.get("derek").put("cpu", 5000.0);
+ resourceUserPool.get("derek").put("memory", 16384.0);
+
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_USER_POOLS,
resourceUserPool);
+ Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String,
SchedulerAssignmentImpl>(), config);
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "jerry");
+
+ TopologyDetails topo1 =
TestUtilsForResourceAwareScheduler.getTopology("topo-1", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 2, 20);
+ TopologyDetails topo2 =
TestUtilsForResourceAwareScheduler.getTopology("topo-2", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 8, 29);
+ TopologyDetails topo3 =
TestUtilsForResourceAwareScheduler.getTopology("topo-3", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 16, 29);
+ TopologyDetails topo4 =
TestUtilsForResourceAwareScheduler.getTopology("topo-4", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 16, 20);
+ TopologyDetails topo5 =
TestUtilsForResourceAwareScheduler.getTopology("topo-5", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 24, 29);
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "bobby");
+
+ TopologyDetails topo6 =
TestUtilsForResourceAwareScheduler.getTopology("topo-6", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 2, 20);
+ TopologyDetails topo7 =
TestUtilsForResourceAwareScheduler.getTopology("topo-7", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 8, 29);
+ TopologyDetails topo8 =
TestUtilsForResourceAwareScheduler.getTopology("topo-8", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 16, 29);
+ TopologyDetails topo9 =
TestUtilsForResourceAwareScheduler.getTopology("topo-9", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 16, 20);
+ TopologyDetails topo10 =
TestUtilsForResourceAwareScheduler.getTopology("topo-10", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 24, 29);
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "derek");
+
+ TopologyDetails topo11 =
TestUtilsForResourceAwareScheduler.getTopology("topo-11", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 2, 20);
+ TopologyDetails topo12 =
TestUtilsForResourceAwareScheduler.getTopology("topo-12", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 8, 29);
+ TopologyDetails topo13 =
TestUtilsForResourceAwareScheduler.getTopology("topo-13", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 16, 29);
+ TopologyDetails topo14 =
TestUtilsForResourceAwareScheduler.getTopology("topo-14", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 16, 20);
+ TopologyDetails topo15 =
TestUtilsForResourceAwareScheduler.getTopology("topo-15", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 24, 29);
+
+ Map<String, TopologyDetails> topoMap = new HashMap<String,
TopologyDetails>();
+ topoMap.put(topo1.getId(), topo1);
+ topoMap.put(topo2.getId(), topo2);
+ topoMap.put(topo3.getId(), topo3);
+ topoMap.put(topo4.getId(), topo4);
+ topoMap.put(topo5.getId(), topo5);
+ topoMap.put(topo6.getId(), topo6);
+ topoMap.put(topo7.getId(), topo7);
+ topoMap.put(topo8.getId(), topo8);
+ topoMap.put(topo9.getId(), topo9);
+ topoMap.put(topo10.getId(), topo10);
+ topoMap.put(topo11.getId(), topo11);
+ topoMap.put(topo12.getId(), topo12);
+ topoMap.put(topo13.getId(), topo13);
+ topoMap.put(topo14.getId(), topo14);
+ topoMap.put(topo15.getId(), topo15);
+
+ Topologies topologies = new Topologies(topoMap);
+
+ ResourceAwareScheduler rs = new ResourceAwareScheduler();
+
+ rs.prepare(config);
+ rs.schedule(topologies, cluster);
+
+ for (TopologyDetails topo : topoMap.values()) {
+
Assert.assertTrue(TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+
+ for (User user : rs.getUserMap().values()) {
+ Assert.assertEquals(user.getTopologiesPending().size(), 0);
+ Assert.assertEquals(user.getTopologiesRunning().size(), 5);
+ }
+ }
+
+ @Test
+ public void testHandlingClusterSubscription() {
+ INimbus iNimbus = new
TestUtilsForResourceAwareScheduler.INimbusTest();
+ Map<String, Number> resourceMap = new HashMap<String, Number>();
+ resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 200.0);
+ resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1024.0 * 10);
+ Map<String, SupervisorDetails> supMap =
TestUtilsForResourceAwareScheduler.genSupervisors(1, 4, resourceMap);
+ Config config = new Config();
+ config.putAll(Utils.readDefaultConfig());
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY,
backtype.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy.class.getName());
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY,
backtype.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy.class.getName());
+ config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY,
backtype.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class.getName());
+ Map<String, Map<String, Number>> resourceUserPool = new
HashMap<String, Map<String, Number>>();
+ resourceUserPool.put("jerry", new HashMap<String, Number>());
+ resourceUserPool.get("jerry").put("cpu", 1000);
+ resourceUserPool.get("jerry").put("memory", 8192.0);
+
+ resourceUserPool.put("bobby", new HashMap<String, Number>());
+ resourceUserPool.get("bobby").put("cpu", 10000.0);
+ resourceUserPool.get("bobby").put("memory", 32768);
+
+ resourceUserPool.put("derek", new HashMap<String, Number>());
+ resourceUserPool.get("derek").put("cpu", 5000.0);
+ resourceUserPool.get("derek").put("memory", 16384.0);
+
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_USER_POOLS,
resourceUserPool);
+ Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String,
SchedulerAssignmentImpl>(), config);
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "jerry");
+
+ TopologyDetails topo1 =
TestUtilsForResourceAwareScheduler.getTopology("topo-1", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 2, 20);
+ TopologyDetails topo2 =
TestUtilsForResourceAwareScheduler.getTopology("topo-2", config, 5, 15, 1, 1,
Time.currentTimeSecs() - 8, 29);
+
+ Map<String, TopologyDetails> topoMap = new HashMap<String,
TopologyDetails>();
+ topoMap.put(topo1.getId(), topo1);
+ topoMap.put(topo2.getId(), topo2);
+
+ Topologies topologies = new Topologies(topoMap);
+
+ ResourceAwareScheduler rs = new ResourceAwareScheduler();
+
+ rs.prepare(config);
+ rs.schedule(topologies, cluster);
+
+ int fullyScheduled = 0;
+ for (TopologyDetails topo : topoMap.values()) {
+ if
(TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())))
{
+ fullyScheduled++;
+ }
+ }
+ Assert.assertEquals("# of Fully scheduled", 1, fullyScheduled);
+ Assert.assertEquals("# of topologies schedule attempted", 1,
rs.getUser("jerry").getTopologiesAttempted().size());
+ Assert.assertEquals("# of topologies running", 1,
rs.getUser("jerry").getTopologiesRunning().size());
+ Assert.assertEquals("# of topologies schedule pending", 0,
rs.getUser("jerry").getTopologiesPending().size());
+ }
+
+ /**
+ * The resources in the cluster is limited. In the first round of
scheduling, all resources in the cluster is used.
+ * User jerry submits another toplogy. Since user jerry has has his
resource guarantees satisfied, and user bobby
+ * has exceeded his resource guarantee, topo-3 from user bobby should
be eviced.
+ */
+ @Test
+ public void testEviction() {
+ INimbus iNimbus = new
TestUtilsForResourceAwareScheduler.INimbusTest();
+ Map<String, Number> resourceMap = new HashMap<String, Number>();
+ resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 100.0);
+ resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1000.0);
+ Map<String, SupervisorDetails> supMap =
TestUtilsForResourceAwareScheduler.genSupervisors(4, 4, resourceMap);
+ Config config = new Config();
+ config.putAll(Utils.readDefaultConfig());
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY,
backtype.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy.class.getName());
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY,
backtype.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy.class.getName());
+ config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY,
backtype.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class.getName());
+ config.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 100.0);
+ config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB,
500);
+ config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB,
500);
+ Map<String, Map<String, Number>> resourceUserPool = new
HashMap<String, Map<String, Number>>();
+ resourceUserPool.put("jerry", new HashMap<String, Number>());
+ resourceUserPool.get("jerry").put("cpu", 200.0);
+ resourceUserPool.get("jerry").put("memory", 2000.0);
+
+ resourceUserPool.put("bobby", new HashMap<String, Number>());
+ resourceUserPool.get("bobby").put("cpu", 100.0);
+ resourceUserPool.get("bobby").put("memory", 1000.0);
+
+ resourceUserPool.put("derek", new HashMap<String, Number>());
+ resourceUserPool.get("derek").put("cpu", 200.0);
+ resourceUserPool.get("derek").put("memory", 2000.0);
+
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_USER_POOLS,
resourceUserPool);
+ Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String,
SchedulerAssignmentImpl>(), config);
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "jerry");
+
+ TopologyDetails topo1 =
TestUtilsForResourceAwareScheduler.getTopology("topo-1", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 10);
+ TopologyDetails topo6 =
TestUtilsForResourceAwareScheduler.getTopology("topo-6", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 20);
+
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "bobby");
+
+ TopologyDetails topo2 =
TestUtilsForResourceAwareScheduler.getTopology("topo-2", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 10);
+ TopologyDetails topo3 =
TestUtilsForResourceAwareScheduler.getTopology("topo-3", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 20);
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "derek");
+
+ TopologyDetails topo4 =
TestUtilsForResourceAwareScheduler.getTopology("topo-4", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 29);
+
+ Map<String, TopologyDetails> topoMap = new HashMap<String,
TopologyDetails>();
+ topoMap.put(topo1.getId(), topo1);
+ topoMap.put(topo2.getId(), topo2);
+ topoMap.put(topo3.getId(), topo3);
+ topoMap.put(topo4.getId(), topo4);
+
+ Topologies topologies = new Topologies(topoMap);
+
+ ResourceAwareScheduler rs = new ResourceAwareScheduler();
+
+ rs.prepare(config);
+ rs.schedule(topologies, cluster);
+
+ for (TopologyDetails topo :
rs.getUser("jerry").getTopologiesRunning()) {
+ Assert.assertTrue("correct topology to evict",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 1,
rs.getUser("jerry").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("jerry").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("jerry").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("jerry").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo :
rs.getUser("derek").getTopologiesRunning()) {
+ Assert.assertTrue("correct topology to evict",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 1,
rs.getUser("derek").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("derek").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("derek").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("derek").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo :
rs.getUser("bobby").getTopologiesRunning()) {
+ Assert.assertTrue("correct topology to evict",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 2,
rs.getUser("bobby").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("bobby").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("bobby").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("bobby").getTopologiesInvalid().size());
+
+ //user jerry submits another topology
+ topoMap.put(topo6.getId(), topo6);
+ topologies = new Topologies(topoMap);
+
+ rs.schedule(topologies, cluster);
+
+ for (TopologyDetails topo :
rs.getUser("jerry").getTopologiesRunning()) {
+ Assert.assertTrue("correct topology to evict",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 2,
rs.getUser("jerry").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("jerry").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("jerry").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("jerry").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo :
rs.getUser("derek").getTopologiesRunning()) {
+ Assert.assertTrue("correct topology to evict",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 1,
rs.getUser("derek").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("derek").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("derek").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("derek").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo :
rs.getUser("bobby").getTopologiesRunning()) {
+ Assert.assertTrue("correct topology to evict",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 1,
rs.getUser("bobby").getTopologiesRunning().size());
+
+ for (TopologyDetails topo :
rs.getUser("bobby").getTopologiesAttempted()) {
+ Assert.assertFalse("correct topology to evict",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of attempted topologies", 1,
rs.getUser("bobby").getTopologiesAttempted().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("bobby").getTopologiesPending().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("bobby").getTopologiesInvalid().size());
+ }
+
+ @Test
+ public void TestEvictMultipleTopologies() {
+ INimbus iNimbus = new
TestUtilsForResourceAwareScheduler.INimbusTest();
+ Map<String, Number> resourceMap = new HashMap<String, Number>();
+ resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 100.0);
+ resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1000.0);
+ Map<String, SupervisorDetails> supMap =
TestUtilsForResourceAwareScheduler.genSupervisors(4, 4, resourceMap);
+ Config config = new Config();
+ config.putAll(Utils.readDefaultConfig());
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY,
backtype.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy.class.getName());
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY,
backtype.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy.class.getName());
+ config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY,
backtype.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class.getName());
+ config.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 100.0);
+ config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB,
500);
+ config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB,
500);
+ Map<String, Map<String, Number>> resourceUserPool = new
HashMap<String, Map<String, Number>>();
+ resourceUserPool.put("jerry", new HashMap<String, Number>());
+ resourceUserPool.get("jerry").put("cpu", 200.0);
+ resourceUserPool.get("jerry").put("memory", 2000.0);
+
+ resourceUserPool.put("derek", new HashMap<String, Number>());
+ resourceUserPool.get("derek").put("cpu", 100.0);
+ resourceUserPool.get("derek").put("memory", 1000.0);
+
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_USER_POOLS,
resourceUserPool);
+ Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String,
SchedulerAssignmentImpl>(), config);
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "jerry");
+
+ TopologyDetails topo1 =
TestUtilsForResourceAwareScheduler.getTopology("topo-1", config, 2, 0, 1, 0,
Time.currentTimeSecs() - 2, 10);
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "bobby");
+
+ TopologyDetails topo2 =
TestUtilsForResourceAwareScheduler.getTopology("topo-2", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 10);
+ TopologyDetails topo3 =
TestUtilsForResourceAwareScheduler.getTopology("topo-3", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 20);
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "derek");
+
+ TopologyDetails topo4 =
TestUtilsForResourceAwareScheduler.getTopology("topo-4", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 29);
+ TopologyDetails topo5 =
TestUtilsForResourceAwareScheduler.getTopology("topo-5", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 29);
+
+ Map<String, TopologyDetails> topoMap = new HashMap<String,
TopologyDetails>();
+ topoMap.put(topo2.getId(), topo2);
+ topoMap.put(topo3.getId(), topo3);
+ topoMap.put(topo4.getId(), topo4);
+ topoMap.put(topo5.getId(), topo5);
+
+ Topologies topologies = new Topologies(topoMap);
+
+ ResourceAwareScheduler rs = new ResourceAwareScheduler();
+
+ rs.prepare(config);
+ rs.schedule(topologies, cluster);
+
+ for (TopologyDetails topo :
rs.getUser("derek").getTopologiesRunning()) {
+ Assert.assertTrue("correct topology to evict",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 2,
rs.getUser("derek").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("derek").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("derek").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("derek").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo :
rs.getUser("bobby").getTopologiesRunning()) {
+ Assert.assertTrue("correct topology to evict",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 2,
rs.getUser("bobby").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("bobby").getTopologiesPending().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("bobby").getTopologiesInvalid().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("bobby").getTopologiesAttempted().size());
+
+ //user jerry submits another topology
+ topoMap.put(topo1.getId(), topo1);
+ topologies = new Topologies(topoMap);
+ rs.schedule(topologies, cluster);
+
+ for (TopologyDetails topo :
rs.getUser("jerry").getTopologiesRunning()) {
+ Assert.assertTrue("correct topology to evict",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 1,
rs.getUser("jerry").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("jerry").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("jerry").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("jerry").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo :
rs.getUser("derek").getTopologiesRunning()) {
+ Assert.assertTrue("correct topology to evict",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 2,
rs.getUser("derek").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("derek").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("derek").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("derek").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo :
rs.getUser("bobby").getTopologiesAttempted()) {
+ Assert.assertFalse("correct topology to evict",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of attempted topologies", 2,
rs.getUser("bobby").getTopologiesAttempted().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("bobby").getTopologiesPending().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("bobby").getTopologiesInvalid().size());
+ Assert.assertEquals("# of running topologies", 0,
rs.getUser("bobby").getTopologiesRunning().size());
+ }
+
+ /**
+ * Eviction order:
+ * topo-3: since user bobby don't have any resource guarantees and
topo-3 is the lowest priority for user bobby
+ * topo-2: since user bobby don't have any resource guarantees and
topo-2 is the next lowest priority for user bobby
+ * topo-5: since user derek has exceeded his resource guarantee while
user jerry has not. topo-5 and topo-4 has the same priority
+ * but topo-4 was submitted earlier thus we choose that one to evict
+ */
+ @Test
+ public void
TestEvictMultipleTopologiesFromMultipleUsersInCorrectOrder() {
+ INimbus iNimbus = new
TestUtilsForResourceAwareScheduler.INimbusTest();
+ Map<String, Number> resourceMap = new HashMap<String, Number>();
+ resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 100.0);
+ resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1000.0);
+ Map<String, SupervisorDetails> supMap =
TestUtilsForResourceAwareScheduler.genSupervisors(4, 4, resourceMap);
+ Config config = new Config();
+ config.putAll(Utils.readDefaultConfig());
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY,
backtype.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy.class.getName());
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY,
backtype.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy.class.getName());
+ config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY,
backtype.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class.getName());
+ config.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 100.0);
+ config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB,
500);
+ config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB,
500);
+ Map<String, Map<String, Number>> resourceUserPool = new
HashMap<String, Map<String, Number>>();
+ resourceUserPool.put("jerry", new HashMap<String, Number>());
+ resourceUserPool.get("jerry").put("cpu", 300.0);
+ resourceUserPool.get("jerry").put("memory", 3000.0);
+
+ resourceUserPool.put("derek", new HashMap<String, Number>());
+ resourceUserPool.get("derek").put("cpu", 100.0);
+ resourceUserPool.get("derek").put("memory", 1000.0);
+
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_USER_POOLS,
resourceUserPool);
+ Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String,
SchedulerAssignmentImpl>(), config);
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "jerry");
+
+ TopologyDetails topo1 =
TestUtilsForResourceAwareScheduler.getTopology("topo-1", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 10);
+ TopologyDetails topo6 =
TestUtilsForResourceAwareScheduler.getTopology("topo-6", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 10);
+ TopologyDetails topo7 =
TestUtilsForResourceAwareScheduler.getTopology("topo-7", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 10);
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "bobby");
+
+ TopologyDetails topo2 =
TestUtilsForResourceAwareScheduler.getTopology("topo-2", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 10);
+ TopologyDetails topo3 =
TestUtilsForResourceAwareScheduler.getTopology("topo-3", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 20);
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "derek");
+
+ TopologyDetails topo4 =
TestUtilsForResourceAwareScheduler.getTopology("topo-4", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 29);
+ TopologyDetails topo5 =
TestUtilsForResourceAwareScheduler.getTopology("topo-5", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 15, 29);
+
+ Map<String, TopologyDetails> topoMap = new HashMap<String,
TopologyDetails>();
+ topoMap.put(topo2.getId(), topo2);
+ topoMap.put(topo3.getId(), topo3);
+ topoMap.put(topo4.getId(), topo4);
+ topoMap.put(topo5.getId(), topo5);
+
+ Topologies topologies = new Topologies(topoMap);
+
+ ResourceAwareScheduler rs = new ResourceAwareScheduler();
+
+ rs.prepare(config);
+ rs.schedule(topologies, cluster);
+
+ for (TopologyDetails topo :
rs.getUser("derek").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 2,
rs.getUser("derek").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("derek").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("derek").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("derek").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo :
rs.getUser("bobby").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 2,
rs.getUser("bobby").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("bobby").getTopologiesPending().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("bobby").getTopologiesInvalid().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("bobby").getTopologiesAttempted().size());
+
+ //user jerry submits another topology
+ topoMap.put(topo1.getId(), topo1);
+ topologies = new Topologies(topoMap);
+ rs.schedule(topologies, cluster);
+
+ for (TopologyDetails topo :
rs.getUser("jerry").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 1,
rs.getUser("jerry").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("jerry").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("jerry").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("jerry").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo :
rs.getUser("derek").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 2,
rs.getUser("derek").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("derek").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("derek").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("derek").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo :
rs.getUser("bobby").getTopologiesAttempted()) {
+ Assert.assertFalse("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ for (TopologyDetails topo :
rs.getUser("bobby").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of attempted topologies", 1,
rs.getUser("bobby").getTopologiesAttempted().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("bobby").getTopologiesPending().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("bobby").getTopologiesInvalid().size());
+ Assert.assertEquals("# of running topologies", 1,
rs.getUser("bobby").getTopologiesRunning().size());
+ Assert.assertEquals("correct topology to evict",
rs.getUser("bobby").getTopologiesAttempted().iterator().next().getName(),
"topo-3");
+
+ topoMap.put(topo6.getId(), topo6);
+ topologies = new Topologies(topoMap);
+ rs.schedule(topologies, cluster);
+
+ for (TopologyDetails topo :
rs.getUser("jerry").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 2,
rs.getUser("jerry").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("jerry").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("jerry").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("jerry").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo :
rs.getUser("derek").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 2,
rs.getUser("derek").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("derek").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("derek").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("derek").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo :
rs.getUser("bobby").getTopologiesAttempted()) {
+ Assert.assertFalse("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of attempted topologies", 2,
rs.getUser("bobby").getTopologiesAttempted().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("bobby").getTopologiesPending().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("bobby").getTopologiesInvalid().size());
+ Assert.assertEquals("# of running topologies", 0,
rs.getUser("bobby").getTopologiesRunning().size());
+
+ Assert.assertTrue("correct topology to evict",
TestUtilsForResourceAwareScheduler.findTopologyInSetFromName("topo-2",
rs.getUser("bobby").getTopologiesAttempted()) != null);
+ Assert.assertTrue("correct topology to evict",
TestUtilsForResourceAwareScheduler.findTopologyInSetFromName("topo-3",
rs.getUser("bobby").getTopologiesAttempted()) != null);
+
+ topoMap.put(topo7.getId(), topo7);
+ topologies = new Topologies(topoMap);
+ rs.schedule(topologies, cluster);
+
+ for (TopologyDetails topo :
rs.getUser("jerry").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 3,
rs.getUser("jerry").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("jerry").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("jerry").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("jerry").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo :
rs.getUser("derek").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ for (TopologyDetails topo :
rs.getUser("derek").getTopologiesAttempted()) {
+ Assert.assertFalse("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 1,
rs.getUser("derek").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("derek").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 1,
rs.getUser("derek").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("derek").getTopologiesInvalid().size());
+ Assert.assertEquals("correct topology to evict",
rs.getUser("derek").getTopologiesAttempted().iterator().next().getName(),
"topo-4");
+
+ for (TopologyDetails topo :
rs.getUser("bobby").getTopologiesAttempted()) {
+ Assert.assertFalse("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of attempted topologies", 2,
rs.getUser("bobby").getTopologiesAttempted().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("bobby").getTopologiesPending().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("bobby").getTopologiesInvalid().size());
+ Assert.assertEquals("# of running topologies", 0,
rs.getUser("bobby").getTopologiesRunning().size());
+
+ Assert.assertTrue("correct topology to evict",
TestUtilsForResourceAwareScheduler.findTopologyInSetFromName("topo-2",
rs.getUser("bobby").getTopologiesAttempted()) != null);
+ Assert.assertTrue("correct topology to evict",
TestUtilsForResourceAwareScheduler.findTopologyInSetFromName("topo-3",
rs.getUser("bobby").getTopologiesAttempted()) != null);
+ }
+
+ /**
+ * If topologies from other users cannot be evicted to make space
+ * check if there is a topology with lower priority that can be
evicted from the current user
+ */
+ @Test
+ public void TestEvictTopologyFromItself() {
+ INimbus iNimbus = new
TestUtilsForResourceAwareScheduler.INimbusTest();
+ Map<String, Number> resourceMap = new HashMap<String, Number>();
+ resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 100.0);
+ resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1000.0);
+ Map<String, SupervisorDetails> supMap =
TestUtilsForResourceAwareScheduler.genSupervisors(4, 4, resourceMap);
+ Config config = new Config();
+ config.putAll(Utils.readDefaultConfig());
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY,
backtype.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy.class.getName());
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY,
backtype.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy.class.getName());
+ config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY,
backtype.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class.getName());
+ config.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 100.0);
+ config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB,
500);
+ config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB,
500);
+ Map<String, Map<String, Number>> resourceUserPool = new
HashMap<String, Map<String, Number>>();
+ resourceUserPool.put("jerry", new HashMap<String, Number>());
+ resourceUserPool.get("jerry").put("cpu", 200.0);
+ resourceUserPool.get("jerry").put("memory", 2000.0);
+
+ resourceUserPool.put("bobby", new HashMap<String, Number>());
+ resourceUserPool.get("bobby").put("cpu", 100.0);
+ resourceUserPool.get("bobby").put("memory", 1000.0);
+
+ resourceUserPool.put("derek", new HashMap<String, Number>());
+ resourceUserPool.get("derek").put("cpu", 100.0);
+ resourceUserPool.get("derek").put("memory", 1000.0);
+
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_USER_POOLS,
resourceUserPool);
+ Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String,
SchedulerAssignmentImpl>(), config);
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "jerry");
+
+ TopologyDetails topo1 =
TestUtilsForResourceAwareScheduler.getTopology("topo-1", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 20);
+ TopologyDetails topo2 =
TestUtilsForResourceAwareScheduler.getTopology("topo-2", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 20);
+ TopologyDetails topo3 =
TestUtilsForResourceAwareScheduler.getTopology("topo-3", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 29);
+ TopologyDetails topo4 =
TestUtilsForResourceAwareScheduler.getTopology("topo-4", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 10);
+
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "bobby");
+
+ TopologyDetails topo5 =
TestUtilsForResourceAwareScheduler.getTopology("topo-5", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 10);
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "derek");
+
+ TopologyDetails topo6 =
TestUtilsForResourceAwareScheduler.getTopology("topo-6", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 29);
+
+ Map<String, TopologyDetails> topoMap = new HashMap<String,
TopologyDetails>();
+ topoMap.put(topo1.getId(), topo1);
+ topoMap.put(topo2.getId(), topo2);
+ topoMap.put(topo5.getId(), topo5);
+ topoMap.put(topo6.getId(), topo6);
+
+ Topologies topologies = new Topologies(topoMap);
+
+ ResourceAwareScheduler rs = new ResourceAwareScheduler();
+
+ rs.prepare(config);
+ rs.schedule(topologies, cluster);
+
+ for (TopologyDetails topo :
rs.getUser("jerry").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 2,
rs.getUser("jerry").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("jerry").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("jerry").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("jerry").getTopologiesInvalid().size());
+
+
+ for (TopologyDetails topo :
rs.getUser("derek").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 1,
rs.getUser("derek").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("derek").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("derek").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("derek").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo :
rs.getUser("bobby").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 1,
rs.getUser("bobby").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("bobby").getTopologiesPending().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("bobby").getTopologiesInvalid().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("bobby").getTopologiesAttempted().size());
+
+ //user jerry submits another topology into a full cluster
+ // topo3 should not be able to scheduled
+ topoMap.put(topo3.getId(), topo3);
+ topologies = new Topologies(topoMap);
+ rs.schedule(topologies, cluster);
+
+ for (TopologyDetails topo :
rs.getUser("jerry").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 2,
rs.getUser("jerry").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("jerry").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 1,
rs.getUser("jerry").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("jerry").getTopologiesInvalid().size());
+ //make sure that topo-3 didn't get scheduled.
+ Assert.assertEquals("correct topology in attempted queue",
rs.getUser("jerry").getTopologiesAttempted().iterator().next().getName(),
"topo-3");
+
+
+ for (TopologyDetails topo :
rs.getUser("derek").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 1,
rs.getUser("derek").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("derek").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("derek").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("derek").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo :
rs.getUser("bobby").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 1,
rs.getUser("bobby").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("bobby").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("bobby").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("bobby").getTopologiesInvalid().size());
+
+ //user jerry submits another topology but this one should be
scheduled since it has higher priority than than the
+ //rest of jerry's running topologies
+ topoMap.put(topo4.getId(), topo4);
+ topologies = new Topologies(topoMap);
+ rs.schedule(topologies, cluster);
+
+ for (TopologyDetails topo :
rs.getUser("jerry").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 2,
rs.getUser("jerry").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("jerry").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 2,
rs.getUser("jerry").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("jerry").getTopologiesInvalid().size());
+ Assert.assertTrue("correct topology in attempted queue",
TestUtilsForResourceAwareScheduler.findTopologyInSetFromName("topo-3",
rs.getUser("jerry").getTopologiesAttempted()) != null);
+ //Either topo-1 or topo-2 should have gotten evicted
+ Assert.assertTrue("correct topology in attempted queue",
((TestUtilsForResourceAwareScheduler.findTopologyInSetFromName("topo-1",
rs.getUser("jerry").getTopologiesAttempted())) != null)
+ ||
(TestUtilsForResourceAwareScheduler.findTopologyInSetFromName("topo-2",
rs.getUser("jerry").getTopologiesAttempted()) != null));
+ //assert that topo-4 got scheduled
+ Assert.assertTrue("correct topology in running queue",
TestUtilsForResourceAwareScheduler.findTopologyInSetFromName("topo-4",
rs.getUser("jerry").getTopologiesRunning()) != null);
+
+ for (TopologyDetails topo :
rs.getUser("derek").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 1,
rs.getUser("derek").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("derek").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("derek").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("derek").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo :
rs.getUser("bobby").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 1,
rs.getUser("bobby").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("bobby").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("bobby").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("bobby").getTopologiesInvalid().size());
+ }
+
+ /**
+ * If topologies from other users cannot be evicted to make space
+ * check if there is a topology with lower priority that can be
evicted from the current user
+ */
+ @Test
+ public void TestOverGuaranteeEviction() {
+ INimbus iNimbus = new
TestUtilsForResourceAwareScheduler.INimbusTest();
+ Map<String, Number> resourceMap = new HashMap<String, Number>();
+ resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 100.0);
+ resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1000.0);
+ Map<String, SupervisorDetails> supMap =
TestUtilsForResourceAwareScheduler.genSupervisors(4, 4, resourceMap);
+ Config config = new Config();
+ config.putAll(Utils.readDefaultConfig());
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY,
backtype.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy.class.getName());
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY,
backtype.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy.class.getName());
+ config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY,
backtype.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class.getName());
+ config.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 100.0);
+ config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB,
500);
+ config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB,
500);
+ Map<String, Map<String, Number>> resourceUserPool = new
HashMap<String, Map<String, Number>>();
+ resourceUserPool.put("jerry", new HashMap<String, Number>());
+ resourceUserPool.get("jerry").put("cpu", 70.0);
+ resourceUserPool.get("jerry").put("memory", 700.0);
+
+ resourceUserPool.put("bobby", new HashMap<String, Number>());
+ resourceUserPool.get("bobby").put("cpu", 100.0);
+ resourceUserPool.get("bobby").put("memory", 1000.0);
+
+ resourceUserPool.put("derek", new HashMap<String, Number>());
+ resourceUserPool.get("derek").put("cpu", 25.0);
+ resourceUserPool.get("derek").put("memory", 250.0);
+
+ config.put(Config.RESOURCE_AWARE_SCHEDULER_USER_POOLS,
resourceUserPool);
+ Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String,
SchedulerAssignmentImpl>(), config);
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "jerry");
+
+ TopologyDetails topo1 =
TestUtilsForResourceAwareScheduler.getTopology("topo-1", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 20);
+ TopologyDetails topo2 =
TestUtilsForResourceAwareScheduler.getTopology("topo-2", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 20);
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "bobby");
+
+ TopologyDetails topo3 =
TestUtilsForResourceAwareScheduler.getTopology("topo-3", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 10);
+ TopologyDetails topo4 =
TestUtilsForResourceAwareScheduler.getTopology("topo-4", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 10);
+
+ config.put(Config.TOPOLOGY_SUBMITTER_USER, "derek");
+
+ TopologyDetails topo5 =
TestUtilsForResourceAwareScheduler.getTopology("topo-5", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 29);
+ TopologyDetails topo6 =
TestUtilsForResourceAwareScheduler.getTopology("topo-6", config, 1, 0, 1, 0,
Time.currentTimeSecs() - 2, 10);
+
+ Map<String, TopologyDetails> topoMap = new HashMap<String,
TopologyDetails>();
+ topoMap.put(topo1.getId(), topo1);
+ topoMap.put(topo3.getId(), topo3);
+ topoMap.put(topo4.getId(), topo4);
+ topoMap.put(topo5.getId(), topo5);
+
+ Topologies topologies = new Topologies(topoMap);
+
+ ResourceAwareScheduler rs = new ResourceAwareScheduler();
+
+ rs.prepare(config);
+ rs.schedule(topologies, cluster);
+
+ for (TopologyDetails topo :
rs.getUser("jerry").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 1,
rs.getUser("jerry").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("jerry").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("jerry").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("jerry").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo :
rs.getUser("derek").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 1,
rs.getUser("derek").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("derek").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("derek").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("derek").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo :
rs.getUser("bobby").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 2,
rs.getUser("bobby").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("bobby").getTopologiesPending().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("bobby").getTopologiesInvalid().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("bobby").getTopologiesAttempted().size());
+
+ //user derek submits another topology into a full cluster
+ // topo6 should not be able to scheduled
+ topoMap.put(topo6.getId(), topo6);
+ topologies = new Topologies(topoMap);
+ rs.schedule(topologies, cluster);
+
+ for (TopologyDetails topo :
rs.getUser("jerry").getTopologiesRunning()) {
+ Assert.assertTrue("Assert scheduling topology success",
TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
+ }
+ Assert.assertEquals("# of running topologies", 1,
rs.getUser("jerry").getTopologiesRunning().size());
+ Assert.assertEquals("# of pending topologies", 0,
rs.getUser("jerry").getTopologiesPending().size());
+ Assert.assertEquals("# of attempted topologies", 0,
rs.getUser("jerry").getTopologiesAttempted().size());
+ Assert.assertEquals("# of invalid topologies", 0,
rs.getUser("jerry").getTopologiesInvalid().size());
+
+ for (TopologyDetails topo : rs.
--- End diff --
I am not sure we would necessarily know that an executor from `topo-6` was
on the supervisor that died, but it is true that we should evict one of jerry's
topologies just based on overage amounts. Which one gets evicted may be down
to a race.
> Add priorities and per user resource guarantees to Resource Aware Scheduler
> ---------------------------------------------------------------------------
>
> Key: STORM-898
> URL: https://issues.apache.org/jira/browse/STORM-898
> Project: Apache Storm
> Issue Type: New Feature
> Components: storm-core
> Reporter: Robert Joseph Evans
> Assignee: Boyang Jerry Peng
> Attachments: Resource Aware Scheduler for Storm.pdf
>
>
> In a multi-tenant environment we would like to be able to give individual
> users a guarantee of how much CPU/Memory/Network they will be able to use in
> a cluster. We would also like to know which topologies a user feels are the
> most important to keep running if there are not enough resources to run all
> of their topologies.
> Each user should be able to specify if their topology is production, staging,
> or development. Within each of those categories a user should be able to give
> a topology a priority, 0 to 10 with 10 being the highest priority (or
> something like this).
> If there are not enough resources on a cluster to run a topology assume this
> topology is running using resources and find the user that is most over their
> guaranteed resources. Shoot the lowest priority topology for that user, and
> repeat until, this topology is able to run, or this topology would be the one
> shot. Ideally we don't actually shoot anything until we know that we would
> have made enough room.
> If the cluster is over-subscribed and everyone is under their guarantee, and
> this topology would not put the user over their guarantee. Shoot the lowest
> priority topology in this workers resource pool until there is enough room to
> run the topology or this topology is the one that would be shot. We might
> also want to think about what to do if we are going to shoot a production
> topology in an oversubscribed case, and perhaps we can shoot a non-production
> topology instead even if the other user is not over their guarantee.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
