SLIDER-363 chaos monkey starting even when not needed
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c2db9ed6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c2db9ed6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c2db9ed6 Branch: refs/heads/develop Commit: c2db9ed6989a2e89509364b5b422b7853ea4db4a Parents: 9184d70 Author: Steve Loughran <ste...@apache.org> Authored: Tue Aug 26 18:27:55 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Tue Aug 26 18:27:55 2014 +0100 ---------------------------------------------------------------------- .../server/appmaster/SliderAppMaster.java | 25 ++++++----- .../appmaster/monkey/ChaosMonkeyService.java | 44 ++++++++++++++++++-- .../model/monkey/TestMockMonkey.groovy | 41 +++++++++++++++--- 3 files changed, 89 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c2db9ed6/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index b4515f1..5a84c39 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -1808,26 +1808,25 @@ public class SliderAppMaster extends AbstractSliderLaunchedService log.info("Adding Chaos Monkey scheduled every {} seconds ({} hours)", monkeyInterval, monkeyInterval/(60*60)); monkey = new ChaosMonkeyService(metrics, actionQueues); + initAndAddService(monkey); + + // configure the targets int amKillProbability = internals.getOptionInt( InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE, InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE); - if (amKillProbability > 0) { - monkey.addTarget("AM killer", - new ChaosKillAM(actionQueues, -1), amKillProbability - ); - } + monkey.addTarget("AM killer", + new ChaosKillAM(actionQueues, -1), amKillProbability); int containerKillProbability = internals.getOptionInt( InternalKeys.CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE, InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE); - if (containerKillProbability > 0) { - monkey.addTarget("Container killer", - new ChaosKillContainer(appState, actionQueues, rmOperationHandler), - containerKillProbability - ); - } - initAndAddService(monkey); + monkey.addTarget("Container killer", + new ChaosKillContainer(appState, actionQueues, rmOperationHandler), + containerKillProbability); + // and schedule it - schedule(monkey.getChaosAction(monkeyInterval, TimeUnit.SECONDS)); + if (monkey.schedule(monkeyInterval, TimeUnit.SECONDS)) { + log.info("Chaos Monkey is running"); + } return true; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c2db9ed6/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java index 91826f1..fa288af 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java @@ -56,14 +56,31 @@ public class ChaosMonkeyService extends AbstractService { this.queues = queues; } - + /** + * Add a target ... it is only added if <code>probability > 0</code> + * @param name name + * @param target chaos target + * @param probability probability + */ public synchronized void addTarget(String name, ChaosTarget target, long probability) { - log.info("Adding {} with probability {}", name, probability / PERCENT_1); - chaosEntries.add(new ChaosEntry(name, target, probability, metrics)); + if (probability > 0) { + log.info("Adding {} with probability {}", name, probability / PERCENT_1); + chaosEntries.add(new ChaosEntry(name, target, probability, metrics)); + } else { + log.debug("Action {} not enabled", name); + } } /** + * Get the number of targets in the list + * @return the count of added targets + */ + public int getTargetCount() { + return chaosEntries.size(); + } + + /** * Iterate through all the entries and invoke chaos on those wanted */ public void play() { @@ -73,6 +90,27 @@ public class ChaosMonkeyService extends AbstractService { } } + /** + * Schedule the monkey + * @param time interval + * @param timeUnit time unit + * @return true if it was scheduled (i.e. 1+ action) + */ + public boolean schedule(long time, TimeUnit timeUnit) { + if (!chaosEntries.isEmpty()) { + queues.schedule(getChaosAction(time, timeUnit)); + return true; + } else { + return false; + } + } + + /** + * Get the chaos action + * @param time interval + * @param timeUnit time unit + * @return the action to schedule + */ public RenewingAction<MonkeyPlayAction> getChaosAction(long time, TimeUnit timeUnit) { RenewingAction<MonkeyPlayAction> action = new RenewingAction<MonkeyPlayAction>( new MonkeyPlayAction(this, 0, TimeUnit.MILLISECONDS), http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c2db9ed6/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy index c789011..79857c6 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy @@ -35,6 +35,8 @@ import org.apache.slider.server.appmaster.state.RoleInstance import org.junit.Before import org.junit.Test +import java.util.concurrent.TimeUnit + @CompileStatic @Slf4j class TestMockMonkey extends BaseMockAppStateTest { @@ -43,14 +45,15 @@ class TestMockMonkey extends BaseMockAppStateTest { * This queue service is NOT started; tests need to poll the queue * rather than expect them to execute */ - QueueService queues = new QueueService(); - ChaosMonkeyService monkey = new ChaosMonkeyService(metricRegistry, - queues) + QueueService queues + ChaosMonkeyService monkey @Before public void init() { def configuration = new YarnConfiguration() + queues = new QueueService(); queues.init(configuration) + monkey = new ChaosMonkeyService(metricRegistry, queues) monkey.init(configuration) } @@ -60,17 +63,45 @@ class TestMockMonkey extends BaseMockAppStateTest { monkey.stop() } - @Test public void testMonkeyPlay() throws Throwable { ChaosCounter counter = new ChaosCounter() monkey.addTarget("target", counter, ChaosMonkeyService.PERCENT_100) - + assert 1 == monkey.targetCount; monkey.play() assert counter.count == 1 } @Test + public void testMonkeySchedule() throws Throwable { + ChaosCounter counter = new ChaosCounter() + assert 0 == monkey.targetCount; + monkey.addTarget("target", counter, ChaosMonkeyService.PERCENT_100) + assert 1 == monkey.targetCount; + assert monkey.schedule(1, TimeUnit.SECONDS) + assert 1 == queues.scheduledActions.size() + } + + @Test + public void testMonkeyDoesntAddProb0Actions() throws Throwable { + ChaosCounter counter = new ChaosCounter() + monkey.addTarget("target", counter, 0) + assert 0 == monkey.targetCount; + monkey.play() + assert counter.count == 0 + } + + + @Test + public void testMonkeyScheduleProb0Actions() throws Throwable { + ChaosCounter counter = new ChaosCounter() + monkey.addTarget("target", counter, 0) + assert !monkey.schedule(1, TimeUnit.SECONDS) + assert 0 == queues.scheduledActions.size() + } + + + @Test public void testMonkeyPlaySometimes() throws Throwable { ChaosCounter counter = new ChaosCounter() ChaosCounter counter2 = new ChaosCounter()
