SLIDER-570 monkey can simulate launch failure, ongoing work on tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1cd37f57 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1cd37f57 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1cd37f57 Branch: refs/heads/develop Commit: 1cd37f57053933065baf864d109fba1897ee8b95 Parents: f63ac6c Author: Steve Loughran <ste...@apache.org> Authored: Thu Oct 30 18:32:59 2014 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Fri Oct 31 11:07:50 2014 +0000 ---------------------------------------------------------------------- .../org/apache/slider/api/InternalKeys.java | 16 ++ .../org/apache/slider/client/SliderClient.java | 20 +-- .../slider/common/params/ActionDestroyArgs.java | 1 + .../launch/SerializedApplicationReport.java | 5 - .../server/appmaster/SliderAppMaster.java | 20 ++- .../server/appmaster/monkey/ChaosEntry.java | 9 +- .../appmaster/monkey/ChaosMonkeyService.java | 27 ++-- .../model/monkey/TestMockMonkey.groovy | 9 +- .../funtest/framework/CommandTestBase.groovy | 55 +++++-- .../slider/funtest/framework/SliderShell.groovy | 1 - .../funtest/lifecycle/AMFailuresIT.groovy | 4 +- .../lifecycle/AgentClusterLifecycleIT.groovy | 3 +- .../lifecycle/AgentLaunchFailureIT.groovy | 12 +- .../lifecycle/AppsThroughAgentDemo.groovy | 160 +++++++++++++++++++ .../funtest/lifecycle/AppsThroughAgentIT.groovy | 8 +- 15 files changed, 293 insertions(+), 57 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java index 074644d..b360fbe 100644 --- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java +++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java @@ -150,6 +150,12 @@ public interface InternalKeys { int DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = 0; /** + * Probability of a monkey check killing the AM: {@value} + */ + String CHAOS_MONKEY_PROBABILITY_AM_LAUNCH_FAILURE = + CHAOS_MONKEY_PROBABILITY + ".amlaunchfailure"; + + /** * Probability of a monkey check killing a container: {@value} */ @@ -162,4 +168,14 @@ public interface InternalKeys { int DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE = 0; + /** + * 1% of chaos + */ + int PROBABILITY_PERCENT_1 = 100; + + /** + * 100% for chaos values + */ + int PROBABILITY_PERCENT_100 = 100 * PROBABILITY_PERCENT_1; + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index b2e343d..0e87906 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -347,7 +347,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } catch (PathNotFoundException nfe) { throw new NotFoundException(nfe, nfe.toString()); } - } /** @@ -529,7 +528,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe SliderUtils.validateClusterName(clustername); //no=op, it is now mandatory. verifyBindingsDefined(); - verifyNoLiveClusters(clustername); + verifyNoLiveClusters(clustername, "Destroy"); // create the directory path Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername); @@ -772,7 +771,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe SliderUtils.validateClusterName(clustername); verifyBindingsDefined(); if (!liveClusterAllowed) { - verifyNoLiveClusters(clustername); + verifyNoLiveClusters(clustername, "Create"); } Configuration conf = getConfig(); @@ -1100,7 +1099,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe deployedClusterName = clustername; SliderUtils.validateClusterName(clustername); - verifyNoLiveClusters(clustername); + verifyNoLiveClusters(clustername, "Launch"); Configuration config = getConfig(); lookupZKQuorum(); boolean clusterSecure = SliderUtils.isHadoopClusterSecure(config); @@ -1534,17 +1533,21 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe /** * verify that a live cluster isn't there * @param clustername cluster name + * @param action * @throws SliderException with exit code EXIT_CLUSTER_LIVE * if a cluster of that name is either live or starting up. */ - public void verifyNoLiveClusters(String clustername) throws + public void verifyNoLiveClusters(String clustername, String action) throws IOException, YarnException { List<ApplicationReport> existing = findAllLiveInstances(clustername); if (!existing.isEmpty()) { throw new SliderException(EXIT_APPLICATION_IN_USE, - clustername + ": " + E_CLUSTER_RUNNING + " :" + + action +" failed for " + + clustername + + ": " + + E_CLUSTER_RUNNING + " :" + existing.get(0)); } } @@ -2168,10 +2171,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe @Override public int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException { SliderUtils.validateClusterName(clustername); - // see if it is actually running and bail out; verifyBindingsDefined(); - verifyNoLiveClusters(clustername); - + // see if it is actually running and bail out; + verifyNoLiveClusters(clustername, "Start"); //start the cluster return startCluster(clustername, thaw); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java index d4acea6..1203d28 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java @@ -24,6 +24,7 @@ import com.beust.jcommander.Parameters; commandDescription = SliderActions.DESCRIBE_ACTION_DESTROY) public class ActionDestroyArgs extends AbstractActionArgs { + @Override public String getActionName() { return SliderActions.ACTION_DESTROY; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java b/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java index e3d77d0..dfa037d 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java @@ -18,12 +18,7 @@ package org.apache.slider.core.launch; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.slider.core.persist.ApplicationReportSerDeser; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.annotate.JsonSerialize; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 855ecd7..5e2ba11 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -192,7 +192,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService ServiceStateChangeListener, RoleKeys, ProviderCompleted { - + protected static final Logger log = LoggerFactory.getLogger(SliderAppMaster.class); @@ -220,7 +220,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService * Singleton of metrics registry */ public static final MetricRegistry metrics = new MetricRegistry(); - + public static final String E_TRIGGERED_LAUNCH_FAILURE = + "Chaos monkey triggered launch failure"; + /** YARN RPC to communicate with the Resource Manager or Node Manager */ private YarnRPC yarnRPC; @@ -2191,6 +2193,20 @@ public class SliderAppMaster extends AbstractSliderLaunchedService initAndAddService(monkey); // configure the targets + + // launch failure: special case with explicit failure triggered now + int amLaunchFailProbability = internals.getOptionInt( + InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_LAUNCH_FAILURE, + 0); + if (amLaunchFailProbability> 0 && monkey.chaosCheck(amLaunchFailProbability)) { + // trigger a failure + ActionStopSlider stop = new ActionStopSlider("stop", + 0, TimeUnit.SECONDS, + LauncherExitCodes.EXIT_FALSE, + FinalApplicationStatus.FAILED, + E_TRIGGERED_LAUNCH_FAILURE); + } + int amKillProbability = internals.getOptionInt( InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE, InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java index 2869fe9..87a0aaa 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java @@ -22,6 +22,7 @@ import com.codahale.metrics.Counter; import com.codahale.metrics.MetricRegistry; import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; +import org.apache.slider.api.InternalKeys; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,16 +43,16 @@ public class ChaosEntry { /** * Constructor -includes validation of all arguments - * @param name - * @param target - * @param probability + * @param name entry name + * @param target target + * @param probability probability of occurring */ public ChaosEntry(String name, ChaosTarget target, long probability, MetricRegistry metrics) { Preconditions.checkArgument(!StringUtils.isEmpty(name), "missing name"); Preconditions.checkArgument(target != null, "null target"); Preconditions.checkArgument(probability > 0, "negative probability"); - Preconditions.checkArgument(probability <= ChaosMonkeyService.PERCENT_100, + Preconditions.checkArgument(probability <= InternalKeys.PROBABILITY_PERCENT_100, "probability over 100%: "+ probability); this.name = name; this.target = target; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java index 80f981c..27219e4 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java @@ -20,6 +20,7 @@ package org.apache.slider.server.appmaster.monkey; import com.codahale.metrics.MetricRegistry; import org.apache.hadoop.service.AbstractService; +import org.apache.slider.api.InternalKeys; import org.apache.slider.server.appmaster.actions.QueueAccess; import org.apache.slider.server.appmaster.actions.RenewingAction; import org.slf4j.Logger; @@ -36,13 +37,7 @@ import java.util.concurrent.TimeUnit; public class ChaosMonkeyService extends AbstractService { protected static final Logger log = LoggerFactory.getLogger(ChaosMonkeyService.class); - public static final int PERCENT_1 = 100; - public static final double PERCENT_1D = 100.0; - - /** - * the percentage value as multiplied up - */ - public static final int PERCENT_100 = 100 * PERCENT_1; + private final MetricRegistry metrics; private final QueueAccess queues; private final Random random = new Random(); @@ -65,7 +60,7 @@ public class ChaosMonkeyService extends AbstractService { public synchronized void addTarget(String name, ChaosTarget target, long probability) { if (probability > 0) { - log.info("Adding {} with probability {}", name, probability / PERCENT_1); + log.info("Adding {} with probability {}", name, probability / InternalKeys.PROBABILITY_PERCENT_1); chaosEntries.add(new ChaosEntry(name, target, probability, metrics)); } else { log.debug("Action {} not enabled", name); @@ -85,11 +80,25 @@ public class ChaosMonkeyService extends AbstractService { */ public void play() { for (ChaosEntry chaosEntry : chaosEntries) { - long p = random.nextInt(PERCENT_100); + long p = randomPercentage(); chaosEntry.maybeInvokeChaos(p); } } + public int randomPercentage() { + return random.nextInt(InternalKeys.PROBABILITY_PERCENT_100); + } + + /** + * Check for callers to see if chaos should be triggered; shares the + * same random number source as the rest of the monkey entries + * @param probability probability + * @return true if the action should happen + */ + public boolean chaosCheck(long probability) { + return randomPercentage() < probability; + } + /** * Schedule the monkey * http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy index d31c9f6..e4a42fc 100644 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy @@ -21,6 +21,7 @@ package org.apache.slider.server.appmaster.model.monkey import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.yarn.conf.YarnConfiguration +import org.apache.slider.api.InternalKeys import org.apache.slider.server.appmaster.actions.ActionHalt import org.apache.slider.server.appmaster.actions.ActionKillContainer import org.apache.slider.server.appmaster.actions.QueueService @@ -66,7 +67,7 @@ class TestMockMonkey extends BaseMockAppStateTest { @Test public void testMonkeyPlay() throws Throwable { ChaosCounter counter = new ChaosCounter() - monkey.addTarget("target", counter, ChaosMonkeyService.PERCENT_100) + monkey.addTarget("target", counter, InternalKeys.PROBABILITY_PERCENT_100) assert 1 == monkey.targetCount; monkey.play() assert counter.count == 1 @@ -76,7 +77,7 @@ class TestMockMonkey extends BaseMockAppStateTest { public void testMonkeySchedule() throws Throwable { ChaosCounter counter = new ChaosCounter() assert 0 == monkey.targetCount; - monkey.addTarget("target", counter, ChaosMonkeyService.PERCENT_100) + monkey.addTarget("target", counter, InternalKeys.PROBABILITY_PERCENT_100) assert 1 == monkey.targetCount; assert monkey.schedule(0, 1, TimeUnit.SECONDS) assert 1 == queues.scheduledActions.size() @@ -105,8 +106,8 @@ class TestMockMonkey extends BaseMockAppStateTest { public void testMonkeyPlaySometimes() throws Throwable { ChaosCounter counter = new ChaosCounter() ChaosCounter counter2 = new ChaosCounter() - monkey.addTarget("target1", counter, ChaosMonkeyService.PERCENT_1 * 50) - monkey.addTarget("target2", counter2, ChaosMonkeyService.PERCENT_1 * 25) + monkey.addTarget("target1", counter, InternalKeys.PROBABILITY_PERCENT_1 * 50) + monkey.addTarget("target2", counter2, InternalKeys.PROBABILITY_PERCENT_1 * 25) for (int i = 0; i < 100; i++) { monkey.play() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy index 1c65394..7c479af 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy @@ -457,6 +457,7 @@ abstract class CommandTestBase extends SliderTestUtils { * @param cluster */ static void setupCluster(String cluster) { + describe "setting up $cluster" ensureClusterDestroyed(cluster) } @@ -655,7 +656,6 @@ abstract class CommandTestBase extends SliderTestUtils { SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL)); commands.addAll(extraArgs) SliderShell shell = new SliderShell(commands) - shell.execute() if (0 != shell.execute()) { // app has failed. @@ -678,7 +678,7 @@ abstract class CommandTestBase extends SliderTestUtils { return shell } - public File createAppReportFile() { + public static File createAppReportFile() { File reportFile = File.createTempFile( "launch", ".json", @@ -693,7 +693,7 @@ abstract class CommandTestBase extends SliderTestUtils { * @param option option to probe and use * @return the (possibly extended) list */ - public List<String> maybeAddCommandOption( + public static List<String> maybeAddCommandOption( List<String> args, List<String> commands, String option) { if ( SliderUtils.isSet(option)) { args.addAll(commands) @@ -702,7 +702,7 @@ abstract class CommandTestBase extends SliderTestUtils { return args } - public SerializedApplicationReport maybeLoadAppReport(File reportFile) { + public static SerializedApplicationReport maybeLoadAppReport(File reportFile) { if (reportFile.exists() && reportFile.length()> 0) { ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser() def report = serDeser.fromFile(reportFile) @@ -711,7 +711,7 @@ abstract class CommandTestBase extends SliderTestUtils { return null; } - public SerializedApplicationReport maybeLookupFromLaunchReport(File launchReport) { + public static SerializedApplicationReport maybeLookupFromLaunchReport(File launchReport) { def report = maybeLoadAppReport(launchReport) if (report) { return lookupApplication(report.applicationId) @@ -725,7 +725,7 @@ abstract class CommandTestBase extends SliderTestUtils { * @param id application ID * @return an application report or null */ - public SerializedApplicationReport lookupApplication(String id) { + public static SerializedApplicationReport lookupApplication(String id) { File reportFile = createAppReportFile(); try { def shell = lookup(id, reportFile) @@ -787,6 +787,7 @@ abstract class CommandTestBase extends SliderTestUtils { } } + protected void ensureApplicationIsUp(String application) { repeatUntilTrue(this.&isApplicationRunning, 30, @@ -816,26 +817,58 @@ abstract class CommandTestBase extends SliderTestUtils { protected boolean isApplicationRunning(Map<String, String> args) { String applicationName = args['application']; - return isApplicationInState(YarnApplicationState.RUNNING, applicationName); + return isApplicationUp(applicationName); } protected boolean isApplicationUp(String applicationName) { - return isApplicationInState(YarnApplicationState.RUNNING, applicationName); + return isApplicationInState( + applicationName, + YarnApplicationState.RUNNING + ); } + protected void ensureYarnApplicationIsUp(String application) { + repeatUntilTrue(this.&isApplicationRunning, + 30, + SLIDER_CONFIG.getInt(KEY_TEST_INSTANCE_LAUNCH_TIME, + DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS), + [application: application], + true, + 'Application did not start, failing test.') { + describe "final state of app that tests say is not up" + exists(application, true).dumpOutput() + } + } + /** - * + * is an application in a desired yarn state * @param yarnState * @param applicationName * @return */ - public static boolean isApplicationInState(YarnApplicationState yarnState, String applicationName) { + public static boolean isApplicationInState( + String applicationName, + YarnApplicationState yarnState) { SliderShell shell = slider( [ACTION_EXISTS, applicationName, ARG_STATE, yarnState.toString()]) - return shell.ret == 0 } + + /** + * is a yarn application in a desired yarn state + * @param yarnState + * @param applicationName + * @return + */ + public static boolean isYarnApplicationInState( + String applicationId, + YarnApplicationState yarnState) { + def sar = lookupApplication(applicationId) + assert sar != null; + return yarnState.toString() == sar.state + } + /** * Repeat a probe until it succeeds, if it does not execute a failure * closure then raise an exception with the supplied message http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy index 31830d9..ae40d6a 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy @@ -247,7 +247,6 @@ class SliderShell extends Shell { Shell exec(Object... args) { Process proc = "$shell".execute() script = args.join("\n") - LOG.debug("${shell} << __EOT__\n${script}\n__EOT__"); ByteArrayOutputStream baosErr = new ByteArrayOutputStream(4096); ByteArrayOutputStream baosOut = new ByteArrayOutputStream(4096); proc.consumeProcessOutput(baosOut, baosErr) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy index 10a3dfe..750a3d4 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy @@ -92,8 +92,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { killAMUsingVagrantShell() // Check that the application is not running (and is in ACCEPTED state) - assert isApplicationInState(YarnApplicationState.ACCEPTED, - APPLICATION_NAME), + assert isApplicationInState(APPLICATION_NAME, YarnApplicationState.ACCEPTED + ), 'App should be in ACCEPTED state (since AM got killed)' log.info("After AM KILL: application {} is in ACCEPTED state", APPLICATION_NAME) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy index 8b123c3..3783a37 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy @@ -48,7 +48,6 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase @Before public void prepareCluster() { setupCluster(CLUSTER) - describe("Create a 0-role cluster, so testing AM start/stop") } @After @@ -62,7 +61,7 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase describe "Walk a 0-role cluster through its lifecycle" // sanity check to verify the config is correct - assert clusterFS.uri.scheme!="file" + assert clusterFS.uri.scheme != "file" def clusterpath = buildClusterPath(CLUSTER) assert !clusterFS.exists(clusterpath) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy index 1a0d2c3..4dcbed0 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy @@ -20,7 +20,7 @@ package org.apache.slider.funtest.lifecycle import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import org.apache.slider.api.InternalKeys +import static org.apache.slider.api.InternalKeys.* import org.apache.slider.common.SliderExitCodes import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.SliderActions @@ -41,7 +41,6 @@ public class AgentLaunchFailureIT extends AgentCommandTestBase static String APP_RESOURCE2 = "../slider-core/src/test/app_packages/test_command_log/resources_no_role.json" - @Before public void prepareCluster() { setupCluster(CLUSTER) @@ -59,16 +58,15 @@ public class AgentLaunchFailureIT extends AgentCommandTestBase // verify no cluster assert 0 != exists(CLUSTER).ret - // create an AM which fails to launch within a second + // create an AM which fails to launch File launchReportFile = createAppReportFile(); SliderShell shell = createTemplatedSliderApplication(CLUSTER, APP_TEMPLATE, APP_RESOURCE2, [ - ARG_OPTION, InternalKeys.CHAOS_MONKEY_ENABLED, "true", - ARG_OPTION, InternalKeys.CHAOS_MONKEY_DELAY_SECONDS, "1", - ARG_OPTION, InternalKeys.CHAOS_MONKEY_INTERVAL_SECONDS, "60", - ARG_OPTION, InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE, "100", + ARG_OPTION, CHAOS_MONKEY_ENABLED, "true", + ARG_OPTION, CHAOS_MONKEY_PROBABILITY_AM_LAUNCH_FAILURE, + Integer.toString(PROBABILITY_PERCENT_100), ], launchReportFile) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy new file mode 100644 index 0000000..eb825fc --- /dev/null +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.funtest.lifecycle + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.slider.common.SliderExitCodes +import org.apache.slider.common.params.Arguments +import org.apache.slider.common.params.SliderActions +import org.apache.slider.funtest.framework.AgentCommandTestBase +import org.apache.slider.funtest.framework.FuntestProperties +import org.apache.slider.funtest.framework.SliderShell +import org.junit.After +import org.junit.Before +import org.junit.Test + +@CompileStatic +@Slf4j +public class AppsThroughAgentDemo extends AgentCommandTestBase +implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { + + private static String COMMAND_LOGGER = "COMMAND_LOGGER" + private static String APPLICATION_NAME = "agent-demo" + + + @Before + public void prepareCluster() { + setupCluster(APPLICATION_NAME) + } + + @Test + public void testCreateFlex() throws Throwable { + assumeAgentTestsEnabled() + + cleanup(APPLICATION_NAME) + SliderShell shell = createTemplatedSliderApplication(APPLICATION_NAME, + APP_TEMPLATE, + APP_RESOURCE) + + logShell(shell) + + ensureApplicationIsUp(APPLICATION_NAME) + + //flex + slider(EXIT_SUCCESS, + [ + ACTION_FLEX, + APPLICATION_NAME, + ARG_COMPONENT, + COMMAND_LOGGER, + "2"]) + + // sleep till the new instance starts + sleep(1000 * 10) + + shell = slider(EXIT_SUCCESS, + [ + ACTION_STATUS, + APPLICATION_NAME]) + + expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 2) + + String amWebUrl = getInfoAmWebUrl(APPLICATION_NAME) + log.info("Dumping data from AM Web URL"); + log.info(amWebUrl.toURL().text); + + ensureRegistryCallSucceeds(APPLICATION_NAME) + + // get log folders + shell = slider(EXIT_SUCCESS, + [ + ACTION_REGISTRY, + ARG_NAME, + APPLICATION_NAME, + ARG_LISTEXP]) + if(!containsString(shell, "container_log_dirs") || + !containsString(shell, "container_work_dirs")) { + logShell(shell) + assert fail("Should list default exports container_log_dirs or container_work_dirs") + } + + // get log folders + shell = slider(EXIT_SUCCESS, + [ + ACTION_REGISTRY, + ARG_NAME, + APPLICATION_NAME, + ARG_GETEXP, + "container_log_dirs"]) + if(!containsString(shell, "\"tag\" : \"COMMAND_LOGGER\"", 2) + || !containsString(shell, "\"level\" : \"component\"", 2)) { + logShell(shell) + assert fail("Should list 2 entries for log folders") + } + + // get log folders + shell = slider(EXIT_SUCCESS, + [ + ACTION_REGISTRY, + ARG_NAME, + APPLICATION_NAME, + ARG_GETEXP, + "container_work_dirs"]) + if(!containsString(shell, "\"tag\" : \"COMMAND_LOGGER\"", 2) + || !containsString(shell, "\"level\" : \"component\"", 2)) { + logShell(shell) + assert fail("Should list 2 entries for work folder") + } + + // get cl-site config + shell = slider( + [ + ACTION_REGISTRY, + ARG_NAME, + APPLICATION_NAME, + ARG_GETCONF, + "cl-site", + ARG_FORMAT, + "json"]) + + for (int i = 0; i < 10; i++) { + if (shell.getRet() != EXIT_SUCCESS) { + println "Waiting for the cl-site to show up" + sleep(1000 * 10) + shell = slider( + [ + ACTION_REGISTRY, + ARG_NAME, + APPLICATION_NAME, + ARG_GETCONF, + "cl-site", + ARG_FORMAT, + "json"]) + } + } + assert shell.getRet() == EXIT_SUCCESS, "cl-site should be retrieved" + if (!containsString(shell, "\"pattern.for.test.to.verify\" : \"verify this pattern\"", 1)) { + logShell(shell) + assert fail("Should have exported cl-site") + } + + assert isApplicationUp(APPLICATION_NAME), 'App is not running.' + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy index fd8330b..488173b 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy @@ -27,6 +27,7 @@ import org.apache.slider.funtest.framework.AgentCommandTestBase import org.apache.slider.funtest.framework.FuntestProperties import org.apache.slider.funtest.framework.SliderShell import org.junit.After +import org.junit.Before import org.junit.Test @CompileStatic @@ -35,8 +36,13 @@ public class AppsThroughAgentIT extends AgentCommandTestBase implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { private static String COMMAND_LOGGER = "COMMAND_LOGGER" - private static String APPLICATION_NAME = "happy-path-with-flex" + private static String APPLICATION_NAME = "apps-through-agent" + @Before + public void prepareCluster() { + setupCluster(APPLICATION_NAME) + } + @After public void destroyCluster() { cleanup(APPLICATION_NAME)