SLIDER-570: tuning of tests, agent failure tests working (i.e. regression fixed)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e1023ead Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e1023ead Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e1023ead Branch: refs/heads/develop Commit: e1023eadf896866ce6c63f06e47bf835adc8576c Parents: 6e171c7 Author: Steve Loughran <ste...@apache.org> Authored: Fri Oct 31 13:25:26 2014 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Fri Oct 31 13:25:26 2014 +0000 ---------------------------------------------------------------------- .../framework/AgentCommandTestBase.groovy | 10 ++++ .../funtest/framework/CommandTestBase.groovy | 54 ++++++++++++-------- .../funtest/lifecycle/AMFailuresIT.groovy | 3 +- .../lifecycle/AgentClusterLifecycleIT.groovy | 1 + .../funtest/lifecycle/AgentFailures2IT.groovy | 12 ++--- .../funtest/lifecycle/AgentFailuresIT.groovy | 8 +-- .../lifecycle/AgentLaunchFailureIT.groovy | 3 +- .../funtest/lifecycle/AgentRegistryIT.groovy | 2 - .../lifecycle/AppsThroughAgentDemo.groovy | 27 ++++++---- .../funtest/lifecycle/AppsThroughAgentIT.groovy | 16 +++--- .../AppsThroughAgentQueueAndLabelsIT.groovy | 6 ++- 11 files changed, 84 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy index 6c99ab6..6e05056 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy @@ -21,6 +21,7 @@ package org.apache.slider.funtest.framework import groovy.util.logging.Slf4j import org.apache.hadoop.fs.Path import org.apache.hadoop.security.UserGroupInformation +import org.apache.hadoop.yarn.api.records.YarnApplicationState import org.apache.slider.common.SliderExitCodes import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.SliderActions @@ -167,4 +168,13 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { assert fail("Old cluster either should not exist or should get destroyed; destroy exit code = ${shell.ret}") } } + + /** + * Assert that the application is running (i.e in state + * {@link YarnApplicationState#RUNNING}) + * @param appId application ID + */ + def assertAppRunning(String appId) { + assertInYarnState(appId, YarnApplicationState.RUNNING) + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy index fc74129..7ea84df 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy @@ -94,8 +94,7 @@ abstract class CommandTestBase extends SliderTestUtils { public static final int CONTAINER_LAUNCH_TIMEOUT = 90000 public static final int PROBE_SLEEP_TIME = 4000 public static final int REGISTRY_STARTUP_TIMEOUT = 60000 - public static - final String E_LAUNCH_FAIL = 'Application did not start' + public static final String E_LAUNCH_FAIL = 'Application did not start' /* Static initializer for test configurations. If this code throws exceptions @@ -380,14 +379,18 @@ abstract class CommandTestBase extends SliderTestUtils { static SliderShell lookup(int result, String id, File out) { assert id def commands = [ACTION_LOOKUP, ARG_ID, id] - if (out) commands += [ARG_OUTPUT, out.absolutePath] + if (out) { + commands += [ARG_OUTPUT, out.absolutePath] + } slider(result, commands) } static SliderShell lookup(String id, File out) { assert id def commands = [ACTION_LOOKUP, ARG_ID, id] - if (out) commands += [ARG_OUTPUT, out.absolutePath] + if (out) { + commands += [ARG_OUTPUT, out.absolutePath] + } slider(commands) } @@ -707,7 +710,7 @@ abstract class CommandTestBase extends SliderTestUtils { } return args } - + public static SerializedApplicationReport maybeLoadAppReport(File reportFile) { if (reportFile.exists() && reportFile.length()> 0) { ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser() @@ -776,19 +779,24 @@ abstract class CommandTestBase extends SliderTestUtils { ARG_MESSAGE, "suicide" ]) - sleep(5000) ensureApplicationIsUp(cluster) - -/* - def sleeptime = SLIDER_CONFIG.getInt(KEY_AM_RESTART_SLEEP_TIME, - DEFAULT_AM_RESTART_SLEEP_TIME) - sleep(sleeptime) -*/ - ClusterDescription status + return sliderClient.clusterDescription + } + public ClusterDescription killAmAndWaitForRestart( + SliderClient sliderClient, String cluster, String appId) { + + assert cluster + slider(0, [ + ACTION_AM_SUICIDE, cluster, + ARG_EXITCODE, "1", + ARG_WAIT, "1000", + ARG_MESSAGE, "suicide" + ]) - status = sliderClient.clusterDescription - return status + sleep(5000) + ensureYarnApplicationIsUp(appId) + return sliderClient.clusterDescription } protected void ensureRegistryCallSucceeds(String application) { @@ -844,7 +852,6 @@ abstract class CommandTestBase extends SliderTestUtils { ); } - /** * is an application in a desired yarn state * @param yarnState @@ -1057,25 +1064,30 @@ abstract class CommandTestBase extends SliderTestUtils { int expectedCount = args['limit'].toInteger(); int requestedCount = queryRequestedCount(application, role) - log.debug("requested count = $requestedCount; expected=$expectedCount") + log.debug("requested $role count = $requestedCount; expected=$expectedCount") return Outcome.fromBool(requestedCount >= expectedCount) } - void expectContainerRequestedCountReached(String application, String role, int limit) { + void expectContainerRequestedCountReached(String application, String role, int limit, + int container_launch_timeout) { repeatUntilSuccess( this.&hasRequestedContainerCountReached, - CONTAINER_LAUNCH_TIMEOUT, + container_launch_timeout, PROBE_SLEEP_TIME, [limit : Integer.toString(limit), role : role, application: application], true, "countainer count not reached") { - describe "container count not reached" + int requestedCount = queryRequestedCount(application, role) + + def message = "expected count of $role = $limit not reached: $requestedCount" + + " after $container_launch_timeout mS" + describe message ClusterDescription cd = execStatus(application); log.info("Parsed status \n$cd") - status(application).dumpOutput() + fail(message) }; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy index 2e28c84..7cc01b8 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy @@ -70,7 +70,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { logShell(shell) def appId = ensureYarnApplicationIsUp(launchReportFile) - expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1) + expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1, + CONTAINER_LAUNCH_TIMEOUT) // Wait for 20 secs for AM and agent to both reach STARTED state sleep(1000 * 20) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy index 3e5cec7..cb137ce 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy @@ -183,6 +183,7 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase //start with a restart count set to enable restart describe "the kill/restart phase may fail if yarn.resourcemanager.am.max-attempts is too low" + thaw(CLUSTER, [ ARG_WAIT, Integer.toString(THAW_WAIT_TIME), http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy index 6c6b52b..bfae6ec 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy @@ -20,7 +20,6 @@ package org.apache.slider.funtest.lifecycle import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import org.apache.hadoop.yarn.api.records.YarnApplicationState import org.apache.slider.common.SliderExitCodes import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.SliderActions @@ -60,13 +59,12 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { logShell(shell) def appId = ensureYarnApplicationIsUp(launchReportFile) - expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 3) + expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 2, + CONTAINER_LAUNCH_TIMEOUT * 2) sleep(1000 * 20) - def cd = execStatus(APPLICATION_NAME) - assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 3 - assertInYarnState(appId, YarnApplicationState.RUNNING) + expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 3, + CONTAINER_LAUNCH_TIMEOUT * 2) + assertAppRunning(appId) } - - } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy index 0410881..efd6194 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy @@ -63,13 +63,13 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { logShell(shell) def appId = ensureYarnApplicationIsUp(launchReportFile) - - expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 2) + expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 2, + CONTAINER_LAUNCH_TIMEOUT) sleep(1000 * 20) - assertInYarnState(appId, YarnApplicationState.RUNNING) + assertAppRunning(appId) def cd = expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1) assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 2 - assertInYarnState(appId, YarnApplicationState.RUNNING) + assertAppRunning(appId) } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy index 5087c06..791123c 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy @@ -28,7 +28,6 @@ import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.SliderActions import org.apache.slider.funtest.framework.AgentCommandTestBase import org.apache.slider.funtest.framework.FuntestProperties -import org.apache.slider.funtest.framework.SliderShell import org.junit.After import org.junit.Before import org.junit.Test @@ -62,7 +61,7 @@ public class AgentLaunchFailureIT extends AgentCommandTestBase // create an AM which fails to launch File launchReportFile = createAppReportFile(); - SliderShell shell = createTemplatedSliderApplication(CLUSTER, + createTemplatedSliderApplication(CLUSTER, APP_TEMPLATE, APP_RESOURCE2, [ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy index b5ee23d..5b8b3cd 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy @@ -132,7 +132,5 @@ public class AgentRegistryIT extends AgentCommandTestBase //cluster now missing exists(EXIT_UNKNOWN_INSTANCE, CLUSTER) - - } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy index eb825fc..5be7211 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy @@ -49,13 +49,17 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { assumeAgentTestsEnabled() cleanup(APPLICATION_NAME) + File launchReportFile = createAppReportFile(); + SliderShell shell = createTemplatedSliderApplication(APPLICATION_NAME, APP_TEMPLATE, - APP_RESOURCE) + APP_RESOURCE, + [], + launchReportFile) logShell(shell) - ensureApplicationIsUp(APPLICATION_NAME) + def appId = ensureYarnApplicationIsUp(launchReportFile) //flex slider(EXIT_SUCCESS, @@ -69,9 +73,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { // sleep till the new instance starts sleep(1000 * 10) - shell = slider(EXIT_SUCCESS, - [ - ACTION_STATUS, + slider(EXIT_SUCCESS, + [ACTION_STATUS, APPLICATION_NAME]) expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 2) @@ -88,7 +91,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { ACTION_REGISTRY, ARG_NAME, APPLICATION_NAME, - ARG_LISTEXP]) + ARG_LISTEXP + ]) if(!containsString(shell, "container_log_dirs") || !containsString(shell, "container_work_dirs")) { logShell(shell) @@ -102,7 +106,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { ARG_NAME, APPLICATION_NAME, ARG_GETEXP, - "container_log_dirs"]) + "container_log_dirs" + ]) if(!containsString(shell, "\"tag\" : \"COMMAND_LOGGER\"", 2) || !containsString(shell, "\"level\" : \"component\"", 2)) { logShell(shell) @@ -116,7 +121,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { ARG_NAME, APPLICATION_NAME, ARG_GETEXP, - "container_work_dirs"]) + "container_work_dirs" + ]) if(!containsString(shell, "\"tag\" : \"COMMAND_LOGGER\"", 2) || !containsString(shell, "\"level\" : \"component\"", 2)) { logShell(shell) @@ -132,7 +138,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { ARG_GETCONF, "cl-site", ARG_FORMAT, - "json"]) + "json" + ]) for (int i = 0; i < 10; i++) { if (shell.getRet() != EXIT_SUCCESS) { @@ -155,6 +162,6 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { assert fail("Should have exported cl-site") } - assert isApplicationUp(APPLICATION_NAME), 'App is not running.' + assertAppRunning(appId) } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy index 5a5b964..75807c3 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy @@ -20,7 +20,6 @@ package org.apache.slider.funtest.lifecycle import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import org.apache.hadoop.yarn.api.records.YarnApplicationState import org.apache.slider.common.SliderExitCodes import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.SliderActions @@ -76,10 +75,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { // sleep till the new instance starts sleep(1000 * 10) - shell = slider(EXIT_SUCCESS, - [ - ACTION_STATUS, - APPLICATION_NAME]) + status(0, APPLICATION_NAME) expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 2) @@ -145,7 +141,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { if (shell.getRet() != EXIT_SUCCESS) { println "Waiting for the cl-site to show up" sleep(1000 * 10) - shell = slider( + shell = slider(0, [ ACTION_REGISTRY, ARG_NAME, @@ -156,12 +152,14 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { "json"]) } } - assert shell.getRet() == EXIT_SUCCESS, "cl-site should be retrieved" if (!containsString(shell, "\"pattern.for.test.to.verify\" : \"verify this pattern\"", 1)) { logShell(shell) - assert fail("Should have exported cl-site") + + fail("Should have exported cl-site; got " + + "stdout" +shell.stdErrHistory + + " \nstderr:" + shell.stdErrHistory) } - assertInYarnState(appId, YarnApplicationState.RUNNING) + assertAppRunning(appId) } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy index ee418dc..f6a1b1e 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy @@ -92,7 +92,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { def appId = ensureYarnApplicationIsUp(launchReportFile) - expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1 ) + expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1, + CONTAINER_LAUNCH_TIMEOUT) expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1) //flex @@ -107,7 +108,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { // spin till the flexed instance starts ensureYarnApplicationIsUp(appId) - expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 3) + expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 3, + CONTAINER_LAUNCH_TIMEOUT) sleep(1000 * 20)