SLIDER-570: failure output in failed tests...making sure most recent runs get reported, even failures
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/09d16a13 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/09d16a13 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/09d16a13 Branch: refs/heads/feature/SLIDER-531-registry-enhancements Commit: 09d16a130c3ccb4f2f0195ca23d2fa53074e138b Parents: e23e680 Author: Steve Loughran <ste...@apache.org> Authored: Tue Oct 28 15:48:23 2014 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Fri Oct 31 11:07:49 2014 +0000 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 70 +++++----- .../slider/client/SliderYarnClientImpl.java | 7 +- .../org/apache/slider/common/Constants.java | 1 - .../apache/slider/common/tools/SliderUtils.java | 128 ++++++++++++++----- .../framework/AgentCommandTestBase.groovy | 6 +- .../funtest/framework/CommandTestBase.groovy | 18 ++- 6 files changed, 159 insertions(+), 71 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/09d16a13/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 06efadf..392280a 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -161,7 +161,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Map.Entry; -import java.util.Properties; import java.util.Set; import java.util.regex.Pattern; @@ -193,8 +192,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private SliderYarnClientImpl yarnClient; private YarnAppListClient YarnAppListClient; private AggregateConf launchedInstanceDefinition; -// private SliderRegistryService registry; - /** * The YARN registry service @@ -1009,7 +1006,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe serviceArgs.isDebug()); applicationId = launchedApplication.getApplicationId(); - return waitForAppAccepted(launchedApplication, launchArgs.getWaittime()); + int waittime = launchArgs.getWaittime(); + if (waittime > 0) { + return waitForAppRunning(launchedApplication, waittime, waittime); + } else { + // no waiting + return EXIT_SUCCESS; + } } /** @@ -1384,34 +1387,38 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe /** - * Wait for the launched app to be accepted - * @param waittime time in millis - * @return exit code + * Wait for the launched app to be accepted in the time + * and, optionally running. + * <p> + * If the application + * + * @param launchedApplication application + * @param acceptWaitMillis time in millis to wait for accept + * @param runWaitMillis time in millis to wait for the app to be running. + * May be null, in which case no wait takes place + * @return exit code: success * @throws YarnException * @throws IOException */ - public int waitForAppAccepted(LaunchedApplication launchedApplication, - int waittime) throws - YarnException, - IOException { + public int waitForAppRunning(LaunchedApplication launchedApplication, + int acceptWaitMillis, int runWaitMillis) throws YarnException, IOException { assert launchedApplication != null; int exitCode; // wait for the submit state to be reached ApplicationReport report = launchedApplication.monitorAppToState( YarnApplicationState.ACCEPTED, - new Duration(Constants.ACCEPT_TIME)); - + new Duration(acceptWaitMillis)); // may have failed, so check that if (SliderUtils.hasAppFinished(report)) { exitCode = buildExitCode(report); } else { // exit unless there is a wait - exitCode = EXIT_SUCCESS; - if (waittime != 0) { + + if (runWaitMillis != 0) { // waiting for state to change - Duration duration = new Duration(waittime * 1000); + Duration duration = new Duration(runWaitMillis * 1000); duration.start(); report = launchedApplication.monitorAppToState( YarnApplicationState.RUNNING, duration); @@ -1419,10 +1426,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe report.getYarnApplicationState() == YarnApplicationState.RUNNING) { exitCode = EXIT_SUCCESS; } else { - - launchedApplication.kill(""); exitCode = buildExitCode(report); } + } else { + exitCode = EXIT_SUCCESS; } } return exitCode; @@ -1579,16 +1586,17 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } /** - * Build an exit code for an application Id and its report. - * If the report parameter is null, the app is killed - * @param report report + * Build an exit code for an application from its report. + * If the report parameter is null, its interpreted as a timeout + * @param report report application report * @return the exit code + * @throws IOException + * @throws YarnException */ private int buildExitCode(ApplicationReport report) throws IOException, YarnException { if (null == report) { - forceKillApplication("Reached client specified timeout for application"); return EXIT_TIMED_OUT; } @@ -1615,6 +1623,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe log.info("Application Failed. YarnState={}, DSFinalStatus={}", state, dsStatus); return EXIT_YARN_SERVICE_FAILED; + default: //not in any of these states return EXIT_SUCCESS; @@ -1730,7 +1739,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // and those the RM knows about List<ApplicationReport> instances = listSliderInstances(null); - SliderUtils.sortApplicationReport(instances); + SliderUtils.sortApplicationsByMostRecent(instances); Map<String, ApplicationReport> reportMap = SliderUtils.buildApplicationReportMap(instances, min, max); log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}", @@ -1746,6 +1755,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe if (persistent == null) { throw unknownClusterException(clustername); } + // create a new map with only that instance in it. + // this restricts the output of results to this instance persistentInstances = new HashMap<String, Path>(); persistentInstances.put(clustername, persistent); } @@ -1767,6 +1778,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return listed > 0 ? EXIT_SUCCESS: EXIT_FALSE; } + /** + * Convert the instance details of an application to a string + * @param name instance name + * @param report the application report + * @param verbose verbose output + * @return a string + */ String instanceDetailsToString(String name, ApplicationReport report, boolean verbose) { @@ -1994,12 +2012,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return YarnAppListClient.findAllLiveInstances(appname); } - - public ApplicationReport findClusterInInstanceList(List<ApplicationReport> instances, - String appname) { - return yarnClient.findClusterInInstanceList(instances, appname); - } - /** * Connect to a Slider AM * @param app application report providing the details on the application http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/09d16a13/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java index a00f3b2..2f18b7a 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java @@ -252,9 +252,10 @@ public class SliderYarnClientImpl extends YarnClientImpl { /** * Find a cluster in the instance list; biased towards live instances - * @param instances - * @param appname - * @return + * @param instances list of instances + * @param appname application name + * @return the first found instance, else a failed/finished instance, or null + * if there are none of those */ public ApplicationReport findClusterInInstanceList(List<ApplicationReport> instances, String appname) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/09d16a13/slider-core/src/main/java/org/apache/slider/common/Constants.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/Constants.java b/slider-core/src/main/java/org/apache/slider/common/Constants.java index 2fe0250..868ea57 100644 --- a/slider-core/src/main/java/org/apache/slider/common/Constants.java +++ b/slider-core/src/main/java/org/apache/slider/common/Constants.java @@ -19,7 +19,6 @@ package org.apache.slider.common; public class Constants { - public static final int ACCEPT_TIME = 60000; public static final int CONNECT_TIMEOUT = 10000; public static final int RPC_TIMEOUT = 15000; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/09d16a13/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 0f622c9..ee8693f 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -76,6 +76,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InterruptedIOException; import java.io.PrintWriter; +import java.io.Serializable; import java.io.StringWriter; import java.net.InetSocketAddress; import java.net.ServerSocket; @@ -84,6 +85,8 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLDecoder; +import java.text.DateFormat; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -97,6 +100,7 @@ import java.util.Locale; import java.util.Map; import java.util.Properties; import java.util.Set; +import java.util.TimeZone; import java.util.Timer; import java.util.TimerTask; import java.util.TreeMap; @@ -599,36 +603,55 @@ public final class SliderUtils { builder.append(tag).append(separator); } } + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm"); + dateFormat.setTimeZone(TimeZone.getDefault()); builder.append("state: ").append(r.getYarnApplicationState()); - builder.append(separator).append("URL: ").append(r.getTrackingUrl()); + String trackingUrl = r.getTrackingUrl(); + if (isSet(trackingUrl)) { + builder.append(separator).append("URL: ").append(trackingUrl); + } builder.append(separator) - .append("Started ") - .append(new Date(r.getStartTime()).toGMTString()); + .append("Started: ") + .append(dateFormat.format(new Date(r.getStartTime()))); long finishTime = r.getFinishTime(); if (finishTime > 0) { builder.append(separator) - .append("Finished ") - .append(new Date(finishTime).toGMTString()); + .append("Finished: ") + .append(dateFormat.format(new Date(finishTime))); + } + String rpcHost = r.getHost(); + if (!isSet(rpcHost)) { + builder.append(separator) + .append("RPC :") + .append(rpcHost) + .append(':') + .append(r.getRpcPort()); } - builder.append(separator) - .append("RPC :") - .append(r.getHost()) - .append(':') - .append(r.getRpcPort()); String diagnostics = r.getDiagnostics(); - if (!diagnostics.isEmpty()) { + if (!isSet(diagnostics)) { builder.append(separator).append("Diagnostics :").append(diagnostics); } return builder.toString(); } + + /** + * Sorts the given list of application reports, most recently started + * or finished instance first. + * + * @param instances list of instances + */ + public static void sortApplicationsByMostRecent(List<ApplicationReport> instances) { + Collections.sort(instances, new MostRecentlyStartedOrFinishedFirst()); + } + /** * Sorts the given list of application reports * Finished instances are ordered by finished time and running/accepted instances are * ordered by start time * Finally Instance are order by finished instances coming after running instances * - * @param instances list of intances + * @param instances list of instances */ public static void sortApplicationReport(List<ApplicationReport> instances) { if (instances.size() <= 1) { @@ -650,28 +673,10 @@ public final class SliderUtils { } if (liveInstance.size() > 1) { - Comparator<ApplicationReport> liveInstanceComparator = - new Comparator<ApplicationReport>() { - @Override - public int compare(ApplicationReport r1, ApplicationReport r2) { - long x = r1.getStartTime(); - long y = r2.getStartTime(); - return (x < y) ? -1 : ((x == y) ? 0 : 1); - } - }; - Collections.sort(liveInstance, liveInstanceComparator); + Collections.sort(liveInstance, new MostRecentlyStartedAppFirst()); } if (nonLiveInstance.size() > 1) { - Comparator<ApplicationReport> nonLiveInstanceComparator = - new Comparator<ApplicationReport>() { - @Override - public int compare(ApplicationReport r1, ApplicationReport r2) { - long x = r1.getFinishTime(); - long y = r2.getFinishTime(); - return (x < y) ? -1 : ((x == y) ? 0 : 1); - } - }; - Collections.sort(nonLiveInstance, nonLiveInstanceComparator); + Collections.sort(nonLiveInstance, new MostRecentAppFinishFirst()); } instances.clear(); instances.addAll(liveInstance); @@ -2067,4 +2072,63 @@ public final class SliderUtils { } return result; } + + /** + * Compare the times of two applications: most recent app comes first + * Specifically: the one whose start time value is greater. + */ + private static class MostRecentlyStartedAppFirst + implements Comparator<ApplicationReport>, Serializable { + @Override + public int compare(ApplicationReport r1, ApplicationReport r2) { + long x = r1.getStartTime(); + long y = r2.getStartTime(); + return compareTwoLongs(x, y); + } + } + + /** + * Compare the times of two applications: most recent app comes first. + * "Recent"== the app whose start time <i>or finish time</i> is the greatest. + */ + private static class MostRecentlyStartedOrFinishedFirst + implements Comparator<ApplicationReport>, Serializable { + @Override + public int compare(ApplicationReport r1, ApplicationReport r2) { + long started1 = r1.getStartTime(); + long started2 = r2.getStartTime(); + long finished1 = r1.getFinishTime(); + long finished2 = r2.getFinishTime(); + long lastEvent1 = Math.max(started1, finished1); + long lastEvent2 = Math.max(started2, finished2); + return compareTwoLongs(lastEvent1, lastEvent2); + } + } + + /** + * Compare the times of two applications: most recently finished app comes first + * Specifically: the one whose finish time value is greater. + */ + private static class MostRecentAppFinishFirst + implements Comparator<ApplicationReport>, Serializable { + @Override + public int compare(ApplicationReport r1, ApplicationReport r2) { + long x = r1.getFinishTime(); + long y = r2.getFinishTime(); + return compareTwoLongs(x, y); + } + } + + /** + * Compare two long values for sorting. As the return value for + * comparators must be int, the simple value of <code>x-y</code> + * is inapplicable + * @param x x value + * @param y y value + * @return -ve if x is less than y, +ve if y is greater than x; 0 for equality + */ + public static int compareTwoLongs(long x, long y) { + return (x < y) ? -1 : ((x == y) ? 0 : 1); + } + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/09d16a13/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy index b44ae07..6c99ab6 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy @@ -111,9 +111,9 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { SliderShell shell = slider(EXIT_SUCCESS, [ ACTION_INSTALL_PACKAGE, - Arguments.ARG_NAME, TEST_APP_PKG_NAME, - Arguments.ARG_PACKAGE, zipFileName.absolutePath, - Arguments.ARG_REPLACE_PKG + ARG_NAME, TEST_APP_PKG_NAME, + ARG_PACKAGE, zipFileName.absolutePath, + ARG_REPLACE_PKG ]) logShell(shell) log.info "App pkg uploaded at home directory .slider/package/$TEST_APP_PKG_NAME/$TEST_APP_PKG_FILE" http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/09d16a13/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy index f3d91d3..7b50c60 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy @@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.slider.api.StatusKeys import org.apache.slider.common.tools.ConfigHelper -import org.apache.slider.core.main.LauncherExitCodes import org.apache.slider.core.main.ServiceLauncher import org.apache.slider.common.SliderKeys import org.apache.slider.common.SliderXmlConfKeys @@ -615,7 +614,6 @@ abstract class CommandTestBase extends SliderTestUtils { ARG_TEMPLATE, appTemplate, ARG_RESOURCES, resourceTemplate, ARG_WAIT, Integer.toString(THAW_WAIT_TIME) - ] maybeAddCommandOption(commands, @@ -631,7 +629,21 @@ abstract class CommandTestBase extends SliderTestUtils { [ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL], SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL)); commands.addAll(extraArgs) - SliderShell shell = slider(LauncherExitCodes.EXIT_SUCCESS, commands) + SliderShell shell = new SliderShell(commands) + shell.execute() + if (!shell.execute()) { + // app has failed. + + // grab the app report of the last known instance of this app + // which may not be there if it was a config failure; may be out of date + // from a previous run + log.error("Launch failed with exit code ${shell.ret}.\nLast instance of $name:") + slider([ACTION_LIST, name, ARG_VERBOSE]).dumpOutput() + + // trigger the assertion failure + shell.assertExitCode(EXIT_SUCCESS) + } + return shell }