SLIDER-306 new list options to get full persistent list, live clusters, clusters in given states
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bd8384c9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bd8384c9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bd8384c9 Branch: refs/heads/develop Commit: bd8384c90ca7d70d4b6969bc18082df6dbbd0d9c Parents: b19e157 Author: Steve Loughran <ste...@apache.org> Authored: Fri Oct 24 21:46:23 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Fri Oct 24 21:51:10 2014 +0100 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 200 ++++++++----- .../apache/slider/client/SliderClientAPI.java | 5 +- .../slider/common/params/ActionListArgs.java | 10 +- .../common/params/ActionRegistryArgs.java | 1 + .../apache/slider/common/params/Arguments.java | 1 - .../slider/common/tools/CoreFileSystem.java | 30 ++ .../apache/slider/common/tools/SliderUtils.java | 290 +++++++++++-------- .../apache/slider/core/build/InstanceIO.java | 8 +- .../slider/core/registry/YarnAppListClient.java | 21 +- .../slider/agent/actions/TestActionList.groovy | 98 ++++--- .../standalone/TestStandaloneAMDestroy.groovy | 2 +- .../funtest/framework/CommandTestBase.groovy | 10 +- .../funtest/commands/ListCommandIT.groovy | 14 +- .../lifecycle/AgentClusterLifecycleIT.groovy | 26 +- .../funtest/HBaseClusterLifecycleIT.groovy | 2 +- 15 files changed, 456 insertions(+), 262 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 4ac4031..66797ae 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathNotFoundException; @@ -1666,100 +1667,162 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } /** - * List Slider instances belonging to a specific user - * @param user user: "" means all users + * List Slider instances belonging to a specific user. This will include + * failed and killed instances; there may be duplicates + * @param user user: "" means all users, null means "default" * @return a possibly empty list of Slider AMs */ @VisibleForTesting public List<ApplicationReport> listSliderInstances(String user) throws YarnException, IOException { - return YarnAppListClient.listInstances(); + return YarnAppListClient.listInstances(user); } - @Override /** - * Implement the list action: list all nodes - * - * live: List out only live instances - * history: List out only history instances - * - * If arguments are not given then list out both finished and - * running instances - * - * @param clustername List out specific cluster - * @param args Action list arguments - * @return exit code of 0 if a list was created + * A basic list action to list live instances + * @param clustername cluster name + * @return success if the listing was considered successful + * @throws IOException + * @throws YarnException */ + public int actionList(String clustername) throws IOException, YarnException { + ActionListArgs args = new ActionListArgs(); + args.live = true; + return actionList(clustername, args); + } + + /** + * Implement the list action: list all nodes + + * @param clustername List out specific instance name + * @param args Action list arguments + * @return 0 if one or more entries were listed + */ + @Override @VisibleForTesting public int actionList(String clustername, ActionListArgs args) throws IOException, YarnException { verifyBindingsDefined(); - String user = UserGroupInformation.getCurrentUser().getUserName(); - List<ApplicationReport> instances = listSliderInstances(user); - SliderUtils.sortApplicationReport(instances); boolean live = args.live; - boolean history = args.history; - if (isUnset(clustername)) { - // no cluster name: list all - log.info("Instances for {}: {}", - (user != null ? user : "all users"), - instances.size()); - for (ApplicationReport report : instances) { - logAppReport(report, live, history); - } - // and always succeed + String state = args.state; + boolean verbose = args.verbose; + + if (live && !state.isEmpty()) { + throw new BadCommandArgumentsException( + Arguments.ARG_LIVE + " and " + Arguments.ARG_STATE + " are exclusive"); + } + // flag to indicate only services in a specific state are to be listed + boolean listOnlyInState = live || !state.isEmpty(); + + YarnApplicationState min, max; + if (live) { + min = YarnApplicationState.NEW; + max = YarnApplicationState.RUNNING; + } else if (!state.isEmpty()) { + YarnApplicationState stateVal = extractYarnApplicationState(state); + min = max = stateVal; + } else { + min = YarnApplicationState.NEW; + max = YarnApplicationState.KILLED; + } + // get the complete list of persistent instances + Map<String, Path> persistentInstances = sliderFileSystem.listPersistentInstances(); + + if (persistentInstances.isEmpty() && isUnset(clustername)) { + // an empty listing is a success if no cluster was named + log.debug("No application instances found"); return EXIT_SUCCESS; + } + + // and those the RM knows about + List<ApplicationReport> instances = listSliderInstances(null); + SliderUtils.sortApplicationReport(instances); + Map<String, ApplicationReport> reportMap = + SliderUtils.buildApplicationReportMap(instances, min, max); + log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}", + persistentInstances.size(), + instances.size(), + min, max, + reportMap.size() ); + + if (isSet(clustername)) { + // only one instance is expected + // resolve the persistent value + Path persistent = persistentInstances.get(clustername); + if (persistent == null) { + throw unknownClusterException(clustername); + } + persistentInstances = new HashMap<String, Path>(); + persistentInstances.put(clustername, persistent); + } + + // at this point there is either the entire list or a stripped down instance + int listed = 0; + + for (String name : persistentInstances.keySet()) { + ApplicationReport report = reportMap.get(name); + if (!listOnlyInState || report != null) { + // list the details if all were requested, or the filtering contained + // a report + listed++; + String details = instanceDetailsToString(name, report, verbose); + print(details); + } + } + + return listed > 0 ? EXIT_SUCCESS: EXIT_FALSE; + } + + String instanceDetailsToString(String name, + ApplicationReport report, + boolean verbose) { + // format strings + String staticf = "%-30s"; + String reportedf = staticf + " %10s %-40s"; + String livef = reportedf + " %s"; + StringBuilder builder = new StringBuilder(200); + if (report == null) { + builder.append(String.format(staticf, name)); } else { - // cluster name provided - SliderUtils.validateClusterName(clustername); - log.debug("Listing cluster named {}, live={}, history={}", - clustername, live, history); - boolean instanceFound = false; - if (history) { - for (ApplicationReport report : instances) { - if (report.getName().equals(clustername)) { - logAppReport(report, live, true); - instanceFound = true; - } - } + // there's a report to look at + String appId = report.getApplicationId().toString(); + String state = report.getYarnApplicationState().toString(); + if (report.getYarnApplicationState() == YarnApplicationState.RUNNING) { + // running: there's a URL + builder.append(String.format(livef, name, state, appId ,report.getTrackingUrl())); } else { - // no history flag, only list live value - ApplicationReport report = - findClusterInInstanceList(instances, clustername); - if (report != null) { - instanceFound = logAppReport(report, true, false); - } + builder.append(String.format(reportedf, name, state, appId)); } - // exit code if the instance was found - if (instanceFound) { - return EXIT_SUCCESS; - } else { - throw unknownClusterException(clustername); + if (verbose) { + builder.append('\n'); + builder.append(SliderUtils.appReportToString(report, "\n ")); } } + + builder.append('\n'); + return builder.toString(); } /** - * Log the application report at INFO - * @param report report to log - * @param live only list live apps - * @param history list historical containers - * @return whether the report was logged or not + * Extract the state of a Yarn application --state argument + * @param state state argument + * @return the application state + * @throws BadCommandArgumentsException if the argument did not match + * any known state */ - public boolean logAppReport(ApplicationReport report, - boolean live, - boolean history) { - // app is active if it is accepted or running - boolean active = isApplicationActive(report); - - boolean toLog = (active && live) || (!active && history); - if (toLog) { - log.info(SliderUtils.appReportToString(report, "\n")); + private YarnApplicationState extractYarnApplicationState(String state) throws + BadCommandArgumentsException { + YarnApplicationState stateVal; + try { + stateVal = YarnApplicationState.valueOf(state.toUpperCase(Locale.ENGLISH)); + } catch (IllegalArgumentException e) { + throw new BadCommandArgumentsException("Unknown state: " + state); + } - return toLog; + return stateVal; } - + /** * Is an application active: accepted or running * @param report the application report @@ -1839,8 +1902,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // scan for instance in single --state state List<ApplicationReport> userInstances = yarnClient.listInstances(""); state = state.toUpperCase(Locale.ENGLISH); - YarnApplicationState desiredState = - YarnApplicationState.valueOf(state); + YarnApplicationState desiredState = extractYarnApplicationState(state); ApplicationReport foundInstance = yarnClient.findAppInInstanceList(userInstances, name, desiredState); if (foundInstance != null) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java index 09ca8fa..61948e3 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java @@ -19,6 +19,7 @@ package org.apache.slider.client; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.hadoop.service.Service; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -32,6 +33,7 @@ import org.apache.slider.common.params.ActionFreezeArgs; import org.apache.slider.common.params.ActionInstallKeytabArgs; import org.apache.slider.common.params.ActionInstallPackageArgs; import org.apache.slider.common.params.ActionKillContainerArgs; +import org.apache.slider.common.params.ActionListArgs; import org.apache.slider.common.params.ActionRegistryArgs; import org.apache.slider.common.params.ActionResolveArgs; import org.apache.slider.common.params.ActionStatusArgs; @@ -41,6 +43,7 @@ import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.providers.AbstractClientProvider; import java.io.IOException; +import java.util.List; /** * Interface of those method calls in the slider API that are intended @@ -138,7 +141,7 @@ public interface SliderClientAPI extends Service { * Implement the list action: list all nodes * @return exit code of 0 if a list was created */ - int actionList(String clustername) throws IOException, YarnException; + int actionList(String clustername, ActionListArgs args) throws IOException, YarnException; /** * Implement the islive action: probe for a cluster of the given name existing http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java index 586f0a1..30cc93e 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java @@ -34,9 +34,13 @@ public class ActionListArgs extends AbstractActionArgs { description = "List only live application instances") public boolean live; - @Parameter(names = {ARG_HISTORY}, - description = "List only historical application instances") - public boolean history; + @Parameter(names = {ARG_STATE}, + description = "list only applications in the specific YARN state") + public String state = ""; + + @Parameter(names = {ARG_VERBOSE}, + description = "print out information in details") + public boolean verbose = false; /** * Get the min #of params expected http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java index 36ba341..4b4c30a 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java @@ -137,6 +137,7 @@ public class ActionRegistryArgs extends AbstractActionArgs { @Parameter(names = {ARG_USER}, description = "the name of the user whose application is being resolved") public String user; + /** * validate health of all the different operations * @throws BadCommandArgumentsException http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java index 693ad2f..06d9dfb 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java @@ -55,7 +55,6 @@ public interface Arguments { String ARG_GETEXP = "--getexp"; String ARG_GETFILES = "--getfiles"; String ARG_HELP = "--help"; - String ARG_HISTORY = "--history"; String ARG_ID = "--id"; String ARG_IMAGE = "--image"; String ARG_INTERNAL = "--internal"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java index 23eeefe..7377dd3 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java @@ -555,6 +555,36 @@ public class CoreFileSystem { return builder.toString(); } + /** + * List all application instances persisted for this user, giving the + * patha. The instance name is the last element in the path + * @return a possibly empty map of application instance names to paths + */ + public Map<String, Path> listPersistentInstances() throws IOException { + FileSystem fs = getFileSystem(); + Path path = new Path(getBaseApplicationPath(), SliderKeys.CLUSTER_DIRECTORY); + if (!fs.exists(path)) { + // special case: no instances have ever been created + return new HashMap<String, Path>(0); + } + FileStatus[] statuses = fs.listStatus(path); + Map<String, Path> instances = new HashMap<String, Path>(statuses.length); + + // enum the child entries + for (FileStatus status : statuses) { + if (status.isDirectory()) { + // for directories, look for an internal.json underneath + Path child = status.getPath(); + Path internalJson = new Path(child, Filenames.INTERNAL); + if (fs.exists(internalJson)) { + // success => this is an instance + instances.put(child.getName(), child); + } + } + } + return instances; + } + public void touch(Path path, boolean overwrite) throws IOException { FSDataOutputStream out = fileSystem.create(path, overwrite); out.close(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 106a052..a51366e 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -97,6 +97,7 @@ import java.util.Properties; import java.util.Set; import java.util.Timer; import java.util.TimerTask; +import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; @@ -513,7 +514,7 @@ public final class SliderUtils { int length = separator.length(); String s = b.toString(); return (trailing || s.isEmpty()) ? - s : (b.substring(0, b.length() - length)); + s : (b.substring(0, b.length() - length)); } /** @@ -631,12 +632,15 @@ public final class SliderUtils { if (instances.size() <= 1) { return; } - List<ApplicationReport> nonLiveInstance = new ArrayList<ApplicationReport>(instances.size()); - List<ApplicationReport> liveInstance = new ArrayList<ApplicationReport>(instances.size()); + List<ApplicationReport> nonLiveInstance = + new ArrayList<ApplicationReport>(instances.size()); + List<ApplicationReport> liveInstance = + new ArrayList<ApplicationReport>(instances.size()); for (ApplicationReport report : instances) { if (report.getYarnApplicationState() == YarnApplicationState.RUNNING - || report.getYarnApplicationState() == YarnApplicationState.ACCEPTED) { + || + report.getYarnApplicationState() == YarnApplicationState.ACCEPTED) { liveInstance.add(report); } else { nonLiveInstance.add(report); @@ -644,21 +648,23 @@ public final class SliderUtils { } if (liveInstance.size() > 1) { - Comparator<ApplicationReport> liveInstanceComparator = new Comparator<ApplicationReport>() { - @Override - public int compare(ApplicationReport r1, ApplicationReport r2) { - return Long.compare(r1.getStartTime(), r2.getStartTime()); - } - }; + Comparator<ApplicationReport> liveInstanceComparator = + new Comparator<ApplicationReport>() { + @Override + public int compare(ApplicationReport r1, ApplicationReport r2) { + return Long.compare(r1.getStartTime(), r2.getStartTime()); + } + }; Collections.sort(liveInstance, liveInstanceComparator); } if (nonLiveInstance.size() > 1) { - Comparator<ApplicationReport> nonLiveInstanceComparator = new Comparator<ApplicationReport>() { - @Override - public int compare(ApplicationReport r1, ApplicationReport r2) { - return Long.compare(r1.getFinishTime(), r2.getFinishTime()); - } - }; + Comparator<ApplicationReport> nonLiveInstanceComparator = + new Comparator<ApplicationReport>() { + @Override + public int compare(ApplicationReport r1, ApplicationReport r2) { + return Long.compare(r1.getFinishTime(), r2.getFinishTime()); + } + }; Collections.sort(nonLiveInstance, nonLiveInstanceComparator); } instances.clear(); @@ -667,6 +673,32 @@ public final class SliderUtils { } /** + * Built a (sorted) map of application reports, mapped to the instance name + * The list is sorted, and the addition process does not add a report + * if there is already one that exists. If the list handed in is sorted, + * those that are listed first form the entries returned + * @param instances list of intances + * @param minState minimum YARN state to be included + * @param maxState maximum YARN state to be included + * @return all reports in the list whose state >= minimum and <= maximum + */ + public static Map<String, ApplicationReport> buildApplicationReportMap( + List<ApplicationReport> instances, + YarnApplicationState minState, YarnApplicationState maxState) { + TreeMap<String, ApplicationReport> map = new TreeMap<String, ApplicationReport>(); + for (ApplicationReport report : instances) { + YarnApplicationState state = report.getYarnApplicationState(); + if (state.ordinal() >= minState.ordinal() && + state.ordinal() <= maxState.ordinal() && + map.get(report.getName()) == null) { + map.put(report.getName(), report); + } + } + return map; + } + + + /** * Merge in one map to another -all entries in the second map are * merged into the first -overwriting any duplicate keys. * @param first first map -the updated one. @@ -1079,7 +1111,8 @@ public final class SliderUtils { UserGroupInformation.AuthenticationMethod.SIMPLE) { throw new BadConfigException("Auth User is not Kerberized %s" + " -security has already been set up with the wrong authentication method. " - + "This can occur if a file system has already been created prior to the loading of " + + + "This can occur if a file system has already been created prior to the loading of " + "the security configuration.", authUser); @@ -1415,7 +1448,7 @@ public final class SliderUtils { String appHomeOption = internalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME); if (!isUnset(imagePathOption)) { - if(!isUnset(appHomeOption)) { + if (!isUnset(appHomeOption)) { throw new BadClusterStateException( ErrorStrings.E_BOTH_IMAGE_AND_HOME_DIR_SPECIFIED); } @@ -1627,7 +1660,9 @@ public final class SliderUtils { * @throws FileNotFoundException if the file is not considered valid * @param logger */ - public static void maybeVerifyWinUtilsValid(Logger logger) throws IOException, SliderException { + public static void maybeVerifyWinUtilsValid(Logger logger) throws + IOException, + SliderException { if (!Shell.WINDOWS) { return; } @@ -1668,19 +1703,19 @@ public final class SliderUtils { * Look for the windows executable and check it has the right headers. * <code>File.canRead()</code> doesn't work on windows, so the reading * is mandatory. - * + * * @param program program name for errors * @param exe executable * @throws IOException IOE */ - public static void verifyWindowsExe(String program, File exe) + public static void verifyWindowsExe(String program, File exe) throws IOException { verifyIsFile(program, exe); verifyFileSize(program, exe, 0x100); // now read two bytes and verify the header. - + FileReader reader = null; try { int[] header = new int[2]; @@ -1811,7 +1846,7 @@ public final class SliderUtils { } } } - if (errorText== null) { + if (errorText == null) { return process; } @@ -1862,107 +1897,110 @@ public final class SliderUtils { execCommand(PYTHON, 0, 5000, logger, "Python", PYTHON, "--version"); } - /** - * return the path to the currently running slider command - * - * @throws NullPointerException - * - If the pathname argument is null - * @throws SecurityException - * - if a security manager exists and its checkPermission method - * doesn't allow getting the ProtectionDomain - */ - public static String getCurrentCommandPath() { - File f = new File(Slider.class.getProtectionDomain().getCodeSource() - .getLocation().getPath()); - return f.getAbsolutePath(); - } - - /** - * return the path to the slider-client.xml used by the current running - * slider command - * - * @throws SecurityException - * - if a security manager exists and its checkPermission method - * denies access to the class loader for the class - */ - public static String getClientConfigPath() { - URL path = ConfigHelper.class.getClassLoader().getResource( - SliderKeys.CLIENT_RESOURCE); - return path.toString(); - } - - /** - * validate if slider-client.xml under the path can be opened - * - * @throws IOException - * : the file can't be found or open - */ - public static void validateClientConfigFile() throws IOException { - URL resURL = SliderVersionInfo.class.getClassLoader().getResource( - SliderKeys.CLIENT_RESOURCE); - if (resURL == null) { - throw new IOException( - "slider-client.xml doesn't exist on the path: " - + getClientConfigPath()); - } - - try { - InputStream inStream = resURL.openStream(); - if (inStream == null) { - throw new IOException("slider-client.xml can't be opened"); - } - } catch (IOException e) { - throw new IOException("slider-client.xml can't be opened: " - + e.toString()); - } - } - - /** - * validate if a file on HDFS can be open - * - * @throws IOException the file can't be found or opened - * @throws URISyntaxException - */ - public static void validateHDFSFile(SliderFileSystem sliderFileSystem, String pathStr) - throws IOException, URISyntaxException{ - URI pathURI = new URI(pathStr); - InputStream inputStream = sliderFileSystem.getFileSystem().open(new Path(pathURI)); - if(inputStream == null){ - throw new IOException("HDFS file " + pathStr + " can't be opened"); - } - } - - /** - * return the version and path of the JDK invoking the current running - * slider command - * - * @throws SecurityException - * - if a security manager exists and its checkPropertyAccess - * method doesn't allow access to the specified system property. - */ - public static String getJDKInfo() { - String version = System.getProperty("java.version"); - String javaHome = System.getProperty("java.home"); - return "The version of the JDK invoking the current running slider command: " - + version + "; The path to it is: " + javaHome; - } - - /** - * return a description of whether the current user has created credential - * cache files from kerberos servers - * - * @throws IOException - * @throws BadConfigException - * @throws SecurityException - * - if a security manager exists and its checkPropertyAccess - * method doesn't allow access to the specified system property. - */ - public static String checkCredentialCacheFile() throws IOException, - BadConfigException { - String result = null; - if (!Shell.WINDOWS) { - result = Shell.execCommand("klist"); - } - return result; - } + /** + * return the path to the currently running slider command + * + * @throws NullPointerException + * - If the pathname argument is null + * @throws SecurityException + * - if a security manager exists and its checkPermission method + * doesn't allow getting the ProtectionDomain + */ + public static String getCurrentCommandPath() { + File f = new File(Slider.class.getProtectionDomain().getCodeSource() + .getLocation().getPath()); + return f.getAbsolutePath(); + } + + /** + * return the path to the slider-client.xml used by the current running + * slider command + * + * @throws SecurityException + * - if a security manager exists and its checkPermission method + * denies access to the class loader for the class + */ + public static String getClientConfigPath() { + URL path = ConfigHelper.class.getClassLoader().getResource( + SliderKeys.CLIENT_RESOURCE); + return path.toString(); + } + + /** + * validate if slider-client.xml under the path can be opened + * + * @throws IOException + * : the file can't be found or open + */ + public static void validateClientConfigFile() throws IOException { + URL resURL = SliderVersionInfo.class.getClassLoader().getResource( + SliderKeys.CLIENT_RESOURCE); + if (resURL == null) { + throw new IOException( + "slider-client.xml doesn't exist on the path: " + + getClientConfigPath()); + } + + try { + InputStream inStream = resURL.openStream(); + if (inStream == null) { + throw new IOException("slider-client.xml can't be opened"); + } + } catch (IOException e) { + throw new IOException("slider-client.xml can't be opened: " + + e.toString()); + } + } + + /** + * validate if a file on HDFS can be open + * + * @throws IOException the file can't be found or opened + * @throws URISyntaxException + */ + public static void validateHDFSFile(SliderFileSystem sliderFileSystem, + String pathStr) + throws IOException, URISyntaxException { + URI pathURI = new URI(pathStr); + InputStream inputStream = + sliderFileSystem.getFileSystem().open(new Path(pathURI)); + if (inputStream == null) { + throw new IOException("HDFS file " + pathStr + " can't be opened"); + } + } + + /** + * return the version and path of the JDK invoking the current running + * slider command + * + * @throws SecurityException + * - if a security manager exists and its checkPropertyAccess + * method doesn't allow access to the specified system property. + */ + public static String getJDKInfo() { + String version = System.getProperty("java.version"); + String javaHome = System.getProperty("java.home"); + return + "The version of the JDK invoking the current running slider command: " + + version + "; The path to it is: " + javaHome; + } + + /** + * return a description of whether the current user has created credential + * cache files from kerberos servers + * + * @throws IOException + * @throws BadConfigException + * @throws SecurityException + * - if a security manager exists and its checkPropertyAccess + * method doesn't allow access to the specified system property. + */ + public static String checkCredentialCacheFile() throws IOException, + BadConfigException { + String result = null; + if (!Shell.WINDOWS) { + result = Shell.execCommand("klist"); + } + return result; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java index f923ef3..25bb4ab 100644 --- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java +++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java @@ -38,15 +38,15 @@ public class InstanceIO { * Load in an instance definition -but do not resolve it * @param sliderFileSystem filesystem * @param clusterDirectory CD - * @return + * @return the unresolved aggregate configuration * @throws IOException * @throws SliderException + * @throws BadClusterStateException if a lock could not be acquired */ public static AggregateConf loadInstanceDefinitionUnresolved( CoreFileSystem sliderFileSystem, - Path clusterDirectory) throws - IOException, - SliderException { + Path clusterDirectory) + throws IOException, SliderException { AggregateConf instanceDefinition = new AggregateConf(); ConfPersister persister = new ConfPersister(sliderFileSystem, clusterDirectory); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java b/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java index 068b687..6f50fca 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java @@ -37,7 +37,6 @@ public class YarnAppListClient { final String username; final Configuration conf; - public YarnAppListClient(SliderYarnClientImpl yarnClient, String username, Configuration conf) { @@ -76,17 +75,29 @@ public class YarnAppListClient { public ApplicationReport findInstance(String appname) throws YarnException, IOException { - List<ApplicationReport> instances = listInstances(); + List<ApplicationReport> instances = listInstances(null); return yarnClient.findClusterInInstanceList(instances, appname); } /** - * List instances belonging to a specific user + * List instances belonging to the specific user * @return a possibly empty list of AMs */ public List<ApplicationReport> listInstances() - throws YarnException, IOException { - return yarnClient.listInstances(username); + throws YarnException, IOException { + return listInstances(null); + } + + /** + * List instances belonging to a specific user + * @return a possibly empty list of AMs + * @param user user if not the default. null means default, "" means all users, + * otherwise it is the name of a user + */ + public List<ApplicationReport> listInstances(String user) + throws YarnException, IOException { + String listUser = user == null ? username : user; + return yarnClient.listInstances(listUser); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy index 4b4daf0..1e09d09 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy @@ -20,6 +20,7 @@ package org.apache.slider.agent.actions import groovy.util.logging.Slf4j import org.apache.hadoop.yarn.api.records.ApplicationReport +import org.apache.hadoop.yarn.api.records.YarnApplicationState import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.slider.agent.AgentMiniClusterTestBase import org.apache.slider.client.SliderClient @@ -27,6 +28,7 @@ import org.apache.slider.common.params.ActionListArgs import org.apache.slider.common.params.ActionThawArgs import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.SliderActions +import org.apache.slider.core.exceptions.BadCommandArgumentsException import org.apache.slider.core.exceptions.UnknownApplicationInstanceException import org.apache.slider.core.main.ServiceLauncher import org.junit.Before @@ -36,7 +38,6 @@ import org.junit.Test * Test List operations */ @Slf4j - class TestActionList extends AgentMiniClusterTestBase { @Before @@ -54,10 +55,9 @@ class TestActionList extends AgentMiniClusterTestBase { @Test public void testSuite() throws Throwable { testListThisUserNoClusters() - testListAllUsersNoClusters() testListLiveCluster() testListMissingCluster() - testActionListHistory() + testActionListStates() } public void testListThisUserNoClusters() throws Throwable { @@ -73,28 +73,15 @@ class TestActionList extends AgentMiniClusterTestBase { ) assert launcher.serviceExitCode == 0 } - - public void testListAllUsersNoClusters() throws Throwable { - log.info("RM address = ${RMAddr}") - ServiceLauncher<SliderClient> launcher = launchClientAgainstMiniMR( - //config includes RM binding info - new YarnConfiguration(miniCluster.config), - //varargs list of command line params - [ - SliderActions.ACTION_LIST, - Arguments.ARG_MANAGER, RMAddr, - ] - ) - assert launcher.serviceExitCode == 0 - } public void testListLiveCluster() throws Throwable { //launch the cluster - String clustername = createClusterName() + String clustername = "testlistlivecluster" ServiceLauncher<SliderClient> launcher = createStandaloneAM( clustername, true, false) + addToTeardown(launcher) //do the low level operations to get a better view of what is going on SliderClient sliderClient = launcher.service @@ -127,7 +114,7 @@ class TestActionList extends AgentMiniClusterTestBase { SliderActions.ACTION_LIST, clustername ] ) - + clusterActionFreeze(sliderClient, clustername, "stopping first cluster") } public void testListMissingCluster() throws Throwable { @@ -151,8 +138,8 @@ class TestActionList extends AgentMiniClusterTestBase { } - public void testActionListHistory() { - String clustername = createClusterName() + public void testActionListStates() { + String clustername = "testactionliststates" ServiceLauncher<SliderClient> launcher = createStandaloneAM( clustername, true, @@ -161,42 +148,71 @@ class TestActionList extends AgentMiniClusterTestBase { SliderClient sliderClient = launcher.service waitForClusterLive(sliderClient) - ActionListArgs args = new ActionListArgs(); + describe "listing" //Listing only live instances - args.live = true; - assert sliderClient.actionList(clustername, args) == 0; + assert sliderClient.actionList(clustername, new ActionListArgs(live: true)) == 0; + assert sliderClient.actionList(clustername, + new ActionListArgs(live: true, verbose:true)) == 0; clusterActionFreeze(sliderClient, clustername, "stopping first cluster") waitForAppToFinish(sliderClient) - //Listing only live instances but prints nothing since instance is freezed/stopped - - args.live = true; - args.history = false; + try { - sliderClient.actionList(clustername, args); - fail("expected a failure") - } catch (UnknownApplicationInstanceException e) { - + // unknown yarn state + int e= sliderClient.actionList(clustername, + new ActionListArgs(state: "undefined")); + fail("expected failure, got return code of $e") + } catch (BadCommandArgumentsException expected) { + + } + + try { + // state and --live options + int e= sliderClient.actionList(clustername, + new ActionListArgs(state: "running", live: true)); + fail("expected failure, got return code of $e") + } catch (BadCommandArgumentsException expected) { + } + //Listing only live instances but prints nothing since instance is frozen/stopped + + describe("after freeze") + // listing finished will work + assert 0 == sliderClient.actionList("", + new ActionListArgs(state: YarnApplicationState.FINISHED.toString())); + assert 0 == sliderClient.actionList(clustername, + new ActionListArgs(state: YarnApplicationState.FINISHED.toString(), + verbose: true)); + + assert -1 == sliderClient.actionList("", new ActionListArgs(live: true)); + assert -1 == sliderClient.actionList(clustername, + new ActionListArgs(live: true)); - // historical list will work - args.history = true; - assert 0 == sliderClient.actionList(clustername, args) + assert -1 == sliderClient.actionList(clustername, + new ActionListArgs(state: YarnApplicationState.RUNNING.toString())); + + assert -1 == sliderClient.actionList("", + new ActionListArgs(state: YarnApplicationState.RUNNING.toString())); // thaw - ActionThawArgs thawArgs = new ActionThawArgs(); - sliderClient.actionThaw(clustername, thawArgs); + sliderClient.actionThaw(clustername, new ActionThawArgs()); waitForClusterLive(sliderClient) + describe("Post-thaw listing") + assert 0 == sliderClient.actionList(clustername, + new ActionListArgs(state: YarnApplicationState.RUNNING.toString())); + //Listing only live instances - args.live = true; - args.history = false; - assert 0 == sliderClient.actionList(clustername, args); + assert 0 == sliderClient.actionList(clustername, + new ActionListArgs(live: true)); //Listing all the instance both history (previously freezed instance) and live args.live = true - args.history = true assert 0 == sliderClient.actionList("", args); + + maybeStopCluster(sliderClient, "", "forced", true) + assert 0 == sliderClient.actionList(clustername, + new ActionListArgs(state: "killed")); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy index ed716e0..6593a0e 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy @@ -65,7 +65,7 @@ class TestStandaloneAMDestroy extends AgentMiniClusterTestBase { [ SliderActions.ACTION_LIST, "no-cluster-of-this-name", - Arguments.ARG_LIVE, Arguments.ARG_HISTORY + Arguments.ARG_LIVE ]) fail("expected a failure") } catch (UnknownApplicationInstanceException e) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy index fb1a980..24bcc0e 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy @@ -370,14 +370,8 @@ abstract class CommandTestBase extends SliderTestUtils { slider(cmd) } - static SliderShell list(int result, String name) { - List<String> cmd = [ - ACTION_LIST - ] - if (name != null) { - cmd << name - } - slider(result, cmd) + static SliderShell list(int result, Collection<String> commands =[]) { + slider(result, [ACTION_LIST] + commands ) } static SliderShell status(String name) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy index 20bac88..7ba7fc0 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy @@ -30,7 +30,19 @@ public class ListCommandIT extends CommandTestBase { @Test public void testListAll() throws Throwable { - assertSuccess(list(null)) + assertSuccess(list("")) + } + + @Test + public void testListAllLive() throws Throwable { + def shell = list("--live") + assert shell.ret == 0 || shell.ret ==1 + } + + @Test + public void testListAllFinishedLive() throws Throwable { + def shell = list("--live") + assert shell.ret == 0 || shell.ret == 1 } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy index dfdbf06..f0fa546 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy @@ -92,7 +92,13 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase exists(0, CLUSTER) //listing the cluster will succeed - list(0, CLUSTER) + list(0, [CLUSTER]) + + list(0, [""]) + list(0, [CLUSTER, ARG_LIVE]) + list(0, [CLUSTER, ARG_STATE, "running"]) + list(0, [ARG_LIVE]) + list(0, [ARG_STATE, "running"]) //simple status status(0, CLUSTER) @@ -132,6 +138,15 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase //condition returns false if it is required to be live exists(EXIT_FALSE, CLUSTER, true) + list( 0, [CLUSTER]) + list( 0, [CLUSTER, ARG_STATE, "FINISHED"]) + list(-1, [CLUSTER, ARG_LIVE]) + list(-1, [CLUSTER, ARG_STATE, "running"]) + + list(-1, [ARG_LIVE]) + list(-1, [ARG_STATE, "running"]) + list( 0, [ARG_STATE, "FINISHED"]) + //start then stop the cluster thaw(CLUSTER, [ @@ -139,6 +154,11 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase ]) exists(0, CLUSTER) describe " >>> Cluster is now thawed." + list(0, [CLUSTER, ARG_LIVE]) + list(0, [CLUSTER, ARG_STATE, "running"]) + list(0, [ARG_LIVE]) + list(0, [ARG_STATE, "running"]) + list(0, [CLUSTER, ARG_STATE, "FINISHED"]) freeze(0, CLUSTER, [ @@ -180,10 +200,14 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase ARG_MESSAGE, "final-shutdown" ]) + list(0, [CLUSTER, "--verbose", "--state", "FINISHED"]).dumpOutput() + destroy(0, CLUSTER) //cluster now missing exists(EXIT_UNKNOWN_INSTANCE, CLUSTER) + list(0, []) + list(EXIT_UNKNOWN_INSTANCE, [CLUSTER]) } finally { jsonStatus.delete() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy ---------------------------------------------------------------------- diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy index 7943464..5ffee83 100644 --- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy +++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy @@ -88,7 +88,7 @@ public class HBaseClusterLifecycleIT extends HBaseCommandTestBase exists(0, CLUSTER) //listing the cluster will succeed - list(0, CLUSTER) + list(0, [CLUSTER]) //simple status status(0, CLUSTER)