Repository: incubator-slider Updated Branches: refs/heads/develop 3105ba9f3 -> 7e8903e36
SLIDER-752 No easy way to get list of applications via API from SliderClient Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7e8903e3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7e8903e3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7e8903e3 Branch: refs/heads/develop Commit: 7e8903e364ba02ba04efb6b99e153a1eb5120558 Parents: 3105ba9 Author: Steve Loughran <ste...@apache.org> Authored: Thu Jan 29 17:25:38 2015 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Thu Jan 29 17:25:38 2015 +0000 ---------------------------------------------------------------------- .../api/types/SliderInstanceDescription.java | 54 ++++++ .../org/apache/slider/client/SliderClient.java | 75 ++++---- .../apache/slider/client/SliderClientAPI.java | 22 +++ .../slider/client/SliderYarnClientImpl.java | 86 ++++++--- .../apache/slider/common/tools/SliderUtils.java | 37 ++++ .../slider/core/registry/YarnAppListClient.java | 93 +++++++++- .../servicemonitor/YarnApplicationProbe.java | 2 +- .../slider/agent/actions/TestActionList.groovy | 173 ++++++++++++------- 8 files changed, 410 insertions(+), 132 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e8903e3/slider-core/src/main/java/org/apache/slider/api/types/SliderInstanceDescription.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/types/SliderInstanceDescription.java b/slider-core/src/main/java/org/apache/slider/api/types/SliderInstanceDescription.java new file mode 100644 index 0000000..3b95f80 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/api/types/SliderInstanceDescription.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.ApplicationReport; + +/** + * Description of a slider instance + */ +public class SliderInstanceDescription { + + public final String name; + public final Path path; + public final ApplicationReport applicationReport; + + public SliderInstanceDescription(String name, + Path path, + ApplicationReport applicationReport) { + this.name = name; + this.path = path; + this.applicationReport = applicationReport; + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("SliderInstanceDescription{"); + sb.append("name='").append(name).append('\''); + sb.append(", path=").append(path); + sb.append(", applicationReport: ") + .append(applicationReport == null + ? "null" + : (" id " + applicationReport.getApplicationId())); + sb.append('}'); + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e8903e3/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index ace5f09..4dfbe4d 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -70,6 +70,7 @@ import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.SliderClusterProtocol; import org.apache.slider.api.StateValues; import org.apache.slider.api.proto.Messages; +import org.apache.slider.api.types.SliderInstanceDescription; import org.apache.slider.common.Constants; import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; @@ -213,7 +214,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * Yarn client service */ private SliderYarnClientImpl yarnClient; - private YarnAppListClient YarnAppListClient; + private YarnAppListClient yarnAppListClient; private AggregateConf launchedInstanceDefinition; /** @@ -457,7 +458,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe yarnClient.start(); } addService(yarnClient); - YarnAppListClient = + yarnAppListClient = new YarnAppListClient(yarnClient, getUsername(), getConfig()); // create the filesystem sliderFileSystem = new SliderFileSystem(getConfig()); @@ -1993,10 +1994,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @param user user: "" means all users, null means "default" * @return a possibly empty list of Slider AMs */ - @VisibleForTesting + public List<ApplicationReport> listSliderInstances(String user) throws YarnException, IOException { - return YarnAppListClient.listInstances(user); + return yarnAppListClient.listInstances(user); } /** @@ -2023,7 +2024,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * was named but it was not found */ @Override - @VisibleForTesting public int actionList(String clustername, ActionListArgs args) throws IOException, YarnException { verifyBindingsDefined(); @@ -2092,7 +2092,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // list the details if all were requested, or the filtering contained // a report listed++; - String details = instanceDetailsToString(name, report, verbose); + String details = SliderUtils.instanceDetailsToString(name, + report, + verbose); print(details); } } @@ -2101,40 +2103,27 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } /** - * Convert the instance details of an application to a string - * @param name instance name - * @param report the application report - * @param verbose verbose output - * @return a string + * Enumerate slider instances for the current user, and the + * most recent app report, where available. + * @param listOnlyInState boolean to indicate that the instances should + * only include those in a YARN state + * <code> minAppState <= currentState <= maxAppState </code> + * + * @param minAppState minimum application state to include in enumeration. + * @param maxAppState maximum application state to include + * @return a map of application instance name to description + * @throws IOException Any IO problem + * @throws YarnException YARN problems */ - String instanceDetailsToString(String name, - ApplicationReport report, - boolean verbose) { - // format strings - String staticf = "%-30s"; - String reportedf = staticf + " %10s %-40s"; - String livef = reportedf + " %s"; - StringBuilder builder = new StringBuilder(200); - if (report == null) { - builder.append(String.format(staticf, name)); - } else { - // there's a report to look at - String appId = report.getApplicationId().toString(); - String state = report.getYarnApplicationState().toString(); - if (report.getYarnApplicationState() == YarnApplicationState.RUNNING) { - // running: there's a URL - builder.append(String.format(livef, name, state, appId ,report.getTrackingUrl())); - } else { - builder.append(String.format(reportedf, name, state, appId)); - } - if (verbose) { - builder.append('\n'); - builder.append(SliderUtils.appReportToString(report, "\n ")); - } - } - - builder.append('\n'); - return builder.toString(); + @Override + public Map<String, SliderInstanceDescription> enumSliderInstances( + boolean listOnlyInState, + YarnApplicationState minAppState, + YarnApplicationState maxAppState) + throws IOException, YarnException { + return yarnAppListClient.enumSliderInstances(listOnlyInState, + minAppState, + maxAppState); } /** @@ -2233,7 +2222,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe appstate.ordinal() < YarnApplicationState.FINISHED.ordinal(); } else { // scan for instance in single --state state - List<ApplicationReport> userInstances = yarnClient.listInstances(""); + List<ApplicationReport> userInstances = yarnClient.listDeployedInstances(""); state = state.toUpperCase(Locale.ENGLISH); YarnApplicationState desiredState = extractYarnApplicationState(state); ApplicationReport foundInstance = @@ -2300,7 +2289,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @return registry client -valid after the service is inited. */ public YarnAppListClient getYarnAppListClient() { - return YarnAppListClient; + return yarnAppListClient; } /** @@ -2312,7 +2301,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe */ private ApplicationReport findInstance(String appname) throws YarnException, IOException { - return YarnAppListClient.findInstance(appname); + return yarnAppListClient.findInstance(appname); } private RunningApplication findApplication(String appname) @@ -2331,7 +2320,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private List<ApplicationReport> findAllLiveInstances(String appname) throws YarnException, IOException { - return YarnAppListClient.findAllLiveInstances(appname); + return yarnAppListClient.findAllLiveInstances(appname); } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e8903e3/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java index 328ec46..836891d 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java @@ -23,7 +23,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.hadoop.service.Service; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.slider.api.types.SliderInstanceDescription; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; import org.apache.slider.common.params.ActionAMSuicideArgs; import org.apache.slider.common.params.ActionDiagnosticArgs; @@ -46,6 +48,7 @@ import org.apache.slider.providers.AbstractClientProvider; import java.io.IOException; import java.util.List; +import java.util.Map; /** * Interface of those method calls in the slider API that are intended @@ -170,6 +173,25 @@ public interface SliderClientAPI extends Service { int actionList(String clustername, ActionListArgs args) throws IOException, YarnException; /** + * Enumerate slider instances for the current user, and the + * most recent app report, where available. + * @param listOnlyInState boolean to indicate that the instances should + * only include those in a YARN state + * <code> minAppState <= currentState <= maxAppState </code> + * + * @param minAppState minimum application state to include in enumeration. + * @param maxAppState maximum application state to include + * @return a map of application instance name to description + * @throws IOException Any IO problem + * @throws YarnException YARN problems + */ + Map<String, SliderInstanceDescription> enumSliderInstances( + boolean listOnlyInState, + YarnApplicationState minAppState, + YarnApplicationState maxAppState) + throws IOException, YarnException; + + /** * Implement the islive action: probe for a cluster of the given name existing * @return exit code */ http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e8903e3/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java index 856b34c..209169b 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java @@ -18,8 +18,9 @@ package org.apache.slider.client; -import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; @@ -32,7 +33,9 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.tools.CoreFileSystem; import org.apache.slider.common.tools.Duration; +import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.exceptions.BadCommandArgumentsException; import org.slf4j.Logger; @@ -43,6 +46,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -55,6 +59,13 @@ public class SliderYarnClientImpl extends YarnClientImpl { log = LoggerFactory.getLogger(SliderYarnClientImpl.class); /** + * Keyword to use in the {@link #emergencyForceKill(String)} + * operation to force kill <i>all</i> application instances belonging + * to a specific user + */ + public static final String KILL_ALL = "all"; + + /** * Get the RM Client RPC interface * @return an RPC interface valid after initialization and authentication */ @@ -64,12 +75,28 @@ public class SliderYarnClientImpl extends YarnClientImpl { /** - * List Slider instances belonging to a specific user + * List Slider <i>running</i>instances belonging to a specific user. + * @deprecated use {@link #listDeployedInstances(String)} * @param user user: "" means all users * @return a possibly empty list of Slider AMs */ public List<ApplicationReport> listInstances(String user) throws YarnException, IOException { + return listDeployedInstances(user); + } + + /** + * List Slider <i>deployed</i>instances belonging to a specific user. + * <p> + * Deployed means: known about in the YARN cluster; it will include + * any that are in the failed/finished state, as well as those queued + * for starting. + * @param user user: "" means all users + * @return a possibly empty list of Slider AMs + */ + public List<ApplicationReport> listDeployedInstances(String user) + throws YarnException, IOException { + Preconditions.checkArgument(user != null, "Null User"); Set<String> types = new HashSet<String>(1); types.add(SliderKeys.APP_TYPE); List<ApplicationReport> allApps = getApplications(types); @@ -84,18 +111,19 @@ public class SliderYarnClientImpl extends YarnClientImpl { /** - * find all instances of a specific app -if there is >1 in the cluster, + * find all instances of a specific app -if there is more than one in the + * YARN cluster, * this returns them all - * @param user user + * @param user user; use "" for all users * @param appname application name * @return the list of all matching application instances */ - @VisibleForTesting public List<ApplicationReport> findAllInstances(String user, - String appname) throws - IOException, - YarnException { - List<ApplicationReport> instances = listInstances(user); + String appname) + throws IOException, YarnException { + Preconditions.checkArgument(appname != null, "Null application name"); + + List<ApplicationReport> instances = listDeployedInstances(user); List<ApplicationReport> results = new ArrayList<ApplicationReport>(instances.size()); for (ApplicationReport report : instances) { @@ -113,6 +141,8 @@ public class SliderYarnClientImpl extends YarnClientImpl { * @return true if the application is considered live */ public boolean isApplicationLive(ApplicationReport app) { + Preconditions.checkArgument(app != null, "Null app report"); + return app.getYarnApplicationState().ordinal() <= YarnApplicationState.RUNNING.ordinal(); } @@ -120,15 +150,16 @@ public class SliderYarnClientImpl extends YarnClientImpl { /** * Kill a running application - * @param applicationId + * @param applicationId app Id + * @param reason reason: reason for log * @return the response * @throws YarnException YARN problems * @throws IOException IO problems */ public KillApplicationResponse killRunningApplication(ApplicationId applicationId, - String reason) throws - YarnException, - IOException { + String reason) + throws YarnException, IOException { + Preconditions.checkArgument(applicationId != null, "Null application Id"); log.info("Killing application {} - {}", applicationId.getClusterTimestamp(), reason); KillApplicationRequest request = @@ -140,19 +171,23 @@ public class SliderYarnClientImpl extends YarnClientImpl { private String getUsername() throws IOException { return UserGroupInformation.getCurrentUser().getShortUserName(); } + /** * Force kill a yarn application by ID. No niceities here + * @param applicationId app Id. "all" means "kill all instances of the current user + * */ - public void emergencyForceKill(String applicationId) throws - YarnException, - IOException { - + public void emergencyForceKill(String applicationId) + throws YarnException, IOException { + + Preconditions.checkArgument(StringUtils.isNotEmpty(applicationId), + "Null/empty application Id"); - if ("all".equals(applicationId)) { + if (KILL_ALL.equals(applicationId)) { // user wants all instances killed String user = getUsername(); log.info("Killing all applications belonging to {}", user); - Collection<ApplicationReport> instances = listInstances(user); + Collection<ApplicationReport> instances = listDeployedInstances(user); for (ApplicationReport instance : instances) { if (isApplicationLive(instance)) { ApplicationId appId = instance.getApplicationId(); @@ -241,7 +276,9 @@ public class SliderYarnClientImpl extends YarnClientImpl { String appname) throws YarnException, IOException { - List<ApplicationReport> instances = listInstances(user); + Preconditions.checkArgument(StringUtils.isNotEmpty(appname), + "Null/empty application name"); + List<ApplicationReport> instances = listDeployedInstances(user); List<ApplicationReport> results = new ArrayList<ApplicationReport>(instances.size()); for (ApplicationReport app : instances) { @@ -262,6 +299,9 @@ public class SliderYarnClientImpl extends YarnClientImpl { */ public ApplicationReport findClusterInInstanceList(List<ApplicationReport> instances, String appname) { + Preconditions.checkArgument(instances != null, "Null instances list"); + Preconditions.checkArgument(StringUtils.isNotEmpty(appname), + "Null/empty application name"); // sort by most recent SliderUtils.sortApplicationsByMostRecent(instances); ApplicationReport found = null; @@ -287,6 +327,10 @@ public class SliderYarnClientImpl extends YarnClientImpl { public ApplicationReport findAppInInstanceList(List<ApplicationReport> instances, String appname, YarnApplicationState desiredState) { + Preconditions.checkArgument(instances != null, "Null instances list"); + Preconditions.checkArgument(StringUtils.isNotEmpty(appname), + "Null/empty application name"); + Preconditions.checkArgument(desiredState != null, "Null desiredState"); ApplicationReport found = null; ApplicationReport foundAndLive = null; log.debug("Searching {} records for instance name {} in state '{}'", @@ -307,6 +351,4 @@ public class SliderYarnClientImpl extends YarnClientImpl { log.debug("No match"); return null; } - - } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e8903e3/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index aeffe6c..3182bb7 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -633,6 +633,43 @@ public final class SliderUtils { return builder.toString(); } + /** + * Convert the instance details of an application to a string + * @param name instance name + * @param report the application report + * @param verbose verbose output + * @return a string + */ + public static String instanceDetailsToString(String name, + ApplicationReport report, + boolean verbose) { + // format strings + String staticf = "%-30s"; + String reportedf = staticf + " %10s %-40s"; + String livef = reportedf + " %s"; + StringBuilder builder = new StringBuilder(200); + if (report == null) { + builder.append(String.format(staticf, name)); + } else { + // there's a report to look at + String appId = report.getApplicationId().toString(); + String state = report.getYarnApplicationState().toString(); + if (report.getYarnApplicationState() == YarnApplicationState.RUNNING) { + // running: there's a URL + builder.append( + String.format(livef, name, state, appId, report.getTrackingUrl())); + } else { + builder.append(String.format(reportedf, name, state, appId)); + } + if (verbose) { + builder.append('\n'); + builder.append(SliderUtils.appReportToString(report, "\n ")); + } + } + + builder.append('\n'); + return builder.toString(); + } /** * Sorts the given list of application reports, most recently started http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e8903e3/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java b/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java index 6f50fca..1bdfb9c 100644 --- a/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java +++ b/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java @@ -20,12 +20,21 @@ package org.apache.slider.core.registry; import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.slider.client.SliderYarnClientImpl; +import org.apache.slider.api.types.SliderInstanceDescription; +import org.apache.slider.common.tools.CoreFileSystem; +import org.apache.slider.common.tools.SliderUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * Client code for interacting with a list of service instances. @@ -33,9 +42,11 @@ import java.util.List; */ public class YarnAppListClient { - final SliderYarnClientImpl yarnClient; - final String username; - final Configuration conf; + private final SliderYarnClientImpl yarnClient; + private final String username; + private final Configuration conf; + private static final Logger log = + LoggerFactory.getLogger(YarnAppListClient.class); public YarnAppListClient(SliderYarnClientImpl yarnClient, String username, @@ -97,8 +108,82 @@ public class YarnAppListClient { public List<ApplicationReport> listInstances(String user) throws YarnException, IOException { String listUser = user == null ? username : user; - return yarnClient.listInstances(listUser); + return yarnClient.listDeployedInstances(listUser); } + /** + * Enumerate slider instances for the current user, and the + * most recent app report, where available. + * @param listOnlyInState boolean to indicate that the instances should + * only include those in a YARN state + * <code> minAppState <= currentState <= maxAppState </code> + * + * @param minAppState minimum application state to include in enumeration. + * @param maxAppState maximum application state to include + * @return a map of application instance name to description + * @throws IOException Any IO problem + * @throws YarnException YARN problems + */ + public Map<String, SliderInstanceDescription> enumSliderInstances( + boolean listOnlyInState, + YarnApplicationState minAppState, + YarnApplicationState maxAppState) + throws IOException, YarnException { + + CoreFileSystem sliderFileSystem = new CoreFileSystem(conf); + Preconditions.checkArgument(!listOnlyInState || minAppState != null, + "null minAppState when listOnlyInState set"); + Preconditions.checkArgument(!listOnlyInState || maxAppState != null, + "null maxAppState when listOnlyInState set"); + if (!listOnlyInState) { + // if there's not filtering, ask for the entire range of states + minAppState = YarnApplicationState.NEW; + maxAppState = YarnApplicationState.KILLED; + } + // get the complete list of persistent instances + Map<String, Path> persistentInstances = + sliderFileSystem.listPersistentInstances(); + Map<String, SliderInstanceDescription> descriptions = + new HashMap<String, SliderInstanceDescription>(persistentInstances.size()); + + if (persistentInstances.isEmpty()) { + // an empty listing is a success if no cluster was named + log.debug("No application instances found"); + return descriptions; + } + + // enum those the RM knows about + List<ApplicationReport> rmInstances = listInstances(); + SliderUtils.sortApplicationsByMostRecent(rmInstances); + Map<String, ApplicationReport> reportMap = + SliderUtils.buildApplicationReportMap(rmInstances, minAppState, + maxAppState); + log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}", + persistentInstances.size(), + rmInstances.size(), + minAppState, maxAppState, + reportMap.size()); + + // at this point there is a list of all persistent instances, and + // a (possibly filtered) list of application reports + + for (Map.Entry<String, Path> entry : persistentInstances.entrySet()) { + // loop through the persistent values + String name = entry.getKey(); + + // look up any report from the (possibly filtered) report set + ApplicationReport report = reportMap.get(name); + if (!listOnlyInState || report != null) { + // if the enum wants to filter in state, only add it if there is + // a report in that range. Otherwise: include all values + SliderInstanceDescription sid = new SliderInstanceDescription( + name, entry.getValue(), report); + descriptions.put(name, sid); + } + } + + return descriptions; + + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e8903e3/slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java index 8bc6dd0..adf613c 100644 --- a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java +++ b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java @@ -73,7 +73,7 @@ public class YarnApplicationProbe extends Probe { try { List<ApplicationReport> instances = - yarnClient.listInstances(username); + yarnClient.listDeployedInstances(username); ApplicationReport instance = yarnClient.findClusterInInstanceList(instances, clustername); if (null == instance) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e8903e3/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy index bf65b0f..15ad701 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy @@ -19,6 +19,7 @@ package org.apache.slider.agent.actions import groovy.util.logging.Slf4j +import org.apache.hadoop.yarn.api.records.ApplicationId import org.apache.hadoop.yarn.api.records.ApplicationReport import org.apache.hadoop.yarn.api.records.YarnApplicationState import org.apache.hadoop.yarn.conf.YarnConfiguration @@ -55,35 +56,17 @@ class TestActionList extends AgentMiniClusterTestBase { @Test public void testActionListSuite() throws Throwable { testListThisUserNoClusters() - testListLiveCluster() testListMissingCluster() - testActionListStates() - } - - public void testListThisUserNoClusters() throws Throwable { - log.info("RM address = ${RMAddr}") - ServiceLauncher<SliderClient> launcher = launchClientAgainstMiniMR( - //config includes RM binding info - new YarnConfiguration(miniCluster.config), - //varargs list of command line params - [ - SliderActions.ACTION_LIST, - Arguments.ARG_MANAGER, RMAddr - ] - ) - assert launcher.serviceExitCode == 0 + testActionList() } - public void testListLiveCluster() throws Throwable { - //launch the cluster - String clustername = "testlistlivecluster" + public void testActionList() { + String clustername = "testactionlist" ServiceLauncher<SliderClient> launcher = createStandaloneAM( clustername, true, - false) - + true) addToTeardown(launcher) - //do the low level operations to get a better view of what is going on SliderClient sliderClient = launcher.service waitForClusterLive(sliderClient) @@ -98,12 +81,13 @@ class TestActionList extends AgentMiniClusterTestBase { ) assert launcher.serviceExitCode == 0 //now look for the explicit sevice - + def serviceRegistryClient = sliderClient.yarnAppListClient ApplicationReport instance = serviceRegistryClient.findInstance(clustername) assert instance != null log.info(instance.toString()) + ApplicationId originalAppId = instance.applicationId; //now list with the named cluster launcher = launchClientAgainstMiniMR( @@ -114,52 +98,36 @@ class TestActionList extends AgentMiniClusterTestBase { SliderActions.ACTION_LIST, clustername ] ) - clusterActionFreeze(sliderClient, clustername, "stopping first cluster") - } - public void testListMissingCluster() throws Throwable { - describe("exec the list command against an unknown cluster") + describe "listing by state" + //Listing only live instances + assert sliderClient.actionList(clustername, new ActionListArgs(live: true)) == 0; + assert sliderClient.actionList(clustername, + new ActionListArgs(live: true, verbose:true)) == 0; - ServiceLauncher<SliderClient> launcher - try { - launcher = launchClientAgainstMiniMR( - //config includes RM binding info - new YarnConfiguration(miniCluster.config), - //varargs list of command line params - [ - SliderActions.ACTION_LIST, - "no-instance" - ] - ) - fail("expected an exception, got a status code " + launcher.serviceExitCode) - } catch (UnknownApplicationInstanceException e) { - //expected - } - } + // find the same via the low-level operations - public void testActionListStates() { - String clustername = "testactionliststates" - ServiceLauncher<SliderClient> launcher = createStandaloneAM( - clustername, - true, - true) - addToTeardown(launcher) - SliderClient sliderClient = launcher.service - waitForClusterLive(sliderClient) + def instances = sliderClient.enumSliderInstances(false, null, null) + assert instances.size() > 0 + def enumeratedInstance = instances[clustername] + assert enumeratedInstance.name == clustername + assert enumeratedInstance.path.toString().endsWith("/" + clustername) + assert enumeratedInstance.applicationReport != null + assert originalAppId == enumeratedInstance.applicationReport.applicationId + assert enumeratedInstance.applicationReport.yarnApplicationState == YarnApplicationState.RUNNING + + instances = sliderClient.enumSliderInstances(true, + YarnApplicationState.RUNNING, YarnApplicationState.RUNNING) + assert instances[clustername] - describe "listing" - //Listing only live instances - assert sliderClient.actionList(clustername, new ActionListArgs(live: true)) == 0; - assert sliderClient.actionList(clustername, - new ActionListArgs(live: true, verbose:true)) == 0; clusterActionFreeze(sliderClient, clustername, "stopping first cluster") waitForAppToFinish(sliderClient) - + try { // unknown yarn state - int e= sliderClient.actionList(clustername, + int e = sliderClient.actionList(clustername, new ActionListArgs(state: "undefined")); fail("expected failure, got return code of $e") } catch (BadCommandArgumentsException expected) { @@ -168,12 +136,13 @@ class TestActionList extends AgentMiniClusterTestBase { try { // state and --live options - int e= sliderClient.actionList(clustername, + int e = sliderClient.actionList(clustername, new ActionListArgs(state: "running", live: true)); fail("expected failure, got return code of $e") } catch (BadCommandArgumentsException expected) { - + // expected } + //Listing only live instances but prints nothing since instance is frozen/stopped describe("after freeze") @@ -194,6 +163,21 @@ class TestActionList extends AgentMiniClusterTestBase { assert -1 == sliderClient.actionList("", new ActionListArgs(state: YarnApplicationState.RUNNING.toString())); + // now look for finished app state + instances = sliderClient.enumSliderInstances(false, null, null) + enumeratedInstance = instances[clustername] + assert enumeratedInstance.applicationReport.yarnApplicationState == + YarnApplicationState.FINISHED + // look for running apps, expect no match + instances = sliderClient.enumSliderInstances(true, + YarnApplicationState.ACCEPTED, YarnApplicationState.RUNNING) + assert null == instances[clustername] + + // look for terminated apps, expect no match + instances = sliderClient.enumSliderInstances(true, + YarnApplicationState.FINISHED, YarnApplicationState.KILLED) + assert instances[clustername] + // thaw sliderClient.actionThaw(clustername, new ActionThawArgs()); waitForClusterLive(sliderClient) @@ -201,7 +185,7 @@ class TestActionList extends AgentMiniClusterTestBase { describe("Post-thaw listing") assert 0 == sliderClient.actionList(clustername, new ActionListArgs(state: YarnApplicationState.RUNNING.toString())); - + //Listing only live instances assert 0 == sliderClient.actionList(clustername, new ActionListArgs(live: true)); @@ -209,9 +193,74 @@ class TestActionList extends AgentMiniClusterTestBase { //Listing all the instance both history (previously freezed instance) and live assert 0 == sliderClient.actionList("", new ActionListArgs(live: true)); + // look for terminated apps, expect no match + instances = sliderClient.enumSliderInstances(true, + YarnApplicationState.RUNNING, YarnApplicationState.RUNNING) + assert instances[clustername] + def runningId = instances[clustername].applicationReport.applicationId + assert runningId != originalAppId + + // stop again + maybeStopCluster(sliderClient, "", "forced", true) assert 0 == sliderClient.actionList(clustername, new ActionListArgs(state: "killed")); + + // look for terminated apps, match + instances = sliderClient.enumSliderInstances(true, + YarnApplicationState.FINISHED, YarnApplicationState.KILLED) + assert instances[clustername] + + // and verify the report picked up is the latest one + def finishedInstance = instances[clustername] + + def finishedAppReport = finishedInstance.applicationReport + assert runningId == finishedAppReport.applicationId + // which was force killed + assert YarnApplicationState.KILLED == finishedAppReport.yarnApplicationState + + // check that an enum for live apps fails + assert 0 == sliderClient.enumSliderInstances(true, + YarnApplicationState.RUNNING, YarnApplicationState.RUNNING).size() + + // check that an enum for non-live apps works + assert 0 < sliderClient.enumSliderInstances(false, + YarnApplicationState.RUNNING, YarnApplicationState.RUNNING).size() + } + + + + public void testListThisUserNoClusters() throws Throwable { + log.info("RM address = ${RMAddr}") + ServiceLauncher<SliderClient> launcher = launchClientAgainstMiniMR( + //config includes RM binding info + new YarnConfiguration(miniCluster.config), + //varargs list of command line params + [ + SliderActions.ACTION_LIST, + Arguments.ARG_MANAGER, RMAddr + ] + ) + assert launcher.serviceExitCode == 0 + } + public void testListMissingCluster() throws Throwable { + describe("exec the list command against an unknown cluster") + + ServiceLauncher<SliderClient> launcher + try { + launcher = launchClientAgainstMiniMR( + //config includes RM binding info + new YarnConfiguration(miniCluster.config), + //varargs list of command line params + [ + SliderActions.ACTION_LIST, + "no-instance" + ] + ) + fail("expected an exception, got a status code " + launcher.serviceExitCode) + } catch (UnknownApplicationInstanceException e) { + //expected + } } }