SLIDER-579 Add CLI operations/--out args to get the app reports of launched apps
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/25bbfebb Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/25bbfebb Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/25bbfebb Branch: refs/heads/develop Commit: 25bbfebb052c59c74950225db759ee93f2f181ba Parents: 7b63000 Author: Steve Loughran <ste...@apache.org> Authored: Wed Oct 29 14:59:00 2014 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Fri Oct 31 11:07:49 2014 +0000 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 54 +++++++++-- .../slider/common/params/ActionCreateArgs.java | 7 ++ .../slider/common/params/ActionExistsArgs.java | 8 +- .../slider/common/params/ActionLookupArgs.java | 76 +++++++++++++++ .../slider/common/params/ActionThawArgs.java | 8 ++ .../apache/slider/common/params/Arguments.java | 10 +- .../apache/slider/common/params/ClientArgs.java | 20 +++- .../common/params/LaunchArgsAccessor.java | 4 + .../common/params/LaunchArgsDelegate.java | 11 +++ .../common/params/SliderAMCreateAction.java | 7 ++ .../slider/common/params/SliderActions.java | 3 + .../slider/core/launch/AppMasterLauncher.java | 49 +++++++--- .../slider/core/launch/LaunchedApplication.java | 6 ++ .../launch/SerializedApplicationReport.java | 97 ++++++++++++++++++++ .../core/persist/ApplicationReportSerDeser.java | 57 ++++++++++++ .../slider/core/persist/JsonSerDeser.java | 21 ++++- .../TestFreezeThawFlexStandaloneAM.groovy | 52 ++++++++++- 17 files changed, 457 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 6ebbcdd..f1f8f97 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -45,6 +45,8 @@ import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.registry.client.api.RegistryOperations; @@ -55,6 +57,7 @@ import org.apache.hadoop.registry.client.exceptions.NoRecordException; import org.apache.hadoop.registry.client.types.Endpoint; import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.ClusterNode; import org.apache.slider.api.InternalKeys; @@ -78,6 +81,7 @@ import org.apache.slider.common.params.ActionFlexArgs; import org.apache.slider.common.params.ActionFreezeArgs; import org.apache.slider.common.params.ActionKillContainerArgs; import org.apache.slider.common.params.ActionListArgs; +import org.apache.slider.common.params.ActionLookupArgs; import org.apache.slider.common.params.ActionRegistryArgs; import org.apache.slider.common.params.ActionResolveArgs; import org.apache.slider.common.params.ActionStatusArgs; @@ -114,7 +118,9 @@ import org.apache.slider.core.launch.CommandLineBuilder; import org.apache.slider.core.launch.JavaCommandLineBuilder; import org.apache.slider.core.launch.LaunchedApplication; import org.apache.slider.core.launch.RunningApplication; +import org.apache.slider.core.launch.SerializedApplicationReport; import org.apache.slider.core.main.RunService; +import org.apache.slider.core.persist.ApplicationReportSerDeser; import org.apache.slider.core.persist.ConfPersister; import org.apache.slider.core.persist.LockAcquireFailedException; import org.apache.slider.core.registry.SliderRegistryUtils; @@ -389,6 +395,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe serviceArgs.getActionAMSuicideArgs()); } else if (ACTION_LIST.equals(action)) { exitCode = actionList(clusterName, serviceArgs.getActionListArgs()); + } else if (ACTION_LOOKUP.equals(action)) { + exitCode = actionLookup(serviceArgs.getActionLookupArgs()); } else if (ACTION_REGISTRY.equals(action)) { exitCode = actionRegistry(serviceArgs.getActionRegistryArgs()); } else if (ACTION_RESOLVE.equals(action)) { @@ -1006,6 +1014,14 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe serviceArgs.isDebug()); applicationId = launchedApplication.getApplicationId(); + if (launchArgs.getOutputFile() != null) { + // output file has been requested. Get the app report and serialize it + ApplicationReport report = + launchedApplication.getApplicationReport(); + SerializedApplicationReport sar = new SerializedApplicationReport(report); + ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser(); + serDeser.save(sar, launchArgs.getOutputFile()); + } int waittime = launchArgs.getWaittime(); if (waittime > 0) { return waitForAppRunning(launchedApplication, waittime, waittime); @@ -1374,12 +1390,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe amLauncher.setQueue(amQueue); } - // Submit the application to the applications manager - // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); - // Ignore the response as either a valid response object is returned on success - // or an exception thrown to denote some form of a failure - - // submit the application LaunchedApplication launchedApplication = amLauncher.submitApplication(); return launchedApplication; @@ -3300,6 +3310,38 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe print("\n"); } + /** + * Implement the lookup action. + * @param args Action arguments + * @return 0 if the entry was found + * @throws IOException + * @throws YarnException + * @throws UnknownApplicationInstanceException if a specific instance + * was named but it was not found + */ + @VisibleForTesting + public int actionLookup(ActionLookupArgs args) + throws IOException, YarnException { + verifyBindingsDefined(); + try { + ApplicationId id = ConverterUtils.toApplicationId(args.id); + ApplicationReport report = yarnClient.getApplicationReport(id); + SerializedApplicationReport sar = new SerializedApplicationReport(report); + ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser(); + if (args.outputFile != null) { + serDeser.save(sar, args.outputFile); + } else { + println(serDeser.toJson(sar)); + } + } catch (IllegalArgumentException e) { + throw new BadCommandArgumentsException(e, "%s : %s", args, e); + } catch (ApplicationAttemptNotFoundException notFound) { + throw new NotFoundException(notFound, notFound.toString()); + } catch (ApplicationNotFoundException notFound) { + throw new NotFoundException(notFound, notFound.toString()); + } + return EXIT_SUCCESS; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java index cfcfb9d..e70f30a 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java @@ -21,6 +21,8 @@ package org.apache.slider.common.params; import com.beust.jcommander.Parameters; import com.beust.jcommander.ParametersDelegate; +import java.io.File; + @Parameters(commandNames = {SliderActions.ACTION_CREATE}, commandDescription = SliderActions.DESCRIBE_ACTION_CREATE) @@ -36,6 +38,11 @@ public class ActionCreateArgs extends AbstractClusterBuildingActionArgs LaunchArgsDelegate launchArgs = new LaunchArgsDelegate(); @Override + public File getOutputFile() { + return launchArgs.getOutputFile(); + } + + @Override public String getRmAddress() { return launchArgs.getRmAddress(); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java index 047d32c..dd1c04b 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java @@ -21,6 +21,8 @@ package org.apache.slider.common.params; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import java.io.File; + @Parameters(commandNames = {SliderActions.ACTION_EXISTS}, commandDescription = SliderActions.DESCRIBE_ACTION_EXISTS) @@ -30,6 +32,7 @@ public class ActionExistsArgs extends AbstractActionArgs { public String getActionName() { return SliderActions.ACTION_EXISTS; } + @Parameter(names = {ARG_LIVE}, description = "verify that the application is running") public boolean live; @@ -37,5 +40,8 @@ public class ActionExistsArgs extends AbstractActionArgs { @Parameter(names = {ARG_STATE}, description = "verify that the application is in the specific YARN state") public String state = ""; - + + @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT}, + description = "output file for any application report") + public File out; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java new file mode 100644 index 0000000..3b69e74 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import org.apache.commons.lang.StringUtils; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; +import org.apache.slider.core.exceptions.UsageException; + +import java.io.File; + +@Parameters(commandNames = {SliderActions.ACTION_LOOKUP}, + commandDescription = SliderActions.DESCRIBE_ACTION_LOOKUP) + +public class ActionLookupArgs extends AbstractActionArgs { + @Override + public String getActionName() { + return SliderActions.ACTION_LOOKUP; + } + + public int getMinParams() { + return 0; + } + public int getMaxParams() { + return 0; + } + + @Parameter(names = {ARG_ID}, + description = "ID of the container") + public String id; + + @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT}, + description = "output file for any application report") + public File outputFile; + + @Override + public void validate() throws BadCommandArgumentsException, UsageException { + super.validate(); + if (StringUtils.isEmpty(id)) { + throw new BadCommandArgumentsException("Missing mandatory argument " + + ARG_ID); + } + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder(SliderActions.ACTION_LOOKUP); + if (id!=null) { + sb.append(" "); + sb.append(ARG_ID).append(" ").append(id); + } + if (outputFile != null) { + sb.append(" "); + sb.append(ARG_OUTPUT).append(" ").append(outputFile.getAbsolutePath()); + } + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java index 8408385..b43a14e 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java @@ -21,6 +21,8 @@ package org.apache.slider.common.params; import com.beust.jcommander.Parameters; import com.beust.jcommander.ParametersDelegate; +import java.io.File; + @Parameters(commandNames = {SliderActions.ACTION_THAW}, commandDescription = SliderActions.DESCRIBE_ACTION_THAW) public class ActionThawArgs extends AbstractActionArgs implements @@ -50,4 +52,10 @@ public class ActionThawArgs extends AbstractActionArgs implements public void setWaittime(int waittime) { launchArgs.setWaittime(waittime); } + + + @Override + public File getOutputFile() { + return launchArgs.getOutputFile(); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java index 06d9dfb..42efb33 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java @@ -49,8 +49,9 @@ public interface Arguments { */ String ARG_FILESYSTEM = "--fs"; String ARG_FILESYSTEM_LONG = "--filesystem"; - String ARG_FORMAT = "--format"; + String ARG_FOLDER = "--folder"; String ARG_FORCE = "--force"; + String ARG_FORMAT = "--format"; String ARG_GETCONF = "--getconf"; String ARG_GETEXP = "--getexp"; String ARG_GETFILES = "--getfiles"; @@ -58,6 +59,7 @@ public interface Arguments { String ARG_ID = "--id"; String ARG_IMAGE = "--image"; String ARG_INTERNAL = "--internal"; + String ARG_KEYTAB = "--keytab"; String ARG_LEVEL = "--level"; String ARG_LIST = "--list"; String ARG_LISTCONF = "--listconf"; @@ -67,19 +69,17 @@ public interface Arguments { String ARG_MANAGER = "--manager"; String ARG_MANAGER_SHORT = "--m"; String ARG_MESSAGE = "--message"; + String ARG_NAME = "--name"; String ARG_OPTION = "--option"; String ARG_OPTION_SHORT = "-O"; - String ARG_NAME = "--name"; - String ARG_FOLDER = "--folder"; String ARG_OUTPUT = "--out"; String ARG_OUTPUT_SHORT = "-o"; + String ARG_OVERWRITE = "--overwrite"; String ARG_PACKAGE = "--package"; - String ARG_KEYTAB = "--keytab"; String ARG_PATH = "--path"; String ARG_PROVIDER = "--provider"; String ARG_QUEUE = "--queue"; String ARG_REPLACE_PKG = "--replacepkg"; - String ARG_OVERWRITE = "--overwrite"; String ARG_RESOURCES = "--resources"; String ARG_RES_COMP_OPT = "--rescompopt"; String ARG_RES_COMP_OPT_SHORT = "--rco"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java index 5a1c049..5ee6cd5 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java @@ -43,28 +43,31 @@ public class ClientArgs extends CommonArgs { * This is not bonded to jcommander, it is set up * after the construction to point to the relevant * entry + * + * KEEP IN ALPHABETICAL ORDER */ private AbstractClusterBuildingActionArgs buildingActionArgs; private final ActionAMSuicideArgs actionAMSuicideArgs = new ActionAMSuicideArgs(); private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs(); - private final ActionInstallPackageArgs actionInstallPackageArgs = new ActionInstallPackageArgs(); - private final ActionInstallKeytabArgs actionInstallKeytabArgs = new ActionInstallKeytabArgs(); - private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs(); private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs(); private final ActionDestroyArgs actionDestroyArgs = new ActionDestroyArgs(); + private final ActionDiagnosticArgs actionDiagnosticArgs = new ActionDiagnosticArgs(); private final ActionExistsArgs actionExistsArgs = new ActionExistsArgs(); private final ActionFlexArgs actionFlexArgs = new ActionFlexArgs(); private final ActionFreezeArgs actionFreezeArgs = new ActionFreezeArgs(); + private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs(); + private final ActionInstallPackageArgs actionInstallPackageArgs = new ActionInstallPackageArgs(); + private final ActionInstallKeytabArgs actionInstallKeytabArgs = new ActionInstallKeytabArgs(); private final ActionKillContainerArgs actionKillContainerArgs = new ActionKillContainerArgs(); private final ActionListArgs actionListArgs = new ActionListArgs(); + private final ActionLookupArgs actionLookupArgs = new ActionLookupArgs(); private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs(); private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs(); private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs(); private final ActionThawArgs actionThawArgs = new ActionThawArgs(); + private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs(); private final ActionVersionArgs actionVersionArgs = new ActionVersionArgs(); - private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs(); - private final ActionDiagnosticArgs actionDiagnosticArgs = new ActionDiagnosticArgs(); public ClientArgs(String[] args) { @@ -171,6 +174,10 @@ public class ClientArgs extends CommonArgs { return actionListArgs; } + public ActionLookupArgs getActionLookupArgs() { + return actionLookupArgs; + } + public ActionRegistryArgs getActionRegistryArgs() { return actionRegistryArgs; } @@ -242,6 +249,9 @@ public class ClientArgs extends CommonArgs { } else if (SliderActions.ACTION_LIST.equals(action)) { bindCoreAction(actionListArgs); + } else if (SliderActions.ACTION_LOOKUP.equals(action)) { + bindCoreAction(actionLookupArgs); + } else if (SliderActions.ACTION_REGISTRY.equals(action)) { bindCoreAction(actionRegistryArgs); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java b/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java index c36a968..7524053 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java @@ -18,9 +18,13 @@ package org.apache.slider.common.params; +import java.io.File; + /** * Launch args for create and start and anything else that can start something */ public interface LaunchArgsAccessor extends WaitTimeAccessor { String getRmAddress(); + + File getOutputFile(); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java index f0068e2..bc7e94c 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java @@ -20,6 +20,8 @@ package org.apache.slider.common.params; import com.beust.jcommander.Parameter; +import java.io.File; + /** * Any launch-time args */ @@ -37,4 +39,13 @@ public class LaunchArgsDelegate extends WaitArgsDelegate implements public String getRmAddress() { return rmAddress; } + + @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT}, + description = "output file for any application report") + public File outputFile; + + @Override + public File getOutputFile() { + return outputFile; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java index 0d084da..197c22b 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java @@ -22,6 +22,8 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import com.beust.jcommander.ParametersDelegate; +import java.io.File; + @Parameters(commandNames = {SliderActions.ACTION_CREATE}, commandDescription = SliderActions.DESCRIBE_ACTION_CREATE) @@ -63,5 +65,10 @@ public class SliderAMCreateAction extends AbstractActionArgs implements public void setWaittime(int waittime) { launchArgs.setWaittime(waittime); } + + @Override + public File getOutputFile() { + return launchArgs.getOutputFile(); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java index 87e4582..4b92a32 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java @@ -36,6 +36,7 @@ public interface SliderActions { String ACTION_HELP = "help"; String ACTION_KILL_CONTAINER = "kill-container"; String ACTION_LIST = "list"; + String ACTION_LOOKUP = "lookup"; String ACTION_PREFLIGHT = "preflight"; String ACTION_RECONFIGURE = "reconfigure"; String ACTION_REGISTRY = "registry"; @@ -68,6 +69,8 @@ public interface SliderActions { String DESCRIBE_ACTION_HELP = "Print help information"; String DESCRIBE_ACTION_LIST = "List running Slider applications"; + String DESCRIBE_ACTION_LOOKUP = + "look up a YARN application"; String DESCRIBE_ACTION_MONITOR = "Monitor a running application"; String DESCRIBE_ACTION_REGISTRY = http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java index 303f777..7023c80 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java @@ -46,18 +46,19 @@ public class AppMasterLauncher extends AbstractLauncher { private static final Logger log = LoggerFactory.getLogger(AppMasterLauncher.class); - protected final YarnClientApplication application; - private final String name; - private final String type; - private final ApplicationSubmissionContext submissionContext; - private final ApplicationId appId; - private final boolean secureCluster; + public final YarnClientApplication application; + public final String name; + public final String type; + public final ApplicationSubmissionContext submissionContext; + public final ApplicationId appId; + public final boolean secureCluster; private int maxAppAttempts = 0; private boolean keepContainersOverRestarts = true; private String queue = YarnConfiguration.DEFAULT_QUEUE_NAME; private int priority = 1; private final Resource resource = Records.newRecord(Resource.class); private final SliderYarnClientImpl yarnClient; + private Long submitTime; /** * Build the AM Launcher @@ -65,10 +66,15 @@ public class AppMasterLauncher extends AbstractLauncher { * @param type applicatin type * @param conf hadoop config * @param fs filesystem binding - * @param application precreated YARN client app instance + * @param yarnClient yarn client * @param secureCluster is the cluster secure? - * @param options map of options. All values are extracted in this constructor only * -the map is not retained. + * @param secureCluster flag to indicate secure cluster + * @param options map of options. All values are extracted in this constructor only + * @param resourceGlobalOptions global options + * @param applicationTags any app tags + * @throws IOException + * @throws YarnException */ public AppMasterLauncher(String name, String type, @@ -192,7 +198,6 @@ public class AppMasterLauncher extends AbstractLauncher { completeContainerLaunch(); submissionContext.setAMContainerSpec(containerLaunchContext); return submissionContext; - } /** @@ -216,13 +221,35 @@ public class AppMasterLauncher extends AbstractLauncher { fs.addDelegationTokens(tokenRenewer, credentials); } - + /** + * Submit the application. + * @return a launched application representing the submitted application + * @throws IOException + * @throws YarnException + */ public LaunchedApplication submitApplication() throws IOException, YarnException { completeAppMasterLaunch(); log.info("Submitting application to Resource Manager"); ApplicationId applicationId = yarnClient.submitApplication(submissionContext); + // implicit success; record the time + submitTime = System.currentTimeMillis(); return new LaunchedApplication(applicationId, yarnClient); } - + + /** + * Build a serializable application report. This is a very minimal + * report that contains the application Id, name and type âthe information + * available + * @return a data structure which can be persisted + */ + public SerializedApplicationReport createSerializedApplicationReport() { + SerializedApplicationReport sar = new SerializedApplicationReport(); + sar.applicationId = appId.toString(); + sar.name = name; + sar.applicationType = type; + sar.queue = queue; + sar.submitTime = submitTime; + return sar; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java b/slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java index e5a025c..632e3fd 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java @@ -95,6 +95,12 @@ public class LaunchedApplication { return yarnClient.killRunningApplication(applicationId, reason); } + /** + * Get the application report of this application + * @return an application report + * @throws YarnException + * @throws IOException + */ public ApplicationReport getApplicationReport() throws YarnException, IOException { return yarnClient.getApplicationReport(applicationId); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java b/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java new file mode 100644 index 0000000..e3d77d0 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.launch; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.slider.core.persist.ApplicationReportSerDeser; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.io.IOException; + +/** + * Serialized form of an application report which can be persisted + * and then parsed. It can not be converted back into a + * real YARN application report + * + * Useful for testing + */ + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) + +public class SerializedApplicationReport { + + public String applicationId; + public String applicationAttemptId; + public String name; + public String applicationType; + public String user; + public String queue; + public String host; + public Integer rpcPort; + public String state; + public String diagnostics; + public String url; + /** + * This value is non-null only when a report is generated from a submission context. + * The YARN {@link ApplicationReport} structure does not propagate this value + * from the RM. + */ + public Long submitTime; + public Long startTime; + public Long finishTime; + public String finalStatus; + public String origTrackingUrl; + public Float progress; + + public SerializedApplicationReport() { + } + + public SerializedApplicationReport(ApplicationReport report) { + this.applicationId = report.getApplicationId().toString(); + this.applicationAttemptId = report.getCurrentApplicationAttemptId().toString(); + this.name = report.getName(); + this.applicationType = report.getApplicationType(); + this.user = report.getUser(); + this.queue = report.getQueue(); + this.host = report.getHost(); + this.rpcPort = report.getRpcPort(); + this.state = report.getYarnApplicationState().toString(); + this.diagnostics = report.getDiagnostics(); + this.startTime = report.getStartTime(); + this.finishTime = report.getFinishTime(); + this.finalStatus = report.getFinalApplicationStatus().toString(); + this.progress = report.getProgress(); + } + + @Override + public String toString() { + try { + return ApplicationReportSerDeser.toString(this); + } catch (IOException e) { + return super.toString(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/core/persist/ApplicationReportSerDeser.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/ApplicationReportSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/ApplicationReportSerDeser.java new file mode 100644 index 0000000..a8c72ce --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/core/persist/ApplicationReportSerDeser.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.persist; + +import org.apache.slider.core.launch.SerializedApplicationReport; +import org.codehaus.jackson.JsonGenerationException; +import org.codehaus.jackson.JsonParseException; +import org.codehaus.jackson.map.JsonMappingException; + +import java.io.IOException; + +/** + * Persistence of {@link SerializedApplicationReport} + * + */ +public class ApplicationReportSerDeser + extends JsonSerDeser<SerializedApplicationReport> { + public ApplicationReportSerDeser() { + super(SerializedApplicationReport.class); + } + + + private static final ApplicationReportSerDeser + staticinstance = new ApplicationReportSerDeser(); + + /** + * Convert an instance to a JSON string -sync access to a shared ser/deser + * object instance + * @param instance object to convert + * @return a JSON string description + * @throws JsonParseException parse problems + * @throws JsonMappingException O/J mapping problems + */ + public static String toString(SerializedApplicationReport instance) + throws IOException, JsonGenerationException, JsonMappingException { + synchronized (staticinstance) { + return staticinstance.toJson(instance); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java index f77d6df..2c119ff 100644 --- a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java +++ b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java @@ -33,12 +33,13 @@ import org.codehaus.jackson.map.SerializationConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.DataOutputStream; import java.io.EOFException; import java.io.File; import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; /** * Support for marshalling objects to and from JSON. @@ -126,6 +127,7 @@ public class JsonSerDeser<T> { * @throws IOException IO problems * @throws JsonMappingException failure to map from the JSON to this class */ + @SuppressWarnings("IOResourceOpenedButNotSafelyClosed") public synchronized T fromResource(String resource) throws IOException, JsonParseException, JsonMappingException { InputStream resStream = null; @@ -206,9 +208,10 @@ public class JsonSerDeser<T> { /** - * Save a cluster description to a hadoop filesystem + * Save to a hadoop filesystem * @param fs filesystem * @param path path + * @param instance instance to save * @param overwrite should any existing file be overwritten * @throws IOException IO exception */ @@ -220,13 +223,23 @@ public class JsonSerDeser<T> { } /** + * Save an instance to a file + * @param instance instance to save + * @param file file + * @throws IOException + */ + public void save(T instance, File file) throws + IOException { + writeJsonAsBytes(instance, new FileOutputStream(file)); + } + + /** * Write the json as bytes -then close the file * @param dataOutputStream an outout stream that will always be closed * @throws IOException on any failure */ private void writeJsonAsBytes(T instance, - DataOutputStream dataOutputStream) throws - IOException { + OutputStream dataOutputStream) throws IOException { try { String json = toJson(instance); byte[] b = json.getBytes(UTF_8); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy index 7e7e52c..70d5ba5 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy @@ -24,11 +24,19 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic import org.apache.hadoop.fs.FileSystem as HadoopFS import org.apache.hadoop.fs.Path import org.apache.hadoop.yarn.api.records.FinalApplicationStatus +import org.apache.hadoop.yarn.api.records.YarnApplicationState import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.slider.agent.AgentMiniClusterTestBase import org.apache.slider.client.SliderClient +import org.apache.slider.common.params.ActionLookupArgs +import org.apache.slider.common.params.Arguments import org.apache.slider.common.tools.SliderUtils +import org.apache.slider.core.exceptions.BadCommandArgumentsException +import org.apache.slider.core.exceptions.NotFoundException +import org.apache.slider.core.exceptions.SliderException +import org.apache.slider.core.main.ServiceLaunchException import org.apache.slider.core.main.ServiceLauncher +import org.apache.slider.core.persist.ApplicationReportSerDeser import org.junit.Test /** @@ -80,12 +88,35 @@ class TestFreezeThawFlexStandaloneAM extends AgentMiniClusterTestBase { localFS.delete(tempConfPath,true) //now start the cluster - ServiceLauncher launcher2 = thawCluster(clustername, [], true); + File appreport = new File("target/$clustername/appreport.json") + ServiceLauncher launcher2 = thawCluster(clustername, + [Arguments.ARG_OUTPUT, appreport.absolutePath], + true); + SliderClient newCluster = launcher2.service addToTeardown(newCluster); + ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser(); + def sar = serDeser.fromFile(appreport) + log.info(sar.toString()) + assert sar.applicationId != null + + describe("lookup") + + // now via lookup + appreport.delete() + def lookup1 = new ActionLookupArgs() + lookup1.id = sar.applicationId + + assert 0 == newCluster.actionLookup(lookup1) + lookup1.outputFile = appreport + assert 0 == newCluster.actionLookup(lookup1) + sar = serDeser.fromFile(appreport) + assert sar.state == YarnApplicationState.RUNNING.toString() + newCluster.getClusterDescription(clustername); + describe("no change flex") // while running, flex it with no changes newCluster.flex(clustername, [:]); @@ -95,9 +126,28 @@ class TestFreezeThawFlexStandaloneAM extends AgentMiniClusterTestBase { report = newCluster.applicationReport assert report.finalApplicationStatus == FinalApplicationStatus.KILLED + assert 0 == newCluster.actionLookup(lookup1) + sar = serDeser.fromFile(appreport) + assert sar.finalStatus == FinalApplicationStatus.KILLED.toString() + //stop again assert 0 == clusterActionFreeze(newCluster, clustername) + // and add some invalid lookup operations for + + def lookup2 = new ActionLookupArgs() + lookup2.id = "invalid" + try { + newCluster.actionLookup(lookup2) + fail("expected $lookup2 to fail") + } catch (BadCommandArgumentsException expected) { + } + try { + lookup2.id = "application_1414593568640_0002" + newCluster.actionLookup(lookup2) + fail("expected $lookup2 to fail") + } catch (NotFoundException expected) { + } } }