YARN-5735. Make the service REST API use the app timeout feature YARN-4205. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be695958 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be695958 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be695958 Branch: refs/heads/yarn-native-services Commit: be695958b8216557f9aa46ed410ce9320cc094bf Parents: b279e66 Author: Gour Saha <gourks...@apache.org> Authored: Fri Oct 14 17:40:51 2016 -0700 Committer: Jian He <jia...@apache.org> Committed: Wed Dec 7 13:00:06 2016 -0800 ---------------------------------------------------------------------- .../api/impl/ApplicationApiService.java | 10 ++++-- .../org/apache/slider/client/SliderClient.java | 33 ++++++++++---------- .../AbstractClusterBuildingActionArgs.java | 5 +++ .../slider/common/params/ActionThawArgs.java | 6 ++++ .../apache/slider/common/params/Arguments.java | 1 + 5 files changed, 36 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/be695958/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java index 21cf113..73df4a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java @@ -347,7 +347,7 @@ public class ApplicationApiService implements ApplicationApi { if (queueName != null && queueName.trim().length() > 0) { createArgs.queue = queueName.trim(); } - + createArgs.lifetime = application.getLifetime(); return invokeSliderClientRunnable(new SliderClientContextRunnable<String>() { @Override public String run(SliderClient sliderClient) throws YarnException, @@ -1246,13 +1246,17 @@ public class ApplicationApiService implements ApplicationApi { }); } - private Response startSliderApplication(final String appName) + private Response startSliderApplication(final String appName, Application app) throws IOException, YarnException, InterruptedException { return invokeSliderClientRunnable(new SliderClientContextRunnable<Response>() { @Override public Response run(SliderClient sliderClient) throws YarnException, IOException, InterruptedException { ActionThawArgs thawArgs = new ActionThawArgs(); + if (app.getLifetime() == null) { + app.setLifetime(DEFAULT_UNLIMITED_LIFETIME); + } + thawArgs.lifetime = app.getLifetime(); int returnCode = sliderClient.actionThaw(appName, thawArgs); if (returnCode == 0) { logger.info("Successfully started application {}", appName); @@ -1344,7 +1348,7 @@ public class ApplicationApiService implements ApplicationApi { try { int livenessCheck = getSliderList(appName); if (livenessCheck != 0) { - return startSliderApplication(appName); + return startSliderApplication(appName, updateAppData); } else { logger.info("Application {} is already running", appName); ApplicationStatus applicationStatus = new ApplicationStatus(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/be695958/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index fe4f1d2..2840c4b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -54,6 +54,7 @@ import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.NodeReport; @@ -734,7 +735,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe sliderFileSystem.getFileSystem().delete(clusterDirectory, true); throw e; } - return startCluster(clustername, createArgs); + return startCluster(clustername, createArgs, createArgs.lifetime); } @Override @@ -1960,14 +1961,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * * @param clustername name of the cluster. * @param launchArgs launch arguments + * @param lifetime * @return the exit code * @throws YarnException * @throws IOException */ - protected int startCluster(String clustername, - LaunchArgsAccessor launchArgs) throws - YarnException, - IOException { + protected int startCluster(String clustername, LaunchArgsAccessor launchArgs, + long lifetime) throws YarnException, IOException { Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername); AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved( clustername, @@ -1975,7 +1975,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe LaunchedApplication launchedApplication = launchApplication(clustername, clusterDirectory, instanceDefinition, - serviceArgs.isDebug()); + serviceArgs.isDebug(), lifetime); if (launchArgs.getOutputFile() != null) { // output file has been requested. Get the app report and serialize it @@ -2044,9 +2044,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } protected AppMasterLauncher setupAppMasterLauncher(String clustername, - Path clusterDirectory, - AggregateConf instanceDefinition, - boolean debugAM) + Path clusterDirectory, AggregateConf instanceDefinition, boolean debugAM, + long lifetime) throws YarnException, IOException{ deployedClusterName = clustername; validateClusterName(clustername); @@ -2119,7 +2118,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe ApplicationId appId = amLauncher.getApplicationId(); // set the application name; amLauncher.setKeepContainersOverRestarts(true); - + // set lifetime in submission context; + Map<ApplicationTimeoutType, Long> appTimeout = new HashMap<>(); + appTimeout.put(ApplicationTimeoutType.LIFETIME, lifetime); + amLauncher.submissionContext.setApplicationTimeouts(appTimeout); int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0); amLauncher.setMaxAppAttempts(maxAppAttempts); @@ -2383,20 +2385,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @param clusterDirectory cluster dir * @param instanceDefinition the instance definition * @param debugAM enable debug AM options + * @param lifetime * @return the launched application * @throws YarnException * @throws IOException */ - public LaunchedApplication launchApplication(String clustername, - Path clusterDirectory, - AggregateConf instanceDefinition, - boolean debugAM) + public LaunchedApplication launchApplication(String clustername, Path clusterDirectory, + AggregateConf instanceDefinition, boolean debugAM, long lifetime) throws YarnException, IOException { AppMasterLauncher amLauncher = setupAppMasterLauncher(clustername, clusterDirectory, instanceDefinition, - debugAM); + debugAM, lifetime); applicationId = amLauncher.getApplicationId(); log.info("Submitting application {}", applicationId); @@ -3254,7 +3255,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe verifyNoLiveClusters(clustername, "Start"); //start the cluster - return startCluster(clustername, thaw); + return startCluster(clustername, thaw, thaw.lifetime); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/be695958/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java index 2a5eedc..3cb75e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java @@ -102,6 +102,11 @@ public abstract class AbstractClusterBuildingActionArgs extends description = "Queue to submit the application") public String queue; + @Parameter(names = {ARG_LIFETIME}, + description = "Life time of the application since application started at" + + " running state") + public long lifetime; + @ParametersDelegate public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/be695958/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java index b43a14e..2bd856f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java @@ -18,6 +18,7 @@ package org.apache.slider.common.params; +import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import com.beust.jcommander.ParametersDelegate; @@ -43,6 +44,11 @@ public class ActionThawArgs extends AbstractActionArgs implements @ParametersDelegate LaunchArgsDelegate launchArgs = new LaunchArgsDelegate(); + @Parameter(names = {ARG_LIFETIME}, + description = "Life time of the application since application started at" + + " running state") + public long lifetime; + @Override public String getRmAddress() { return launchArgs.getRmAddress(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/be695958/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java index aec4e26..cbf7e59 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java @@ -103,6 +103,7 @@ public interface Arguments { String ARG_PRINCIPAL = "--principal"; String ARG_PROVIDER = "--provider"; String ARG_QUEUE = "--queue"; + String ARG_LIFETIME = "--lifetime"; String ARG_REPLACE_PKG = "--replacepkg"; String ARG_RESOURCE = "--resource"; String ARG_RESOURCES = "--resources"; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org