Repository: hadoop Updated Branches: refs/heads/yarn-native-services be467574d -> a54f79992
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a54f7999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java index f22d487..df4b1df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java @@ -20,88 +20,71 @@ package org.apache.hadoop.yarn.service.client; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.client.cli.ApplicationCLI; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.service.ClientAMProtocol; import org.apache.hadoop.yarn.service.api.records.Component; -import org.apache.hadoop.yarn.service.client.params.ClientArgs; +import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.conf.ExampleAppJson; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; -import org.apache.hadoop.yarn.util.Records; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.List; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS; -import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE; import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH; -import static org.mockito.Mockito.*; public class TestServiceCLI { + private static final Logger LOG = LoggerFactory.getLogger(TestServiceCLI + .class); - protected Configuration conf = new YarnConfiguration(); + private Configuration conf = new YarnConfiguration(); private File basedir; - private ServiceCLI cli; private SliderFileSystem fs; + private String basedirProp; - private void buildApp(String appDef) throws Throwable { - String[] args = - { "build", ARG_FILE, ExampleAppJson.resourceName(appDef)}; - ClientArgs clientArgs = new ClientArgs(args); - clientArgs.parse(); - cli.exec(clientArgs); + private void runCLI(String[] args) throws Exception { + LOG.info("running CLI: yarn {}", Arrays.asList(args)); + ApplicationCLI cli = new ApplicationCLI(); + cli.setSysOutPrintStream(System.out); + cli.setSysErrPrintStream(System.err); + int res = ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args)); + cli.stop(); + } + + private void buildApp(String serviceName, String appDef) throws Throwable { + String[] args = {"app", "-D", basedirProp, "-save", serviceName, + ExampleAppJson.resourceName(appDef)}; + runCLI(args); + } + + private void buildApp(String serviceName, String appDef, String lifetime, + String queue) throws Throwable { + String[] args = {"app", "-D", basedirProp, "-save", serviceName, + ExampleAppJson.resourceName(appDef), "-updateLifetime", lifetime, + "-changeQueue", queue}; + runCLI(args); } @Before public void setup() throws Throwable { basedir = new File("target", "apps"); + basedirProp = YARN_SERVICE_BASE_PATH + "=" + basedir.getAbsolutePath(); conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath()); - conf.setLong(RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, 0); - conf.setLong(RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1); - conf.setInt( - CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); - conf.setInt(CommonConfigurationKeysPublic. - IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0); fs = new SliderFileSystem(conf); if (basedir.exists()) { FileUtils.deleteDirectory(basedir); } else { basedir.mkdirs(); } - - // create a CLI and skip connection to AM - cli = new ServiceCLI() { - @Override protected void createServiceClient() { - client = new ServiceClient() { - @Override - protected void serviceInit(Configuration configuration) - throws Exception { - super.serviceInit(conf); - yarnClient = spy(yarnClient); - ApplicationReport report = Records.newRecord(ApplicationReport.class); - report.setYarnApplicationState(YarnApplicationState.RUNNING); - report.setHost("localhost"); - doReturn(report).when(yarnClient).getApplicationReport(anyObject()); - } - @Override - protected ClientAMProtocol createAMProxy(String host, int port) - throws IOException { - return mock(ClientAMProtocol.class); - } - }; - client.init(conf); - client.start(); - } - }; } @After @@ -111,41 +94,31 @@ public class TestServiceCLI { } } - // Test flex components count are persisted. @Test public void testFlexComponents() throws Throwable { + // currently can only test building apps, since that is the only + // operation that doesn't require an RM + // TODO: expand CLI test to try other commands String serviceName = "app-1"; - buildApp(ExampleAppJson.APP_JSON); - checkCompCount("master",serviceName, 1L); - - // increase by 2 - String[] flexUpArgs = {"flex", serviceName, "--component", "master" , "+2"}; - ClientArgs clientArgs = new ClientArgs(flexUpArgs); - clientArgs.parse(); - cli.exec(clientArgs); - checkCompCount("master", serviceName, 3L); - - // decrease by 1 - String[] flexDownArgs = {"flex", serviceName, "--component", "master", "-1"}; - clientArgs = new ClientArgs(flexDownArgs); - clientArgs.parse(); - cli.exec(clientArgs); - checkCompCount("master", serviceName, 2L); + buildApp(serviceName, ExampleAppJson.APP_JSON); + checkApp(serviceName, "master", 1L, 3600L, null); - String[] flexAbsoluteArgs = {"flex", serviceName, "--component", "master", "10"}; - clientArgs = new ClientArgs(flexAbsoluteArgs); - clientArgs.parse(); - cli.exec(clientArgs); - checkCompCount("master", serviceName, 10L); + serviceName = "app-2"; + buildApp(serviceName, ExampleAppJson.APP_JSON, "1000", "qname"); + checkApp(serviceName, "master", 1L, 1000L, "qname"); } - private void checkCompCount(String compName, String serviceName, long count) - throws IOException { - List<Component> components = - ServiceApiUtil.getComponents(fs, serviceName); + private void checkApp(String serviceName, String compName, long count, Long + lifetime, String queue) throws IOException { + Service service = ServiceApiUtil.loadService(fs, serviceName); + Assert.assertEquals(serviceName, service.getName()); + Assert.assertEquals(lifetime, service.getLifetime()); + Assert.assertEquals(queue, service.getQueue()); + List<Component> components = service.getComponents(); for (Component component : components) { if (component.getName().equals(compName)) { - Assert.assertEquals(count, component.getNumberOfContainers().longValue()); + Assert.assertEquals(count, component.getNumberOfContainers() + .longValue()); return; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a54f7999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java new file mode 100644 index 0000000..6310178 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java @@ -0,0 +1,222 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.client.api; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; + +import java.io.IOException; +import java.util.Map; + +/** + * Client for managing applications. + */ +@Public +@Unstable +public abstract class AppAdminClient extends CompositeService { + public static final String YARN_APP_ADMIN_CLIENT_PREFIX = "yarn" + + ".application.admin.client.class."; + public static final String DEFAULT_TYPE = "yarn-service"; + public static final String DEFAULT_CLASS_NAME = "org.apache.hadoop.yarn" + + ".service.client.ServiceClient"; + + @Private + protected AppAdminClient() { + super(AppAdminClient.class.getName()); + } + + /** + * <p> + * Create a new instance of AppAdminClient. + * </p> + * + * @param appType application type + * @param conf configuration + * @return app admin client + */ + @Public + @Unstable + public static AppAdminClient createAppAdminClient(String appType, + Configuration conf) { + Map<String, String> clientClassMap = + conf.getPropsWithPrefix(YARN_APP_ADMIN_CLIENT_PREFIX); + if (!clientClassMap.containsKey(DEFAULT_TYPE)) { + clientClassMap.put(DEFAULT_TYPE, DEFAULT_CLASS_NAME); + } + if (!clientClassMap.containsKey(appType)) { + throw new IllegalArgumentException("App admin client class name not " + + "specified for type " + appType); + } + String clientClassName = clientClassMap.get(appType); + Class<? extends AppAdminClient> clientClass; + try { + clientClass = (Class<? extends AppAdminClient>) Class.forName( + clientClassName); + } catch (ClassNotFoundException e) { + throw new YarnRuntimeException("Invalid app admin client class", e); + } + + AppAdminClient appAdminClient = ReflectionUtils.newInstance(clientClass, + conf); + appAdminClient.init(conf); + appAdminClient.start(); + return appAdminClient; + } + + /** + * <p> + * Launch a new YARN application. + * </p> + * + * @param fileName specification of application + * @param appName name of the application + * @param lifetime lifetime of the application + * @param queue queue of the application + * @return exit code + * @throws IOException IOException + * @throws YarnException exception in client or server + */ + @Public + @Unstable + public abstract int actionLaunch(String fileName, String appName, Long + lifetime, String queue) throws IOException, YarnException; + + /** + * <p> + * Stop a YARN application (attempt to stop gracefully before killing the + * application). In the case of a long-running service, the service may be + * restarted later. + * </p> + * + * @param appName the name of the application + * @return exit code + * @throws IOException IOException + * @throws YarnException exception in client or server + */ + @Public + @Unstable + public abstract int actionStop(String appName) throws IOException, + YarnException; + + /** + * <p> + * Start a YARN application from a previously saved specification. In the + * case of a long-running service, the service must have been previously + * launched/started and then stopped, or previously saved but not started. + * </p> + * + * @param appName the name of the application + * @return exit code + * @throws IOException IOException + * @throws YarnException exception in client or server + */ + @Public + @Unstable + public abstract int actionStart(String appName) throws IOException, + YarnException; + + /** + * <p> + * Save the specification for a YARN application / long-running service. + * The application may be started later. + * </p> + * + * @param fileName specification of application to save + * @param appName name of the application + * @param lifetime lifetime of the application + * @param queue queue of the application + * @return exit code + * @throws IOException IOException + * @throws YarnException exception in client or server + */ + @Public + @Unstable + public abstract int actionSave(String fileName, String appName, Long + lifetime, String queue) throws IOException, YarnException; + + /** + * <p> + * Remove the specification and all application data for a YARN application. + * The application cannot be running. + * </p> + * + * @param appName the name of the application + * @return exit code + * @throws IOException IOException + * @throws YarnException exception in client or server + */ + @Public + @Unstable + public abstract int actionDestroy(String appName) throws IOException, + YarnException; + + /** + * <p> + * Change the number of running containers for a component of a YARN + * application / long-running service. + * </p> + * + * @param appName the name of the application + * @param componentCounts map of component name to new component count or + * amount to change existing component count (e.g. + * 5, +5, -5) + * @return exit code + * @throws IOException IOException + * @throws YarnException exception in client or server + */ + @Public + @Unstable + public abstract int actionFlex(String appName, Map<String, String> + componentCounts) throws IOException, YarnException; + + /** + * <p> + * Upload AM dependencies to HDFS. This makes future application launches + * faster since the dependencies do not have to be uploaded on each launch. + * </p> + * + * @return exit code + * @throws IOException IOException + * @throws YarnException exception in client or server + */ + @Public + @Unstable + public abstract int enableFastLaunch() throws IOException, + YarnException; + + /** + * <p> + * Get detailed status string for a YARN application. + * </p> + * + * @param applicationId application id + * @return status string + * @throws IOException IOException + * @throws YarnException exception in client or server + */ + @Public + @Unstable + public abstract String getStatusString(String applicationId) throws + IOException, YarnException; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a54f7999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 2a9b3bc..fb08fcd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -23,12 +23,7 @@ import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.nio.charset.Charset; import java.text.DecimalFormat; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumSet; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; @@ -54,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.client.api.AppAdminClient; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; @@ -85,6 +81,7 @@ public class ApplicationCLI extends YarnCLI { "%30s\t%20s\t%20s\t%20s\t%20s\t%20s\t%35s" + System.getProperty("line.separator"); + public static final String APP = "app"; public static final String APPLICATION = "application"; public static final String APPLICATION_ATTEMPT = "applicationattempt"; public static final String CONTAINER = "container"; @@ -93,22 +90,52 @@ public class ApplicationCLI extends YarnCLI { public static final String UPDATE_LIFETIME = "updateLifetime"; public static final String CHANGE_APPLICATION_QUEUE = "changeQueue"; + // app admin options + public static final String LAUNCH_CMD = "launch"; + public static final String STOP_CMD = "stop"; + public static final String START_CMD = "start"; + public static final String SAVE_CMD = "save"; + public static final String DESTROY_CMD = "destroy"; + public static final String FLEX_CMD = "flex"; + public static final String COMPONENT = "component"; + public static final String ENABLE_FAST_LAUNCH = "enableFastLaunch"; + + private static String firstArg = null; + private boolean allAppStates; public static void main(String[] args) throws Exception { ApplicationCLI cli = new ApplicationCLI(); cli.setSysOutPrintStream(System.out); cli.setSysErrPrintStream(System.err); - int res = ToolRunner.run(cli, args); + int res = ToolRunner.run(cli, preProcessArgs(args)); cli.stop(); System.exit(res); } + @VisibleForTesting + public static String[] preProcessArgs(String[] args) { + if (args.length > 0) { + // first argument (app|application|applicationattempt|container) must + // be stripped off for GenericOptionsParser to work + firstArg = args[0]; + return Arrays.copyOfRange(args, 1, args.length); + } else { + return args; + } + } + @Override public int run(String[] args) throws Exception { Options opts = new Options(); String title = null; - if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATION)) { + if (firstArg != null) { + title = firstArg; + } else if (args.length > 0) { + title = args[0]; + } + if (title != null && (title.equalsIgnoreCase(APPLICATION) || title + .equalsIgnoreCase(APP))) { title = APPLICATION; opts.addOption(STATUS_CMD, true, "Prints the status of the application."); @@ -168,8 +195,52 @@ public class ApplicationCLI extends YarnCLI { opts.getOption(UPDATE_PRIORITY).setArgName("Priority"); opts.getOption(UPDATE_LIFETIME).setArgName("Timeout"); opts.getOption(CHANGE_APPLICATION_QUEUE).setArgName("Queue Name"); - } else if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) { - title = APPLICATION_ATTEMPT; + opts.addOption(LAUNCH_CMD, true, "Launches application from " + + "specification file (saves specification and starts application). " + + "Options -updateLifetime and -changeQueue can be specified to alter" + + " the values provided in the file. Supports -appTypes option to " + + "specify which client implementation to use."); + opts.addOption(STOP_CMD, true, "Stops application gracefully (may be " + + "started again later). If name is provided, appType must be " + + "provided unless it is the default yarn-service. If ID is provided," + + " the appType will be looked up. Supports -appTypes option to " + + "specify which client implementation to use."); + opts.addOption(START_CMD, true, "Starts a previously saved " + + "application. Supports -appTypes option to specify which client " + + "implementation to use."); + opts.addOption(SAVE_CMD, true, "Saves specification file for " + + "an application. Options -updateLifetime and -changeQueue can be " + + "specified to alter the values provided in the file. Supports " + + "-appTypes option to specify which client implementation to use."); + opts.addOption(DESTROY_CMD, true, "Destroys a saved application " + + "specification and removes all application data permanently. " + + "Supports -appTypes option to specify which client implementation " + + "to use."); + opts.addOption(FLEX_CMD, true, "Changes number of " + + "running containers for a component of an application / " + + "long-running service. Requires -component option. If name is " + + "provided, appType must be provided unless it is the default " + + "yarn-service. If ID is provided, the appType will be looked up. " + + "Supports -appTypes option to specify which client implementation " + + "to use."); + opts.addOption(COMPONENT, true, "Works with -flex option to change " + + "the number of components/containers running for an application / " + + "long-running service. Supports absolute or relative changes, such " + + "as +1, 2, or -3."); + opts.addOption(ENABLE_FAST_LAUNCH, false, "Uploads AM dependencies " + + "to HDFS to make future launches faster. Supports -appTypes option" + + " to specify which client implementation to use."); + opts.getOption(LAUNCH_CMD).setArgName("Application Name> <File Name"); + opts.getOption(LAUNCH_CMD).setArgs(2); + opts.getOption(START_CMD).setArgName("Application Name"); + opts.getOption(STOP_CMD).setArgName("Application Name or ID"); + opts.getOption(SAVE_CMD).setArgName("Application Name> <File Name"); + opts.getOption(SAVE_CMD).setArgs(2); + opts.getOption(DESTROY_CMD).setArgName("Application Name"); + opts.getOption(FLEX_CMD).setArgName("Application Name or ID"); + opts.getOption(COMPONENT).setArgName("Component Name> <Count"); + opts.getOption(COMPONENT).setArgs(2); + } else if (title != null && title.equalsIgnoreCase(APPLICATION_ATTEMPT)) { opts.addOption(STATUS_CMD, true, "Prints the status of the application attempt."); opts.addOption(LIST_CMD, true, @@ -179,8 +250,7 @@ public class ApplicationCLI extends YarnCLI { opts.getOption(STATUS_CMD).setArgName("Application Attempt ID"); opts.getOption(LIST_CMD).setArgName("Application ID"); opts.getOption(FAIL_CMD).setArgName("Application Attempt ID"); - } else if (args.length > 0 && args[0].equalsIgnoreCase(CONTAINER)) { - title = CONTAINER; + } else if (title != null && title.equalsIgnoreCase(CONTAINER)) { opts.addOption(STATUS_CMD, true, "Prints the status of the container."); opts.addOption(LIST_CMD, true, @@ -205,23 +275,53 @@ public class ApplicationCLI extends YarnCLI { printUsage(title, opts); return exitCode; } + String[] unparsedArgs = cliParser.getArgs(); + if (firstArg == null) { + if (unparsedArgs.length != 1) { + printUsage(title, opts); + return exitCode; + } + } else { + if (unparsedArgs.length != 0) { + printUsage(title, opts); + return exitCode; + } + } if (cliParser.hasOption(STATUS_CMD)) { - if (args.length != 3) { + if (hasAnyOtherCLIOptions(cliParser, opts, STATUS_CMD)) { printUsage(title, opts); return exitCode; } - if (args[0].equalsIgnoreCase(APPLICATION)) { - exitCode = printApplicationReport(cliParser.getOptionValue(STATUS_CMD)); - } else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) { + if (title.equalsIgnoreCase(APPLICATION) || + title.equalsIgnoreCase(APP)) { + ApplicationReport report = printApplicationReport(cliParser + .getOptionValue(STATUS_CMD)); + if (report == null) { + exitCode = -1; + } else { + exitCode = 0; + String appType = report.getApplicationType(); + try { + AppAdminClient client = AppAdminClient.createAppAdminClient(appType, + getConf()); + sysout.println("Detailed Application Status :"); + sysout.println(client.getStatusString(cliParser.getOptionValue( + STATUS_CMD))); + } catch (IllegalArgumentException e) { + // app type does not have app admin client implementation + } + } + } else if (title.equalsIgnoreCase(APPLICATION_ATTEMPT)) { exitCode = printApplicationAttemptReport(cliParser .getOptionValue(STATUS_CMD)); - } else if (args[0].equalsIgnoreCase(CONTAINER)) { + } else if (title.equalsIgnoreCase(CONTAINER)) { exitCode = printContainerReport(cliParser.getOptionValue(STATUS_CMD)); } return exitCode; } else if (cliParser.hasOption(LIST_CMD)) { - if (args[0].equalsIgnoreCase(APPLICATION)) { + if (title.equalsIgnoreCase(APPLICATION) || + title.equalsIgnoreCase(APP)) { allAppStates = false; Set<String> appTypes = new HashSet<String>(); if (cliParser.hasOption(APP_TYPE_CMD)) { @@ -272,21 +372,21 @@ public class ApplicationCLI extends YarnCLI { } } listApplications(appTypes, appStates, appTags); - } else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) { - if (args.length != 3) { + } else if (title.equalsIgnoreCase(APPLICATION_ATTEMPT)) { + if (hasAnyOtherCLIOptions(cliParser, opts, LIST_CMD)) { printUsage(title, opts); return exitCode; } listApplicationAttempts(cliParser.getOptionValue(LIST_CMD)); - } else if (args[0].equalsIgnoreCase(CONTAINER)) { - if (args.length != 3) { + } else if (title.equalsIgnoreCase(CONTAINER)) { + if (hasAnyOtherCLIOptions(cliParser, opts, LIST_CMD)) { printUsage(title, opts); return exitCode; } listContainers(cliParser.getOptionValue(LIST_CMD)); } } else if (cliParser.hasOption(KILL_CMD)) { - if (args.length < 3 || hasAnyOtherCLIOptions(cliParser, opts, KILL_CMD)) { + if (hasAnyOtherCLIOptions(cliParser, opts, KILL_CMD)) { printUsage(title, opts); return exitCode; } @@ -299,7 +399,7 @@ public class ApplicationCLI extends YarnCLI { moveApplicationAcrossQueues(cliParser.getOptionValue(MOVE_TO_QUEUE_CMD), cliParser.getOptionValue(QUEUE_CMD)); } else if (cliParser.hasOption(FAIL_CMD)) { - if (!args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) { + if (!title.equalsIgnoreCase(APPLICATION_ATTEMPT)) { printUsage(title, opts); return exitCode; } @@ -314,6 +414,103 @@ public class ApplicationCLI extends YarnCLI { } updateApplicationPriority(cliParser.getOptionValue(APP_ID), cliParser.getOptionValue(UPDATE_PRIORITY)); + } else if (cliParser.hasOption(SIGNAL_CMD)) { + if (hasAnyOtherCLIOptions(cliParser, opts, SIGNAL_CMD)) { + printUsage(title, opts); + return exitCode; + } + final String[] signalArgs = cliParser.getOptionValues(SIGNAL_CMD); + final String containerId = signalArgs[0]; + SignalContainerCommand command = + SignalContainerCommand.OUTPUT_THREAD_DUMP; + if (signalArgs.length == 2) { + command = SignalContainerCommand.valueOf(signalArgs[1]); + } + signalToContainer(containerId, command); + } else if (cliParser.hasOption(LAUNCH_CMD)) { + if (hasAnyOtherCLIOptions(cliParser, opts, LAUNCH_CMD, APP_TYPE_CMD, + UPDATE_LIFETIME, CHANGE_APPLICATION_QUEUE)) { + printUsage(title, opts); + return exitCode; + } + String appType = getSingleAppTypeFromCLI(cliParser); + Long lifetime = null; + if (cliParser.hasOption(UPDATE_LIFETIME)) { + lifetime = Long.parseLong(cliParser.getOptionValue(UPDATE_LIFETIME)); + } + String queue = null; + if (cliParser.hasOption(CHANGE_APPLICATION_QUEUE)) { + queue = cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE); + } + String[] nameAndFile = cliParser.getOptionValues(LAUNCH_CMD); + return AppAdminClient.createAppAdminClient(appType, getConf()) + .actionLaunch(nameAndFile[1], nameAndFile[0], lifetime, queue); + } else if (cliParser.hasOption(STOP_CMD)) { + if (hasAnyOtherCLIOptions(cliParser, opts, STOP_CMD, APP_TYPE_CMD)) { + printUsage(title, opts); + return exitCode; + } + String[] appNameAndType = getAppNameAndType(cliParser, STOP_CMD); + return AppAdminClient.createAppAdminClient(appNameAndType[1], getConf()) + .actionStop(appNameAndType[0]); + } else if (cliParser.hasOption(START_CMD)) { + if (hasAnyOtherCLIOptions(cliParser, opts, START_CMD, APP_TYPE_CMD)) { + printUsage(title, opts); + return exitCode; + } + String appType = getSingleAppTypeFromCLI(cliParser); + return AppAdminClient.createAppAdminClient(appType, getConf()) + .actionStart(cliParser.getOptionValue(START_CMD)); + } else if (cliParser.hasOption(SAVE_CMD)) { + if (hasAnyOtherCLIOptions(cliParser, opts, SAVE_CMD, APP_TYPE_CMD, + UPDATE_LIFETIME, CHANGE_APPLICATION_QUEUE)) { + printUsage(title, opts); + return exitCode; + } + String appType = getSingleAppTypeFromCLI(cliParser); + Long lifetime = null; + if (cliParser.hasOption(UPDATE_LIFETIME)) { + lifetime = Long.parseLong(cliParser.getOptionValue(UPDATE_LIFETIME)); + } + String queue = null; + if (cliParser.hasOption(CHANGE_APPLICATION_QUEUE)) { + queue = cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE); + } + String[] nameAndFile = cliParser.getOptionValues(SAVE_CMD); + return AppAdminClient.createAppAdminClient(appType, getConf()) + .actionSave(nameAndFile[1], nameAndFile[0], lifetime, queue); + } else if (cliParser.hasOption(DESTROY_CMD)) { + if (hasAnyOtherCLIOptions(cliParser, opts, DESTROY_CMD, APP_TYPE_CMD)) { + printUsage(title, opts); + return exitCode; + } + String appType = getSingleAppTypeFromCLI(cliParser); + return AppAdminClient.createAppAdminClient(appType, getConf()) + .actionDestroy(cliParser.getOptionValue(DESTROY_CMD)); + } else if (cliParser.hasOption(FLEX_CMD)) { + if (!cliParser.hasOption(COMPONENT) || + hasAnyOtherCLIOptions(cliParser, opts, FLEX_CMD, COMPONENT, + APP_TYPE_CMD)) { + printUsage(title, opts); + return exitCode; + } + String[] rawCounts = cliParser.getOptionValues(COMPONENT); + Map<String, String> counts = new HashMap<>(rawCounts.length/2); + for (int i = 0; i < rawCounts.length - 1; i+=2) { + counts.put(rawCounts[i], rawCounts[i+1]); + } + String[] appNameAndType = getAppNameAndType(cliParser, FLEX_CMD); + return AppAdminClient.createAppAdminClient(appNameAndType[1], getConf()) + .actionFlex(appNameAndType[0], counts); + } else if (cliParser.hasOption(ENABLE_FAST_LAUNCH)) { + String appType = getSingleAppTypeFromCLI(cliParser); + if (hasAnyOtherCLIOptions(cliParser, opts, ENABLE_FAST_LAUNCH, + APP_TYPE_CMD)) { + printUsage(title, opts); + return exitCode; + } + return AppAdminClient.createAppAdminClient(appType, getConf()) + .enableFastLaunch(); } else if (cliParser.hasOption(UPDATE_LIFETIME)) { if (!cliParser.hasOption(APP_ID)) { printUsage(title, opts); @@ -332,19 +529,6 @@ public class ApplicationCLI extends YarnCLI { } moveApplicationAcrossQueues(cliParser.getOptionValue(APP_ID), cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE)); - } else if (cliParser.hasOption(SIGNAL_CMD)) { - if (args.length < 3 || args.length > 4) { - printUsage(title, opts); - return exitCode; - } - final String[] signalArgs = cliParser.getOptionValues(SIGNAL_CMD); - final String containerId = signalArgs[0]; - SignalContainerCommand command = - SignalContainerCommand.OUTPUT_THREAD_DUMP; - if (signalArgs.length == 2) { - command = SignalContainerCommand.valueOf(signalArgs[1]); - } - signalToContainer(containerId, command); } else { syserr.println("Invalid Command Usage : "); printUsage(title, opts); @@ -352,6 +536,47 @@ public class ApplicationCLI extends YarnCLI { return 0; } + private ApplicationReport getApplicationReport(ApplicationId applicationId) + throws IOException, YarnException { + ApplicationReport appReport = null; + try { + appReport = client.getApplicationReport(applicationId); + } catch (ApplicationNotFoundException e) { + throw new YarnException("Application with id '" + applicationId + + "' doesn't exist in RM or Timeline Server."); + } + return appReport; + } + + private String[] getAppNameAndType(CommandLine cliParser, String option) + throws IOException, YarnException { + String applicationIdOrName = cliParser.getOptionValue(option); + try { + ApplicationId id = ApplicationId.fromString(applicationIdOrName); + ApplicationReport report = getApplicationReport(id); + return new String[]{report.getName(), report.getApplicationType()}; + } catch (IllegalArgumentException e) { + // assume CLI option provided the app name + // and read appType from command line since id wasn't provided + String appType = getSingleAppTypeFromCLI(cliParser); + return new String[]{applicationIdOrName, appType}; + } + } + + private static String getSingleAppTypeFromCLI(CommandLine cliParser) { + if (cliParser.hasOption(APP_TYPE_CMD)) { + String[] types = cliParser.getOptionValues(APP_TYPE_CMD); + if (types != null) { + for (String type : types) { + if (!type.trim().isEmpty()) { + return StringUtils.toLowerCase(type).trim(); + } + } + } + } + return AppAdminClient.DEFAULT_TYPE; + } + private void updateApplicationTimeout(String applicationId, ApplicationTimeoutType timeoutType, long timeoutInSec) throws YarnException, IOException { @@ -572,7 +797,7 @@ public class ApplicationCLI extends YarnCLI { /** * Kills applications with the application id as appId * - * @param Array of applicationIds + * @param applicationIds Array of applicationIds * @return errorCode * @throws YarnException * @throws IOException @@ -663,10 +888,10 @@ public class ApplicationCLI extends YarnCLI { * Prints the application report for an application id. * * @param applicationId - * @return exitCode + * @return ApplicationReport * @throws YarnException */ - private int printApplicationReport(String applicationId) + private ApplicationReport printApplicationReport(String applicationId) throws YarnException, IOException { ApplicationReport appReport = null; try { @@ -675,7 +900,7 @@ public class ApplicationCLI extends YarnCLI { } catch (ApplicationNotFoundException e) { sysout.println("Application with id '" + applicationId + "' doesn't exist in RM or Timeline Server."); - return -1; + return null; } // Use PrintWriter.println, which uses correct platform line ending. ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -739,11 +964,11 @@ public class ApplicationCLI extends YarnCLI { + "' doesn't exist in RM."); appReportStr.close(); sysout.println(baos.toString("UTF-8")); - return -1; + return null; } appReportStr.close(); sysout.println(baos.toString("UTF-8")); - return 0; + return appReport; } private void printResourceUsage(PrintWriter appReportStr, @@ -856,11 +1081,12 @@ public class ApplicationCLI extends YarnCLI { @SuppressWarnings("unchecked") private boolean hasAnyOtherCLIOptions(CommandLine cliParser, Options opts, - String excludeOption) { + String... excludeOptions) { Collection<Option> ops = opts.getOptions(); + Set<String> excludeSet = new HashSet<>(Arrays.asList(excludeOptions)); for (Option op : ops) { - // Skip exclude option from the option list - if (op.getOpt().equals(excludeOption)) { + // Skip exclude options from the option list + if (excludeSet.contains(op.getOpt())) { continue; } if (cliParser.hasOption(op.getOpt())) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/a54f7999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 84cfb0a..da1659f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -2025,49 +2025,121 @@ public class TestYarnCLI { ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); pw.println("usage: application"); - pw.println(" -appId <Application ID> Specify Application Id to be operated"); - pw.println(" -appStates <States> Works with -list to filter applications"); - pw.println(" based on input comma-separated list of"); - pw.println(" application states. The valid application"); - pw.println(" state can be one of the following:"); - pw.println(" ALL,NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUN"); - pw.println(" NING,FINISHED,FAILED,KILLED"); - pw.println(" -appTags <Tags> Works with -list to filter applications"); - pw.println(" based on input comma-separated list of"); - pw.println(" application tags."); - pw.println(" -appTypes <Types> Works with -list to filter applications"); - pw.println(" based on input comma-separated list of"); - pw.println(" application types."); - pw.println(" -changeQueue <Queue Name> Moves application to a new queue."); - pw.println(" ApplicationId can be passed using 'appId'"); - pw.println(" option. 'movetoqueue' command is"); - pw.println(" deprecated, this new command"); - pw.println(" 'changeQueue' performs same"); - pw.println(" functionality."); - pw.println(" -help Displays help for all commands."); - pw.println(" -kill <Application ID> Kills the application. Set of"); - pw.println(" applications can be provided separated"); - pw.println(" with space"); - pw.println(" -list List applications. Supports optional use"); - pw.println(" of -appTypes to filter applications based"); - pw.println(" on application type, -appStates to filter"); - pw.println(" applications based on application state"); - pw.println(" and -appTags to filter applications based"); - pw.println(" on application tag."); - pw.println(" -movetoqueue <Application ID> Moves the application to a different"); - pw.println(" queue. Deprecated command. Use"); - pw.println(" 'changeQueue' instead."); - pw.println(" -queue <Queue Name> Works with the movetoqueue command to"); - pw.println(" specify which queue to move an"); - pw.println(" application to."); - pw.println(" -status <Application ID> Prints the status of the application."); - pw.println(" -updateLifetime <Timeout> update timeout of an application from"); - pw.println(" NOW. ApplicationId can be passed using"); - pw.println(" 'appId' option. Timeout value is in"); - pw.println(" seconds."); - pw.println(" -updatePriority <Priority> update priority of an application."); - pw.println(" ApplicationId can be passed using 'appId'"); - pw.println(" option."); + pw.println(" -appId <Application ID> Specify Application Id to be"); + pw.println(" operated"); + pw.println(" -appStates <States> Works with -list to filter"); + pw.println(" applications based on input"); + pw.println(" comma-separated list of"); + pw.println(" application states. The valid"); + pw.println(" application state can be one of"); + pw.println(" the following:"); + pw.println(" ALL,NEW,NEW_SAVING,SUBMITTED,ACC"); + pw.println(" EPTED,RUNNING,FINISHED,FAILED,KI"); + pw.println(" LLED"); + pw.println(" -appTags <Tags> Works with -list to filter"); + pw.println(" applications based on input"); + pw.println(" comma-separated list of"); + pw.println(" application tags."); + pw.println(" -appTypes <Types> Works with -list to filter"); + pw.println(" applications based on input"); + pw.println(" comma-separated list of"); + pw.println(" application types."); + pw.println(" -changeQueue <Queue Name> Moves application to a new"); + pw.println(" queue. ApplicationId can be"); + pw.println(" passed using 'appId' option."); + pw.println(" 'movetoqueue' command is"); + pw.println(" deprecated, this new command"); + pw.println(" 'changeQueue' performs same"); + pw.println(" functionality."); + pw.println(" -component <Component Name> <Count> Works with -flex option to"); + pw.println(" change the number of"); + pw.println(" components/containers running"); + pw.println(" for an application /"); + pw.println(" long-running service. Supports"); + pw.println(" absolute or relative changes,"); + pw.println(" such as +1, 2, or -3."); + pw.println(" -destroy <Application Name> Destroys a saved application"); + pw.println(" specification and removes all"); + pw.println(" application data permanently."); + pw.println(" Supports -appTypes option to"); + pw.println(" specify which client"); + pw.println(" implementation to use."); + pw.println(" -enableFastLaunch Uploads AM dependencies to HDFS"); + pw.println(" to make future launches faster."); + pw.println(" Supports -appTypes option to"); + pw.println(" specify which client"); + pw.println(" implementation to use."); + pw.println(" -flex <Application Name or ID> Changes number of running"); + pw.println(" containers for a component of an"); + pw.println(" application / long-running"); + pw.println(" service. Requires -component"); + pw.println(" option. If name is provided,"); + pw.println(" appType must be provided unless"); + pw.println(" it is the default yarn-service."); + pw.println(" If ID is provided, the appType"); + pw.println(" will be looked up. Supports"); + pw.println(" -appTypes option to specify"); + pw.println(" which client implementation to"); + pw.println(" use."); + pw.println(" -help Displays help for all commands."); + pw.println(" -kill <Application ID> Kills the application. Set of"); + pw.println(" applications can be provided"); + pw.println(" separated with space"); + pw.println(" -launch <Application Name> <File Name> Launches application from"); + pw.println(" specification file (saves"); + pw.println(" specification and starts"); + pw.println(" application). Options"); + pw.println(" -updateLifetime and -changeQueue"); + pw.println(" can be specified to alter the"); + pw.println(" values provided in the file."); + pw.println(" Supports -appTypes option to"); + pw.println(" specify which client"); + pw.println(" implementation to use."); + pw.println(" -list List applications. Supports"); + pw.println(" optional use of -appTypes to"); + pw.println(" filter applications based on"); + pw.println(" application type, -appStates to"); + pw.println(" filter applications based on"); + pw.println(" application state and -appTags"); + pw.println(" to filter applications based on"); + pw.println(" application tag."); + pw.println(" -movetoqueue <Application ID> Moves the application to a"); + pw.println(" different queue. Deprecated"); + pw.println(" command. Use 'changeQueue'"); + pw.println(" instead."); + pw.println(" -queue <Queue Name> Works with the movetoqueue"); + pw.println(" command to specify which queue"); + pw.println(" to move an application to."); + pw.println(" -save <Application Name> <File Name> Saves specification file for an"); + pw.println(" application. Options"); + pw.println(" -updateLifetime and -changeQueue"); + pw.println(" can be specified to alter the"); + pw.println(" values provided in the file."); + pw.println(" Supports -appTypes option to"); + pw.println(" specify which client"); + pw.println(" implementation to use."); + pw.println(" -start <Application Name> Starts a previously saved"); + pw.println(" application. Supports -appTypes"); + pw.println(" option to specify which client"); + pw.println(" implementation to use."); + pw.println(" -status <Application ID> Prints the status of the"); + pw.println(" application."); + pw.println(" -stop <Application Name or ID> Stops application gracefully"); + pw.println(" (may be started again later). If"); + pw.println(" name is provided, appType must"); + pw.println(" be provided unless it is the"); + pw.println(" default yarn-service. If ID is"); + pw.println(" provided, the appType will be"); + pw.println(" looked up. Supports -appTypes"); + pw.println(" option to specify which client"); + pw.println(" implementation to use."); + pw.println(" -updateLifetime <Timeout> update timeout of an application"); + pw.println(" from NOW. ApplicationId can be"); + pw.println(" passed using 'appId' option."); + pw.println(" Timeout value is in seconds."); + pw.println(" -updatePriority <Priority> update priority of an"); + pw.println(" application. ApplicationId can"); + pw.println(" be passed using 'appId' option."); pw.close(); String appsHelpStr = baos.toString("UTF-8"); return appsHelpStr; http://git-wip-us.apache.org/repos/asf/hadoop/blob/a54f7999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index 6f00c5c..71c9e80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -37,25 +37,36 @@ User Commands Commands useful for users of a Hadoop cluster. -### `application` +### `application` or `app` Usage: `yarn application [options] ` +Usage: `yarn app [options] ` | COMMAND\_OPTIONS | Description | |:---- |:---- | | -appId \<ApplicationId\> | Specify Application Id to be operated | | -appStates \<States\> | Works with -list to filter applications based on input comma-separated list of application states. The valid application state can be one of the following: Â ALL, NEW, NEW\_SAVING, SUBMITTED, ACCEPTED, RUNNING, FINISHED, FAILED, KILLED | +| -appTags \<Tags\> | Works with -list to filter applications based on input comma-separated list of application tags. | | -appTypes \<Types\> | Works with -list to filter applications based on input comma-separated list of application types. | -| -help | Help | -| -list | Lists applications from the RM. Supports optional use of -appTypes to filter applications based on application type, and -appStates to filter applications based on application state. | -| -kill \<ApplicationId\> | Kills the application. | -| -movetoqueue \<Application Id\> | Moves the application to a different queue. | +| -changeQueue \<Queue Name\> | Moves application to a new queue. ApplicationId can be passed using 'appId' option. 'movetoqueue' command is deprecated, this new command 'changeQueue' performs same functionality. | +| -component \<Component Name\> \<Count\> | Works with -flex option to change the number of components/containers running for an application / long-running service. Supports absolute or relative changes, such as +1, 2, or -3. | +| -destroy \<Application Name\> | Destroys a saved application specification and removes all application data permanently. Supports -appTypes option to specify which client implementation to use. | +| -enableFastLaunch | Uploads AM dependencies to HDFS to make future launches faster. Supports -appTypes option to specify which client implementation to use. | +| -flex \<Application Name or ID\> | Changes number of running containers for a component of an application / long-running service. Requires -component option. If name is provided, appType must be provided unless it is the default yarn-service. If ID is provided, the appType will be looked up. Supports -appTypes option to specify which client implementation to use. | +| -help | Displays help for all commands. | +| -kill \<Application ID\> | Kills the application. Set of applications can be provided separated with space | +| -launch \<Application Name\> \<File Name\> | Launches application from specification file (saves specification and starts application). Options -updateLifetime and -changeQueue can be specified to alter the values provided in the file. Supports -appTypes option to specify which client implementation to use. | +| -list | List applications. Supports optional use of -appTypes to filter applications based on application type, -appStates to filter applications based on application state and -appTags to filter applications based on application tag. | +| -movetoqueue \<Application ID\> | Moves the application to a different queue. Deprecated command. Use 'changeQueue' instead. | | -queue \<Queue Name\> | Works with the movetoqueue command to specify which queue to move an application to. | +| -save \<Application Name\> \<File Name\> | Saves specification file for an application. Options -updateLifetime and -changeQueue can be specified to alter the values provided in the file. Supports -appTypes option to specify which client implementation to use. | +| -start \<Application Name\> | Starts a previously saved application. Supports -appTypes option to specify which client implementation to use. | | -status \<ApplicationId\> | Prints the status of the application. | -| -updateLifetime \<Timeout\> | Update application timeout (from the time of request) in seconds. ApplicationId can be specified using 'appId' option. | +| -stop \<Application Name or ID\> | Stops application gracefully (may be started again later). If name is provided, appType must be provided unless it is the default yarn-service. If ID is provided, the appType will be looked up. Supports -appTypes option to specify which client implementation to use. | +| -updateLifetime \<Timeout\> | Update timeout of an application from NOW. ApplicationId can be passed using 'appId' option. Timeout value is in seconds. | | -updatePriority \<Priority\> | Update priority of an application. ApplicationId can be passed using 'appId' option. | -Prints application(s) report/kill application +Prints application(s) report/kill application/manage long running application ### `applicationattempt` @@ -69,61 +80,6 @@ Usage: `yarn applicationattempt [options] ` prints applicationattempt(s) report -### `service` -Usage `yarn service [sub-command] [service-name] [options]` - -#### `SUB_COMMAND` - -* `build`: Build a service with its specifications, but do not start it. - ``` - Usage: yarn service build [service-name] --file [file] - Fields: - service-name Optional. If specified, it will override the name in the service definition. - - Options: - --file,-f The local path to the service definition file - ``` - -* `create`: create a service, it's equivalent to first invoke build and then start. - ``` - Usage: yarn service create [service-name] --file [file] - Fields: - service-name Optional. If specified, it will override the name in the service definition. - - Options: - --file,-f The local path to the service definition file. - --queue,-q The queue to which the service is submitted. - --example,-e The name of the example service such as: - Sleeper A simple service that launches a few non-docker sleep containers on YARN. - ``` - -* `dependency`: Yarn service framework dependency (libraries) management. - ``` - Usage: yarn service dependency [options] - Option: - --upload Pre-upload the dependency jars onto HDFS to expediate service launch process. - ``` - -* `destroy`: Destroy a stopped service, service must be stopped first before destroying. - ``` - Usage: yarn service destroy [service-name] - ``` -* `flex`: Flex a service's component by increasing or decreasing the number of containers. - ``` - Usage: yarn service flex [service-name] --component [component-name] [count] - Options: - --component [component-name] [count] - Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3. - ``` -* `status`: Get the status of a service. - ``` - Usage: yarn service status [service-name] - ``` -* `start`: Start a service with pre-built specification or a previously stopped service. - ``` - Usage: yarn service start [service-name] - ``` - ### `classpath` Usage: `yarn classpath [--glob |--jar <path> |-h |--help]` @@ -328,12 +284,6 @@ Usage: `yarn timelineserver` Start the TimeLineServer -### apiserver - -Usage: `yarn apiserver` - -Start the API-server for deploying/managing services on YARN - ### registrydns Usage: `yarn registrydns` http://git-wip-us.apache.org/repos/asf/hadoop/blob/a54f7999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md index db6d553..e4881ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md @@ -100,16 +100,18 @@ ProxyPassReverse "/" "balancer://test/" Then run the service with the command: ``` -yarn service create [service-name] --example httpd +yarn app -launch <service-name> httpd ``` -where `service-name` is optional. If omitted, it uses the name defined in the `Yarnfile`. + +The last argument is either the path to a JSON specification of the service, or in this case, the name of an example service. +The directory where examples can be found can be configured by setting the YARN\_EXAMPLES\_DIR environment variable. Once the service is running, navigate to `http://httpd-proxy-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080` to see the root page. The pages should alternately show "Hello from httpd-0!" or "Hello from httpd-1!" The individual httpd URLs can also be visited, `http://httpd-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080` and `http://httpd-1.${SERVICE_NAME}.${USER}.${DOMAIN}:8080`. -If unsure of your hostnames, visit the apiserver REST endpoint `http://<apiserver host>:9191/ws/v1/services/httpd-service`. +If unsure of your hostnames, visit the RM REST endpoint `http://<RM host>:8088/ws/v1/services/httpd-service`. ## Apache web server - httpd (without registry DNS) @@ -151,9 +153,9 @@ hdfs dfs -copyFromLocal ${HADOOP_YARN_HOME}/share/hadoop/yarn/yarn-service-examp Then run the service with the command: ``` -yarn service create [service-name] --example httpd-no-dns +yarn app -launch <service-name> httpd-no-dns ``` where `service-name` is optional. If omitted, it uses the name defined in the `Yarnfile`. -Look up your IPs at the apiserver REST endpoint `http://<apiserver host>:9191/ws/v1/services/httpd-service`. +Look up your IPs at the RM REST endpoint `http://<RM host>:8088/ws/v1/services/httpd-service`. Then visit port 8080 for each IP to view the pages. http://git-wip-us.apache.org/repos/asf/hadoop/blob/a54f7999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md index afcd5f5..512c011 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md @@ -45,11 +45,11 @@ Below is a simple service definition that launches sleep containers on YARN by w ``` User can simply run a pre-built example service on YARN using below command: ``` -yarn service create [service-name] --example [example-name] +yarn app -launch <service-name> <example-name> ``` e.g. Below command launches a `sleeper` service named as `my-sleeper` on YARN. ``` -yarn service create my-sleeper --example sleeper +yarn app -launch my-sleeper sleeper ``` For launching docker based services using YARN Service framework, please refer to [API doc](YarnServiceAPI.md). @@ -58,51 +58,53 @@ Below steps walk you through deploying a services on YARN using CLI. Refer to [Yarn Commands](../YarnCommands.md) for the full list of commands and options. ### Deploy a service ``` -yarn service create --file ${PATH_TO_SERVICE_DEF_FILE} +yarn app -launch ${SERVICE_NAME} ${PATH_TO_SERVICE_DEF_FILE} ``` Params: -- SERVICE_NAME: The name of the service. Note that this needs to be unique across all running services. +- SERVICE_NAME: The name of the service. Note that this needs to be unique across running services for the current user. - PATH_TO_SERVICE_DEF: The path to the service definition file in JSON format. For example: ``` -yarn service create --file /path/to/local/sleeper.json +yarn app -launch sleeper-service /path/to/local/sleeper.json ``` ### Flex a component of a service Increase or decrease the number of containers for a component. ``` -yarn service flex ${SERVICE_NAME} --component ${COMPONENT_NAME} ${NUMBER_OF_CONTAINERS} +yarn app -flex ${SERVICE_NAME} -component ${COMPONENT_NAME} ${NUMBER_OF_CONTAINERS} ``` For example, for a service named `sleeper-service`: Set the `sleeper` component to `2` containers (absolute number). ``` -yarn service flex sleeper-service --component sleeper 2 +yarn app -flex sleeper-service -component sleeper 2 ``` +Relative changes are also supported for the ${NUMBER_OF_CONTAINERS} in the flex command, such as +2 or -2. + ### Stop a service Stopping a service will stop all containers of the service and the ApplicationMaster, but does not delete the state of a service, such as the service root folder on hdfs. ``` -yarn service stop ${SERVICE_NAME} +yarn app -stop ${SERVICE_NAME} ``` ### Restart a stopped service Restarting a stopped service is easy - just call start! ``` -yarn service start ${SERVICE_NAME} +yarn app -start ${SERVICE_NAME} ``` ### Destroy a service In addition to stopping a service, it also deletes the service root folder on hdfs and the records in YARN Service Registry. ``` -yarn service destroy ${SERVICE_NAME} +yarn app -destroy ${SERVICE_NAME} ``` ## Manage services on YARN via REST API -YARN API Server REST API can be activated in two modes: embedded or standalone. +YARN API Server REST API can be activated as part of the ResourceManager. ### Start Embedded API-Server as part of ResourceManager For running inside ResourceManager, add this property to `yarn-site.xml` and restart ResourceManager. @@ -119,13 +121,6 @@ For running inside ResourceManager, add this property to `yarn-site.xml` and res Services can be deployed on YARN through the ResourceManager web endpoint. -### Start Standalone API-Server for deploying services on YARN -API server is the service that sits in front of YARN ResourceManager and lets users submit their service specs via HTTP. -``` - yarn --daemon start apiserver - ``` -The above command starts the API Server on the localhost at port 9191 by default. - Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn. ### Deploy a service @@ -135,19 +130,14 @@ POST the aforementioned example service definition to the ResourceManager api-se POST http://localhost:8088/ws/v1/services ``` -Or standalone API server: -``` -POST http://localhost:9191/ws/v1/services -``` - ### Get a service status ``` -GET http://localhost:9191/ws/v1/services/${SERVICE_NAME} +GET http://localhost:8088/ws/v1/services/${SERVICE_NAME} ``` ### Flex a component of a service ``` -PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}/components/${COMPONENT_NAME} +PUT http://localhost:8088/ws/v1/services/${SERVICE_NAME}/components/${COMPONENT_NAME} ``` `PUT` Request body: ``` @@ -168,7 +158,7 @@ For example: Stopping a service will stop all containers of the service and the ApplicationMaster, but does not delete the state of a service, such as the service root folder on hdfs. ``` -PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME} +PUT http://localhost:8088/ws/v1/services/${SERVICE_NAME} ``` `PUT` Request body: @@ -183,7 +173,7 @@ PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME} Restarting a stopped service is easy. ``` -PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME} +PUT http://localhost:8088/ws/v1/services/${SERVICE_NAME} ``` `PUT` Request body: @@ -196,7 +186,7 @@ PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME} ### Destroy a service In addition to stopping a service, it also deletes the service root folder on hdfs and the records in YARN Service Registry. ``` -DELETE http://localhost:9191/ws/v1/services/${SERVICE_NAME} +DELETE http://localhost:8088/ws/v1/services/${SERVICE_NAME} ``` ## Services UI with YARN UI2 and Timeline Service v2 http://git-wip-us.apache.org/repos/asf/hadoop/blob/a54f7999/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md index e9aaf7e..7a2490e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md @@ -37,7 +37,7 @@ License URL: http://www.apache.org/licenses/LICENSE-2.0.html ### URI scheme Host: host.mycompany.com -Port: 9191(default) +Port: 8088(default RM port) Schemes: HTTP @@ -368,7 +368,7 @@ The current status of a submitted service, returned as a response to the GET API ## Examples ### Create a simple single-component service with most attribute values as defaults -POST URL - http://localhost:9191/ws/v1/services +POST URL - http://localhost:8088/ws/v1/services ##### POST Request JSON ```json @@ -394,7 +394,7 @@ POST URL - http://localhost:9191/ws/v1/services ``` ##### GET Response JSON -GET URL - http://localhost:9191/ws/v1/services/hello-world +GET URL - http://localhost:8088/ws/v1/services/hello-world Note, lifetime value of -1 means unlimited lifetime. @@ -452,7 +452,7 @@ Note, lifetime value of -1 means unlimited lifetime. ``` ### Update to modify the lifetime of a service -PUT URL - http://localhost:9191/ws/v1/services/hello-world +PUT URL - http://localhost:8088/ws/v1/services/hello-world ##### PUT Request JSON @@ -464,7 +464,7 @@ Note, irrespective of what the current lifetime value is, this update request wi } ``` ### Stop a service -PUT URL - http://localhost:9191/ws/v1/services/hello-world +PUT URL - http://localhost:8088/ws/v1/services/hello-world ##### PUT Request JSON ```json @@ -474,7 +474,7 @@ PUT URL - http://localhost:9191/ws/v1/services/hello-world ``` ### Start a service -PUT URL - http://localhost:9191/ws/v1/services/hello-world +PUT URL - http://localhost:8088/ws/v1/services/hello-world ##### PUT Request JSON ```json @@ -484,7 +484,7 @@ PUT URL - http://localhost:9191/ws/v1/services/hello-world ``` ### Update to flex up/down the no of containers (instances) of a component of a service -PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello +PUT URL - http://localhost:8088/ws/v1/services/hello-world/components/hello ##### PUT Request JSON ```json @@ -495,12 +495,12 @@ PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello ``` ### Destroy a service -DELETE URL - http://localhost:9191/ws/v1/services/hello-world +DELETE URL - http://localhost:8088/ws/v1/services/hello-world *** ### Create a complicated service - HBase -POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1 +POST URL - http://localhost:8088:/ws/v1/services/hbase-app-1 ##### POST Request JSON @@ -595,4 +595,4 @@ POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1 "Proxied HBase Master Status UI": "http://app-proxy/${DOMAIN}/${USER}/${SERVICE_NAME}/hbasemaster/16010/"; } } -``` \ No newline at end of file +``` --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
