YARN-7165. Miscellaneous fixes in yarn-native-services. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/541b64c7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/541b64c7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/541b64c7 Branch: refs/heads/trunk Commit: 541b64c7985db425c5ce8bd5f71a2d5980809708 Parents: b9ca13e Author: Gour Saha <gourks...@apache.org> Authored: Thu Sep 7 23:25:31 2017 -0700 Committer: Jian He <jia...@apache.org> Committed: Mon Nov 6 13:30:15 2017 -0800 ---------------------------------------------------------------------- hadoop-project/pom.xml | 4 +- hadoop-project/src/site/site.xml | 10 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 7 +- ...RN-Simplified-V1-API-Layer-For-Services.yaml | 11 +- .../hadoop/yarn/service/ServiceMaster.java | 10 +- .../hadoop/yarn/service/client/ServiceCLI.java | 5 +- .../yarn/service/client/ServiceClient.java | 20 +- .../client/params/AbstractActionArgs.java | 33 +- .../AbstractClusterBuildingActionArgs.java | 22 +- .../service/client/params/ActionBuildArgs.java | 5 + .../service/client/params/ActionCreateArgs.java | 5 + .../client/params/ActionDependencyArgs.java | 4 +- .../service/client/params/ActionExamples.java | 26 + .../service/client/params/ActionUpdateArgs.java | 7 +- .../yarn/service/client/params/Arguments.java | 9 +- .../yarn/service/client/params/ClientArgs.java | 10 - .../yarn/service/client/params/CommonArgs.java | 28 +- .../service/client/params/ServiceAMArgs.java | 50 ++ .../client/params/ServiceAMCreateAction.java | 37 ++ .../service/client/params/SliderAMArgs.java | 57 -- .../client/params/SliderAMCreateAction.java | 73 --- .../service/client/params/SliderActions.java | 9 +- .../ServiceTimelinePublisher.java | 5 +- .../client/TestBuildExternalComponents.java | 6 +- .../yarn/service/client/TestServiceCLI.java | 30 +- .../hadoop/yarn/service/conf/examples/app.json | 2 +- .../registry/client/api/RegistryConstants.java | 2 +- .../src/site/markdown/YarnCommands.md | 56 +- .../native-services/NativeServicesAPI.md | 606 ------------------- .../native-services/NativeServicesDiscovery.md | 144 ----- .../native-services/NativeServicesIntro.md | 107 ---- .../src/site/markdown/yarn-service/Concepts.md | 77 +++ .../src/site/markdown/yarn-service/Overview.md | 58 ++ .../site/markdown/yarn-service/QuickStart.md | 218 +++++++ .../markdown/yarn-service/ServiceDiscovery.md | 150 +++++ .../markdown/yarn-service/YarnServiceAPI.md | 592 ++++++++++++++++++ 36 files changed, 1363 insertions(+), 1132 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 3cb9568..d18c9a8 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -143,10 +143,10 @@ <!-- the version of Hadoop declared in the version resources; can be overridden so that Hadoop 3.x can declare itself a 2.x artifact. --> <declared.hadoop.version>${project.version}</declared.hadoop.version> - + <swagger-annotations-version>1.5.4</swagger-annotations-version> <snakeyaml.version>1.16</snakeyaml.version> - <jcommander.version>1.30</jcommander.version> + <jcommander.version>1.48</jcommander.version> </properties> <dependencyManagement> http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-project/src/site/site.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml index ad893a2..57cff9a 100644 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@ -155,10 +155,12 @@ <item name="Timeline Service V.2" href="hadoop-yarn/hadoop-yarn-site/TimelineServiceV2.html#Timeline_Service_v.2_REST_API"/> </menu> - <menu name="YARN Native Services" inherit="top"> - <item name="Introduction" href="hadoop-yarn/hadoop-yarn-site/native-services/NativeServicesIntro.html"/> - <item name="Native Services API" href="hadoop-yarn/hadoop-yarn-site/native-services/NativeServicesAPI.html"/> - <item name="Native Services Discovery" href="hadoop-yarn/hadoop-yarn-site/native-services/NativeServicesDiscovery.html"/> + <menu name="YARN Service" inherit="top"> + <item name="Overview" href="hadoop-yarn/hadoop-yarn-site/yarn-service/Overview.html"/> + <item name="QuickStart" href="hadoop-yarn/hadoop-yarn-site/yarn-service/QuickStart.html"/> + <item name="Concepts" href="hadoop-yarn/hadoop-yarn-site/yarn-service/Concepts.html"/> + <item name="Yarn Service API" href="hadoop-yarn/hadoop-yarn-site/yarn-service/YarnServiceAPI.html"/> + <item name="Service Discovery" href="hadoop-yarn/hadoop-yarn-site/yarn-service/ServiceDiscovery.html"/> </menu> <menu name="Hadoop Compatible File Systems" inherit="top"> http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/bin/yarn ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 1db585e..05d892b 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -31,7 +31,7 @@ function hadoop_usage hadoop_add_option "--hosts filename" "list of hosts to use in worker mode" hadoop_add_option "--workers" "turn on worker mode" - hadoop_add_subcommand "apiserver" "run yarn-native-service rest server" + hadoop_add_subcommand "apiserver" daemon "run the api-server for deploying/managing services on YARN" hadoop_add_subcommand "application" client "prints application(s) report/kill application" hadoop_add_subcommand "applicationattempt" client "prints applicationattempt(s) report" hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries" @@ -50,10 +50,9 @@ function hadoop_usage hadoop_add_subcommand "router" daemon "run the Router daemon" hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration" hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools" - hadoop_add_subcommand "apiserver" "run yarn-native-service rest server" hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon" - hadoop_add_subcommand "service" "run a service" - hadoop_add_subcommand "sharedcachemanager" admin "run the SharedCacheManager daemon" + hadoop_add_subcommand "service" client "deploy/manage a service on YARN" + hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon" hadoop_add_subcommand "timelinereader" client "run the timeline reader server" hadoop_add_subcommand "timelineserver" daemon "run the timeline server" hadoop_add_subcommand "top" client "view cluster information" http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml index b084be7..b9b5b3a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml @@ -48,17 +48,16 @@ info: url: http://www.apache.org/licenses/LICENSE-2.0.html # the domain of the service host: host.mycompany.com +port: 9191(default) # array of all schemes that your API supports schemes: - http -# will be prefixed to all paths -basePath: /ws/v1/ consumes: - application/json produces: - application/json paths: - /services/version: + /ws/v1/services/version: get: summary: Get current version of the API server. description: Get current version of the API server. @@ -66,7 +65,7 @@ paths: 200: description: Successful request - /services: + /ws/v1/services: get: summary: (TBD) List of services running in the cluster. description: Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name. @@ -103,7 +102,7 @@ paths: schema: $ref: '#/definitions/ServiceStatus' - /services/{service_name}: + /ws/v1/services/{service_name}: put: summary: Update a service or upgrade the binary version of the components of a running service description: Update the runtime properties of a service. Currently the following operations are supported - update lifetime, stop/start a service. @@ -174,7 +173,7 @@ paths: description: Unexpected error schema: $ref: '#/definitions/ServiceStatus' - /services/{service_name}/components/{component_name}: + /ws/v1/services/{service_name}/components/{component_name}: put: summary: Flex a component's number of instances. description: Set a component's desired number of instanes http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java index e700961..2abdae1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java @@ -34,7 +34,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; -import org.apache.hadoop.yarn.service.client.params.SliderAMArgs; +import org.apache.hadoop.yarn.service.client.params.ServiceAMArgs; import org.apache.hadoop.yarn.service.monitor.ServiceMonitor; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; @@ -51,7 +51,7 @@ public class ServiceMaster extends CompositeService { private static final Logger LOG = LoggerFactory.getLogger(ServiceMaster.class); - private static SliderAMArgs amArgs; + private static ServiceAMArgs amArgs; protected ServiceContext context; public ServiceMaster(String name) { @@ -108,7 +108,7 @@ public class ServiceMaster extends CompositeService { } protected Path getAppDir() { - return new Path(amArgs.getAppDefPath()).getParent(); + return new Path(amArgs.getServiceDefPath()).getParent(); } protected ServiceScheduler createServiceScheduler(ServiceContext context) @@ -119,7 +119,7 @@ public class ServiceMaster extends CompositeService { protected void loadApplicationJson(ServiceContext context, SliderFileSystem fs) throws IOException { context.service = ServiceApiUtil - .loadServiceFrom(fs, new Path(amArgs.getAppDefPath())); + .loadServiceFrom(fs, new Path(amArgs.getServiceDefPath())); LOG.info(context.service.toString()); } @@ -138,7 +138,7 @@ public class ServiceMaster extends CompositeService { public static void main(String[] args) throws Exception { Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); StringUtils.startupShutdownMessage(ServiceMaster.class, args, LOG); - amArgs = new SliderAMArgs(args); + amArgs = new ServiceAMArgs(args); amArgs.parse(); try { ServiceMaster serviceMaster = new ServiceMaster("Service Master"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java index cb27021..928c06f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java @@ -18,11 +18,12 @@ package org.apache.hadoop.yarn.service.client; +import com.beust.jcommander.ParameterException; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.client.params.ClientArgs; -import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; +import org.apache.hadoop.yarn.service.exceptions.SliderException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -101,7 +102,7 @@ public class ServiceCLI { ClientArgs clientArgs = new ClientArgs(args); try { clientArgs.parse(); - } catch (BadCommandArgumentsException e) { + } catch (ParameterException | SliderException e) { System.err.println(e.getMessage()); System.exit(-1); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index c3a2752..bfddc44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -158,7 +158,7 @@ public class ServiceClient extends CompositeService private Service loadAppJsonFromLocalFS( AbstractClusterBuildingActionArgs args) throws IOException { - File file = args.getAppDef(); + File file = args.getFile(); Path filePath = new Path(file.getAbsolutePath()); LOG.info("Loading app json from: " + filePath); Service service = jsonSerDeser @@ -166,7 +166,6 @@ public class ServiceClient extends CompositeService if (args.lifetime > 0) { service.setLifetime(args.lifetime); } - service.setName(args.getClusterName()); return service; } @@ -407,7 +406,8 @@ public class ServiceClient extends CompositeService public int actionDestroy(String serviceName) throws Exception { ServiceApiUtil.validateNameFormat(serviceName, getConfig()); - verifyNoLiveAppInRM(serviceName, "Destroy"); + verifyNoLiveAppInRM(serviceName, "destroy"); + Path appDir = fs.buildClusterDirPath(serviceName); FileSystem fileSystem = fs.getFileSystem(); // remove from the appId cache @@ -498,9 +498,15 @@ public class ServiceClient extends CompositeService request.setApplicationStates(liveStates); List<ApplicationReport> reports = yarnClient.getApplications(request); if (!reports.isEmpty()) { - throw new YarnException( - "Failed to " + action + " service, as " + serviceName - + " already exists."); + String message = ""; + if (action.equals("destroy")) { + message = "Failed to destroy service " + serviceName + + ", because it is still running."; + } else { + message = "Failed to " + action + " service " + serviceName + + ", because it already exists."; + } + throw new YarnException(message); } } @@ -592,7 +598,7 @@ public class ServiceClient extends CompositeService CLI.add(ServiceMaster.class.getCanonicalName()); CLI.add(ACTION_CREATE, serviceName); //TODO debugAM CLI.add(Arguments.ARG_DEBUG) - CLI.add(Arguments.ARG_CLUSTER_URI, new Path(appRootDir, serviceName + ".json")); + CLI.add(Arguments.ARG_SERVICE_DEF_PATH, new Path(appRootDir, serviceName + ".json")); // pass the registry binding CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT, RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT); http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java index ea3bb0a..8b2d93e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java @@ -41,22 +41,10 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments { } /** - * URI/binding to the filesystem - */ - @Parameter(names = {ARG_FILESYSTEM, ARG_FILESYSTEM_LONG}, - description = "Filesystem Binding") - public String filesystemBinding; - - @Parameter(names = {ARG_BASE_PATH}, - description = "Service base path on the filesystem", - converter = PathArgumentConverter.class) - public Path basePath; - - /** * This is the default parameter */ @Parameter - public final List<String> parameters = new ArrayList<>(); + public List<String> parameters = new ArrayList<>(); /** * get the name: relies on arg 1 being the cluster name in all operations @@ -76,29 +64,22 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments { */ - @Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions") - public final List<String> definitions = new ArrayList<>(); + @Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions", hidden = true) + public List<String> definitions = new ArrayList<>(); /** * System properties */ @Parameter(names = {ARG_SYSPROP}, arity = 1, description = "system properties in the form name value" + - " These are set after the JVM is started.") - public final List<String> sysprops = new ArrayList<>(0); + " These are set after the JVM is started.", + hidden = true) + public List<String> sysprops = new ArrayList<>(0); - @Parameter(names = {ARG_MANAGER_SHORT, ARG_MANAGER}, - description = "Binding (usually hostname:port) of the YARN resource manager") - public String manager; - - - @Parameter(names = ARG_DEBUG, description = "Debug mode") + @Parameter(names = ARG_DEBUG, description = "Debug mode", hidden = true) public boolean debug = false; - @Parameter(names = {ARG_HELP}, description = "Help", help = true) - public boolean help = false; - /** * Get the min #of params expected * @return the min number of params in the {@link #parameters} field http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java index 3a3a19a..d215436 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java @@ -19,13 +19,8 @@ package org.apache.hadoop.yarn.service.client.params; import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParametersDelegate; -import com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; import java.io.File; -import java.util.List; -import java.util.Map; /** * Abstract Action to build things; shares args across build and @@ -33,12 +28,12 @@ import java.util.Map; */ public abstract class AbstractClusterBuildingActionArgs extends AbstractActionArgs { - @Parameter(names = {ARG_APPDEF}, - description = "Template service definition file in JSON format.") - public File appDef; + @Parameter(names = { ARG_FILE, ARG_FILE_SHORT }, required = true, + description = "The path to the service definition file in JSON format.") + public File file; - public File getAppDef() { - return appDef; + public File getFile() { + return file; } @Parameter(names = { @@ -48,11 +43,4 @@ public abstract class AbstractClusterBuildingActionArgs @Parameter(names = { ARG_LIFETIME }, description = "Lifetime of the service from the time of request") public long lifetime; - - @ParametersDelegate - public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate(); - - @ParametersDelegate - public OptionArgsDelegate optionsDelegate = - new OptionArgsDelegate(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java index c2ff545..cf61525 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java @@ -28,4 +28,9 @@ public class ActionBuildArgs extends AbstractClusterBuildingActionArgs { public String getActionName() { return SliderActions.ACTION_BUILD; } + + @Override + public int getMinParams() { + return 0; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java index eecffb6..c43f4fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java @@ -29,5 +29,10 @@ public class ActionCreateArgs extends AbstractClusterBuildingActionArgs { public String getActionName() { return SliderActions.ACTION_CREATE; } + + @Override + public int getMinParams() { + return 0; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java index 51e07c9..699e6ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java @@ -32,8 +32,8 @@ public class ActionDependencyArgs extends AbstractActionArgs { return SliderActions.ACTION_DEPENDENCY; } - @Parameter(names = { ARG_UPLOAD }, - description = "Upload AM and agent libraries to HDFS for this client") + @Parameter(names = { ARG_UPLOAD }, required = true, + description = "Upload AM libraries to HDFS for this client version") public boolean upload; @Parameter(names = { ARG_OVERWRITE }, http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java new file mode 100644 index 0000000..e489e17 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.client.params; + +import com.beust.jcommander.Parameters; + +@Parameters(commandNames = { SliderActions.ACTION_EXAMPLES}, + commandDescription = SliderActions.DESCRIBE_ACTION_EXAMPLES) +public class ActionExamples { +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java index e310f45..00af69a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java @@ -18,12 +18,17 @@ package org.apache.hadoop.yarn.service.client.params; +import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; @Parameters(commandNames = { SliderActions.ACTION_UPDATE}, commandDescription = SliderActions.DESCRIBE_ACTION_UPDATE) -public class ActionUpdateArgs extends AbstractClusterBuildingActionArgs { +public class ActionUpdateArgs extends AbstractActionArgs { + + @Parameter(names = { + ARG_LIFETIME }, description = "Lifetime of the service from the time of request") + public long lifetime; @Override public String getActionName() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java index 204149b..911440d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java @@ -27,7 +27,8 @@ package org.apache.hadoop.yarn.service.client.params; */ public interface Arguments { - String ARG_APPDEF = "--appdef"; + String ARG_FILE = "--file"; + String ARG_FILE_SHORT = "-f"; String ARG_BASE_PATH = "--basepath"; String ARG_COMPONENT = "--component"; String ARG_COMPONENT_SHORT = "--comp"; @@ -95,9 +96,7 @@ public interface Arguments { RIGHT PLACE IN THE LIST */ - /** - * server: URI for the cluster - */ - String ARG_CLUSTER_URI = "-cluster-uri"; + // Tha path in hdfs to be read by Service AM + String ARG_SERVICE_DEF_PATH = "-cluster-uri"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java index 09cae24..f479cd2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java @@ -85,16 +85,6 @@ public class ClientArgs extends CommonArgs { public void applyDefinitions(Configuration conf) throws BadCommandArgumentsException { super.applyDefinitions(conf); - //RM - if (getManager() != null) { - log.debug("Setting RM to {}", getManager()); - conf.set(YarnConfiguration.RM_ADDRESS, getManager()); - } - if (getBasePath() != null) { - log.debug("Setting basePath to {}", getBasePath()); - conf.set(YarnServiceConf.YARN_SERVICE_BASE_PATH, - getBasePath().toString()); - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java index e1197ea..a5638a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java @@ -156,12 +156,8 @@ public abstract class CommonArgs extends ArgOps implements SliderActions, try { commander.parse(args); } catch (ParameterException e) { - throw new BadCommandArgumentsException(e, "%s in %s", - e.toString(), - (args != null - ? (SliderUtils.join(args, - " ", false)) - : "[]")); + commander.usage(args[0]); + throw e; } //now copy back to this class some of the attributes that are common to all //actions @@ -251,31 +247,11 @@ public abstract class CommonArgs extends ArgOps implements SliderActions, applyDefinitions(definitionMap, conf); } - - /** - * If the Filesystem binding was provided, it overrides anything in - * the configuration - * @param conf configuration - */ - public void applyFileSystemBinding(Configuration conf) { - ArgOps.applyFileSystemBinding(getFilesystemBinding(), conf); - } - public boolean isDebug() { return coreAction.debug; } - public String getFilesystemBinding() { - return coreAction.filesystemBinding; - } - - public Path getBasePath() { return coreAction.basePath; } - - public String getManager() { - return coreAction.manager; - } - public String getAction() { return commander.getParsedCommand(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java new file mode 100644 index 0000000..800bb24 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.yarn.service.client.params; + +/** + * Parameters sent by the Client to the AM + */ +public class ServiceAMArgs extends CommonArgs { + + ServiceAMCreateAction createAction = new ServiceAMCreateAction(); + + public ServiceAMArgs(String[] args) { + super(args); + } + + @Override + protected void addActionArguments() { + addActions(createAction); + } + + // This is the path in hdfs to the service definition JSON file + public String getServiceDefPath() { + return createAction.serviceDefPath; + } + + /** + * Am binding is simple: there is only one action + */ + @Override + public void applyAction() { + bindCoreAction(createAction); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java new file mode 100644 index 0000000..384e8ba --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.client.params; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +@Parameters(commandNames = { SliderActions.ACTION_CREATE}, + commandDescription = SliderActions.DESCRIBE_ACTION_CREATE) + +public class ServiceAMCreateAction extends AbstractActionArgs { + + @Override + public String getActionName() { + return SliderActions.ACTION_CREATE; + } + + @Parameter(names = ARG_SERVICE_DEF_PATH, + description = "Path to the service definition JSON file", required = true) + public String serviceDefPath; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java deleted file mode 100644 index 1c38213..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMArgs.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.apache.hadoop.yarn.service.client.params; - -/** - * Parameters sent by the Client to the AM - */ -public class SliderAMArgs extends CommonArgs { - - SliderAMCreateAction createAction = new SliderAMCreateAction(); - - public SliderAMArgs(String[] args) { - super(args); - } - - @Override - protected void addActionArguments() { - addActions(createAction); - } - - public String getImage() { - return createAction.image; - } - - /** - * This is the URI in the FS to the Slider cluster; the conf file (and any - * other cluster-specifics) can be picked up here - */ - public String getAppDefPath() { - return createAction.sliderClusterURI; - } - - /** - * Am binding is simple: there is only one action - */ - @Override - public void applyAction() { - bindCoreAction(createAction); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java deleted file mode 100644 index a446665..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderAMCreateAction.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.client.params; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.beust.jcommander.ParametersDelegate; - -import java.io.File; - - -@Parameters(commandNames = { SliderActions.ACTION_CREATE}, - commandDescription = SliderActions.DESCRIBE_ACTION_CREATE) - -public class SliderAMCreateAction extends AbstractActionArgs implements - LaunchArgsAccessor { - - - @Override - public String getActionName() { - return SliderActions.ACTION_CREATE; - } - - @Parameter(names = ARG_IMAGE, description = "image", required = false) - public String image; - - /** - * This is the URI in the FS to the Slider cluster; the conf file (and any - * other cluster-specifics) can be picked up here - */ - @Parameter(names = ARG_CLUSTER_URI, - description = "URI to the Slider cluster", required = true) - public String sliderClusterURI; - - @ParametersDelegate LaunchArgsDelegate launchArgs = new LaunchArgsDelegate(); - - @Override - public String getRmAddress() { - return launchArgs.getRmAddress(); - } - - @Override - public int getWaittime() { - return launchArgs.getWaittime(); - } - - @Override - public void setWaittime(int waittime) { - launchArgs.setWaittime(waittime); - } - - @Override - public File getOutputFile() { - return launchArgs.getOutputFile(); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java index 1b2a92d..fa05f2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java @@ -19,9 +19,7 @@ package org.apache.hadoop.yarn.service.client.params; /** - * Actions. - * Only some of these are supported by specific Slider Services; they - * are listed here to ensure the names are consistent + * Actions by client */ public interface SliderActions { String ACTION_BUILD = "build"; @@ -32,11 +30,11 @@ public interface SliderActions { String ACTION_UPGRADE = "upgrade"; String ACTION_DESTROY = "destroy"; String ACTION_EXISTS = "exists"; + String ACTION_EXAMPLES = "examples"; String ACTION_FLEX = "flex"; String ACTION_STOP = "stop"; String ACTION_HELP = "help"; String ACTION_INSTALL_KEYTAB = "install-keytab"; - String ACTION_KDIAG = "kdiag"; String ACTION_KEYTAB = "keytab"; String ACTION_LIST = "list"; @@ -50,7 +48,7 @@ public interface SliderActions { String DESCRIBE_ACTION_BUILD = "Build a service specification, but do not start it"; String DESCRIBE_ACTION_CREATE = - "Build and start a service, it's equivalent to first invoke build and then start"; + "Create a service, it's equivalent to first invoke build and then start"; String DESCRIBE_ACTION_DEPENDENCY = "Yarn service framework dependency (libraries) management"; String DESCRIBE_ACTION_UPDATE = @@ -61,6 +59,7 @@ public interface SliderActions { "Destroy a stopped service, service must be stopped first before destroying."; String DESCRIBE_ACTION_EXISTS = "Probe for a service running"; + String DESCRIBE_ACTION_EXAMPLES = "Run an example service on YARN"; String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers."; String DESCRIBE_ACTION_FREEZE = "Stop a running service"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java index 9e7d8e8..cfe7c1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.ConfigFile; import org.apache.hadoop.yarn.service.api.records.Configuration; import org.apache.hadoop.yarn.service.api.records.Container; +import org.apache.hadoop.yarn.service.api.records.ServiceState; import org.apache.hadoop.yarn.service.component.instance.ComponentInstance; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.slf4j.Logger; @@ -96,8 +97,8 @@ public class ServiceTimelinePublisher extends CompositeService { // create info keys Map<String, Object> entityInfos = new HashMap<String, Object>(); entityInfos.put(ServiceTimelineMetricsConstants.NAME, service.getName()); -// entityInfos.put(ServiceTimelineMetricsConstants.STATE, -// service.getState().toString()); + entityInfos.put(ServiceTimelineMetricsConstants.STATE, + ServiceState.STARTED.toString()); entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_TIME, currentTimeMillis); entity.addInfo(ServiceTimelineMetricsConstants.QUICK_LINKS, http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java index 93a15cc..0498ebf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java @@ -36,7 +36,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF; +import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE; import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH; /** @@ -56,12 +56,12 @@ public class TestBuildExternalComponents { } } - // 1. Build the appDef and store on fs + // 1. Build the def file and store on fs // 2. check component names private void buildAndCheckComponents(String appName, String appDef, SliderFileSystem sfs, Set<String> names) throws Throwable { String[] args = - { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) }; + { "build", ARG_FILE, ExampleAppJson.resourceName(appDef) }; ClientArgs clientArgs = new ClientArgs(args); clientArgs.parse(); ServiceCLI cli = new ServiceCLI() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java index c53ee2b..f22d487 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java @@ -24,7 +24,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.service.ClientAMProtocol; import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.client.params.ClientArgs; @@ -43,7 +42,7 @@ import java.util.List; import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS; import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS; -import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF; +import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE; import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH; import static org.mockito.Mockito.*; @@ -54,9 +53,9 @@ public class TestServiceCLI { private ServiceCLI cli; private SliderFileSystem fs; - private void buildApp(String appName, String appDef) throws Throwable { + private void buildApp(String appDef) throws Throwable { String[] args = - { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) }; + { "build", ARG_FILE, ExampleAppJson.resourceName(appDef)}; ClientArgs clientArgs = new ClientArgs(args); clientArgs.parse(); cli.exec(clientArgs); @@ -115,34 +114,35 @@ public class TestServiceCLI { // Test flex components count are persisted. @Test public void testFlexComponents() throws Throwable { - buildApp("service-1", ExampleAppJson.APP_JSON); - - checkCompCount("master", 1L); + String serviceName = "app-1"; + buildApp(ExampleAppJson.APP_JSON); + checkCompCount("master",serviceName, 1L); // increase by 2 - String[] flexUpArgs = {"flex", "service-1", "--component", "master" , "+2"}; + String[] flexUpArgs = {"flex", serviceName, "--component", "master" , "+2"}; ClientArgs clientArgs = new ClientArgs(flexUpArgs); clientArgs.parse(); cli.exec(clientArgs); - checkCompCount("master", 3L); + checkCompCount("master", serviceName, 3L); // decrease by 1 - String[] flexDownArgs = {"flex", "service-1", "--component", "master", "-1"}; + String[] flexDownArgs = {"flex", serviceName, "--component", "master", "-1"}; clientArgs = new ClientArgs(flexDownArgs); clientArgs.parse(); cli.exec(clientArgs); - checkCompCount("master", 2L); + checkCompCount("master", serviceName, 2L); - String[] flexAbsoluteArgs = {"flex", "service-1", "--component", "master", "10"}; + String[] flexAbsoluteArgs = {"flex", serviceName, "--component", "master", "10"}; clientArgs = new ClientArgs(flexAbsoluteArgs); clientArgs.parse(); cli.exec(clientArgs); - checkCompCount("master", 10L); + checkCompCount("master", serviceName, 10L); } - private void checkCompCount(String compName, long count) throws IOException { + private void checkCompCount(String compName, String serviceName, long count) + throws IOException { List<Component> components = - ServiceApiUtil.getComponents(fs, "service-1"); + ServiceApiUtil.getComponents(fs, serviceName); for (Component component : components) { if (component.getName().equals(compName)) { Assert.assertEquals(count, component.getNumberOfContainers().longValue()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json index 12b51e4..43d3c39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/resources/org/apache/hadoop/yarn/service/conf/examples/app.json @@ -1,5 +1,5 @@ { - "name": "service-1", + "name": "app-1", "id" : "application_1503358878042_0011", "lifetime": "3600", "launch_command": "sleep 3600", http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java index e66a761..0006dfd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java @@ -95,7 +95,7 @@ public interface RegistryConstants { /** * Default DNS port number. */ - int DEFAULT_DNS_PORT = 53; + int DEFAULT_DNS_PORT = 5353; /** * DNSSEC Enabled? http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index fa33b8b..ca46c8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -22,7 +22,7 @@ Overview YARN commands are invoked by the bin/yarn script. Running the yarn script without any arguments prints the description for all commands. -Usage: `yarn [SHELL_OPTIONS] COMMAND [GENERIC_OPTIONS] [COMMAND_OPTIONS]` +Usage: `yarn [SHELL_OPTIONS] COMMAND [GENERIC_OPTIONS] [SUB_COMMAND] [COMMAND_OPTIONS]` YARN has an option parsing framework that employs parsing generic options as well as running classes. @@ -69,6 +69,56 @@ Usage: `yarn applicationattempt [options] ` prints applicationattempt(s) report +### `service` +Usage `yarn service [sub-command] [service-name] [options]` + +#### `SUB_COMMAND` + +* `build`: Build a service with its specifications, but do not start it. + ``` + Usage: yarn service build --file [file] + ``` + + | COMMAND\_OPTIONS | Description | + |:---- |:---- | + | --file or -f | The local path to the service definition file | + +* `create`: create a service, it's equivalent to first invoke build and then start. + ``` + Usage: yarn service create --file [file] + ``` + | COMMAND\_OPTIONS | Description | + |:---- |:---- | + | --file or -f | The local path to the service definition file | + +* `dependency`: Yarn service framework dependency (libraries) management. + ``` + Usage: yarn service dependency [options] + ``` + | COMMAND\_OPTIONS | Description | + |:---- |:---- | + | --upload | Pre-upload the dependency jars onto HDFS to expediate service launch process. | + +* `destroy`: Destroy a stopped service, service must be stopped first before destroying. + ``` + Usage: yarn service destroy [service-name] + ``` +* `flex`: Flex a service's component by increasing or decreasing the number of containers. + ``` + Usage: yarn service flex [service-name] --component [component-name] [count] + ``` + | COMMAND\_OPTIONS | Description | + |:---- |:---- | + | --component [component-name] [count] | Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3.| +* `status`: Get the status of a service. + ``` + Usage: yarn service status [service-name] + ``` +* `start`: Start a service with pre-built specification or a previously stopped service. + ``` + Usage: yarn service start [service-name] + ``` + ### `classpath` Usage: `yarn classpath [--glob |--jar <path> |-h |--help]` @@ -273,6 +323,10 @@ Usage: `yarn timelineserver` Start the TimeLineServer +### apiserver +Usage: `yarn apiserver` +Start the API-server for deploying/managing services on YARN + Files ----- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org