SLIDER-407. Add install package command to install app packages
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b4c88a4b Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b4c88a4b Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b4c88a4b Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: b4c88a4b30309fb78f976c783932a133d9fe0058 Parents: 8da1503 Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Mon Sep 8 06:36:04 2014 -0700 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Mon Sep 8 06:36:04 2014 -0700 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 60 +++++-- .../org/apache/slider/common/SliderKeys.java | 2 + .../common/params/AbstractActionArgs.java | 3 +- .../common/params/ActionInstallPackageArgs.java | 58 +++++++ .../apache/slider/common/params/Arguments.java | 1 + .../apache/slider/common/params/ClientArgs.java | 12 +- .../slider/common/params/SliderActions.java | 2 + .../slider/common/tools/CoreFileSystem.java | 12 ++ .../actions/TestActionInstallPackage.groovy | 158 +++++++++++++++++++ 9 files changed, 296 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index ef6448d..98074ec 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -48,6 +48,7 @@ import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.params.AbstractActionArgs; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; +import org.apache.slider.common.params.ActionInstallPackageArgs; import org.apache.slider.common.params.ActionAMSuicideArgs; import org.apache.slider.common.params.ActionCreateArgs; import org.apache.slider.common.params.ActionEchoArgs; @@ -301,20 +302,22 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe int exitCode = EXIT_SUCCESS; String clusterName = serviceArgs.getClusterName(); // actions - if (ACTION_BUILD.equals(action)) { + if (ACTION_INSTALL_PACKAGE.equals(action)) { + exitCode = actionInstallPkg(serviceArgs.getActionInstallPackageArgs()); + } else if (ACTION_BUILD.equals(action)) { exitCode = actionBuild(clusterName, serviceArgs.getActionBuildArgs()); } else if (ACTION_CREATE.equals(action)) { exitCode = actionCreate(clusterName, serviceArgs.getActionCreateArgs()); } else if (ACTION_FREEZE.equals(action)) { exitCode = actionFreeze(clusterName, - serviceArgs.getActionFreezeArgs()); + serviceArgs.getActionFreezeArgs()); } else if (ACTION_THAW.equals(action)) { exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs()); } else if (ACTION_DESTROY.equals(action)) { exitCode = actionDestroy(clusterName); } else if (ACTION_EXISTS.equals(action)) { exitCode = actionExists(clusterName, - serviceArgs.getActionExistsArgs().live); + serviceArgs.getActionExistsArgs().live); } else if (ACTION_FLEX.equals(action)) { exitCode = actionFlex(clusterName, serviceArgs.getActionFlexArgs()); } else if (ACTION_GETCONF.equals(action)) { @@ -322,15 +325,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } else if (ACTION_HELP.equals(action) || ACTION_USAGE.equals(action)) { log.info(serviceArgs.usage()); - } else if (ACTION_KILL_CONTAINER.equals(action)) { exitCode = actionKillContainer(clusterName, serviceArgs.getActionKillContainerArgs()); - } else if (ACTION_AM_SUICIDE.equals(action)) { exitCode = actionAmSuicide(clusterName, serviceArgs.getActionAMSuicideArgs()); - } else if (ACTION_LIST.equals(action)) { exitCode = actionList(clusterName); } else if (ACTION_REGISTRY.equals(action)) { @@ -592,6 +592,50 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } /** + * Upload application package to user home directory + * + * @param installPkgInfo the arguments needed to build the cluster + * @throws YarnException Yarn problems + * @throws IOException other problems + * @throws BadCommandArgumentsException bad arguments. + */ + public int actionInstallPkg(ActionInstallPackageArgs installPkgInfo) throws + YarnException, + IOException { + + Path srcFile = null; + if (null == installPkgInfo.name || installPkgInfo.name.length() == 0) { + throw new BadCommandArgumentsException("A valid application name is required."); + } + + if (null == installPkgInfo.packageURI || installPkgInfo.packageURI.length() == 0) { + throw new BadCommandArgumentsException("A valid application package is required."); + } else { + File pkgFile = new File(installPkgInfo.packageURI); + if (!pkgFile.exists() || pkgFile.isDirectory()) { + throw new BadCommandArgumentsException("Unable to access supplied pkg file at " + + pkgFile.getAbsolutePath()); + } else { + srcFile = new Path(pkgFile.toURI()); + } + } + + Path pkgPath = sliderFileSystem.buildPackageDirPath(installPkgInfo.name); + sliderFileSystem.getFileSystem().mkdirs(pkgPath); + + Path fileInFs = new Path(pkgPath, srcFile.getName()); + log.info("Installing package {} at {} and overwrite is {}.", srcFile, fileInFs, installPkgInfo.replacePkg); + if (sliderFileSystem.getFileSystem().exists(fileInFs) && !installPkgInfo.replacePkg) { + throw new BadCommandArgumentsException("Pkg exists at " + + fileInFs.toUri().toString() + + ". Use --replacePkg true to overwrite."); + } + + sliderFileSystem.getFileSystem().copyFromLocalFile(false, installPkgInfo.replacePkg, srcFile, fileInFs); + return EXIT_SUCCESS; + } + + /** * Update the cluster specification * * @param clustername cluster name @@ -1056,8 +1100,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe clusterPerms); - // add AM and provider specific artifacts to the resource map - Map<String, LocalResource> providerResources; // standard AM resources sliderAM.prepareAMAndConfigForLaunch(sliderFileSystem, config, @@ -1102,7 +1144,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe clusterDirectory, generatedConfDirPath, clusterSecure - ); + ); // now add the image if it was set http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 1d0a6f5..14528b1 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -112,6 +112,8 @@ public interface SliderKeys extends SliderXmlConfKeys { String CLUSTER_DIRECTORY = "cluster"; + String PACKAGE_DIRECTORY = "package"; + /** * JVM property to define the slider configuration directory; * this is set by the slider script: {@value} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java index d18c85b..41f6a3f 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java @@ -124,7 +124,8 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments { int actionArgSize = parameters.size(); if (minArgs > actionArgSize) { throw new BadCommandArgumentsException( - ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS + getActionName()); + ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS + getActionName() + + " Expected minimum " + minArgs + " but got " + actionArgSize); } int maxArgs = getMaxParams(); if (maxArgs == -1) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java new file mode 100644 index 0000000..646e795 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.beust.jcommander.ParametersDelegate; + +@Parameters(commandNames = {SliderActions.ACTION_INSTALL_PACKAGE}, + commandDescription = SliderActions.DESCRIBE_ACTION_INSTALL_PACKAGE) + +public class ActionInstallPackageArgs extends AbstractActionArgs { + + @Override + public String getActionName() { + return SliderActions.ACTION_INSTALL_PACKAGE; + } + + @Parameter(names = {ARG_PACKAGE}, + description = "Path to app package on local disk") + public String packageURI; + + @Parameter(names = {ARG_NAME}, + description = "The type of the package") + public String name; + + @Parameter(names = {ARG_REPLACE_PKG}, description = "Overwrite existing package") + public boolean replacePkg = false; + + /** + * Get the min #of params expected + * @return the min number of params in the {@link #parameters} field + */ + public int getMinParams() { + return 0; + } + + @Override + public int getMaxParams() { + return 1; + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java index 764ffca..7f8aa83 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java @@ -77,6 +77,7 @@ public interface Arguments { String ARG_ZKPATH = "--zkpath"; String ARG_ZKPORT = "--zkport"; String ARG_ZKHOSTS = "--zkhosts"; + String ARG_REPLACE_PKG = "--replacepkg"; /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java index ca854f1..7173a85 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java @@ -46,6 +46,7 @@ public class ClientArgs extends CommonArgs { private AbstractClusterBuildingActionArgs buildingActionArgs; private final ActionAMSuicideArgs actionAMSuicideArgs = new ActionAMSuicideArgs(); private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs(); + private final ActionInstallPackageArgs actionInstallPackageArgs = new ActionInstallPackageArgs(); private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs(); private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs(); private final ActionDestroyArgs actionDestroyArgs = new ActionDestroyArgs(); @@ -90,7 +91,8 @@ public class ClientArgs extends CommonArgs { actionStatusArgs, actionThawArgs, actionHelpArgs, - actionVersionArgs + actionVersionArgs, + actionInstallPackageArgs ); } @@ -122,6 +124,9 @@ public class ClientArgs extends CommonArgs { return actionBuildArgs; } + public ActionInstallPackageArgs getActionInstallPackageArgs() { + return actionInstallPackageArgs; } + public ActionUpdateArgs getActionUpdateArgs() { return actionUpdateArgs; } @@ -192,7 +197,10 @@ public class ClientArgs extends CommonArgs { } else if (SliderActions.ACTION_UPDATE.equals(action)) { bindCoreAction(actionUpdateArgs); - } else if (SliderActions.ACTION_FREEZE.equals(action)) { + } else if (SliderActions.ACTION_INSTALL_PACKAGE.equals(action)) { + bindCoreAction(actionInstallPackageArgs); + + }else if (SliderActions.ACTION_FREEZE.equals(action)) { bindCoreAction(actionFreezeArgs); } else if (SliderActions.ACTION_THAW.equals(action)) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java index ba2bca9..964f184 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java @@ -44,6 +44,7 @@ public interface SliderActions { String ACTION_THAW = "start"; String ACTION_USAGE = "usage"; String ACTION_VERSION = "version"; + String ACTION_INSTALL_PACKAGE = "install-package"; String DESCRIBE_ACTION_AM_SUICIDE = "Tell the Slider Application Master to simulate a process failure by terminating itself"; String DESCRIBE_ACTION_BUILD = @@ -76,4 +77,5 @@ public interface SliderActions { "Start a stopped application"; String DESCRIBE_ACTION_VERSION = "Print the Slider version information"; + String DESCRIBE_ACTION_INSTALL_PACKAGE = "Install the application package in the home directory under sub-folder packages"; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java index def252a..955d991 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java @@ -122,6 +122,18 @@ public class CoreFileSystem { } /** + * Build up the path string for package install location -no attempt to + * create the directory is made + * + * @return the path for persistent app package + */ + public Path buildPackageDirPath(String packageName) { + Preconditions.checkNotNull(packageName); + Path path = getBaseApplicationPath(); + return new Path(path, SliderKeys.PACKAGE_DIRECTORY + "/" + packageName); + } + + /** * Create the Slider cluster path for a named cluster and all its subdirs * This is a directory; a mkdirs() operation is executed * to ensure that it is there. http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy new file mode 100644 index 0000000..4d89e9b --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.agent.actions + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.hadoop.yarn.conf.YarnConfiguration +import org.apache.slider.agent.AgentMiniClusterTestBase +import org.apache.slider.common.params.Arguments +import org.apache.slider.common.params.SliderActions +import org.apache.slider.core.exceptions.BadCommandArgumentsException +import org.apache.slider.core.main.ServiceLauncher +import org.junit.Before +import org.junit.Test + +/** + * existence tests + */ +@CompileStatic +@Slf4j + +class TestActionInstallPackage extends AgentMiniClusterTestBase { + + @Before + public void setup() { + super.setup() + createMiniCluster("", configuration, 1, false) + } + + @Test + public void testInstallPackageFailsWithNoPackageName() throws Throwable { + try { + ServiceLauncher launcher = launchClientAgainstMiniMR( + //config includes RM binding info + new YarnConfiguration(miniCluster.config), + //varargs list of command line params + [ + SliderActions.ACTION_INSTALL_PACKAGE + ], + ) + fail("expected an exception, got a status code " + launcher.serviceExitCode) + } catch (BadCommandArgumentsException e) { + assert e.message.contains("A valid application name is required") + } + } + + @Test + public void testInstallPackageFailsWithNoPackagePath() throws Throwable { + try { + ServiceLauncher launcher = launchClientAgainstMiniMR( + //config includes RM binding info + new YarnConfiguration(miniCluster.config), + //varargs list of command line params + [ + SliderActions.ACTION_INSTALL_PACKAGE, + Arguments.ARG_NAME, "hbase" + ], + ) + fail("expected an exception, got a status code " + launcher.serviceExitCode) + } catch (BadCommandArgumentsException e) { + assert e.message.contains("A valid application package is required.") + } + } + + @Test + public void testInstallPackageFailsWithInvalidPackagePath() throws Throwable { + try { + ServiceLauncher launcher = launchClientAgainstMiniMR( + //config includes RM binding info + new YarnConfiguration(miniCluster.config), + //varargs list of command line params + [ + SliderActions.ACTION_INSTALL_PACKAGE, + Arguments.ARG_NAME, "hbase", + Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties", + ], + ) + launcher = launchClientAgainstMiniMR( + //config includes RM binding info + new YarnConfiguration(miniCluster.config), + //varargs list of command line params + [ + SliderActions.ACTION_INSTALL_PACKAGE, + Arguments.ARG_NAME, "hbase", + Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties", + ], + ) + fail("expected an exception, got a status code " + launcher.serviceExitCode) + } catch (BadCommandArgumentsException e) { + assert e.message.contains("Use --replacePkg true to overwrite") + } + } + + @Test + public void testInstallPackageFailsWithNeedingReplaceFlag() throws Throwable { + try { + ServiceLauncher launcher = launchClientAgainstMiniMR( + //config includes RM binding info + new YarnConfiguration(miniCluster.config), + //varargs list of command line params + [ + SliderActions.ACTION_INSTALL_PACKAGE, + Arguments.ARG_NAME, "hbase", + Arguments.ARG_PACKAGE, "unlikely_to_be_a_file_path", + ], + ) + fail("expected an exception, got a status code " + launcher.serviceExitCode) + } catch (BadCommandArgumentsException e) { + assert e.message.contains("Unable to access supplied pkg file at") + } + } + + @Test + public void testInstallPackageWithReplace() throws Throwable { + try { + ServiceLauncher launcher = launchClientAgainstMiniMR( + //config includes RM binding info + new YarnConfiguration(miniCluster.config), + //varargs list of command line params + [ + SliderActions.ACTION_INSTALL_PACKAGE, + Arguments.ARG_NAME, "hbase", + Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties", + ], + ) + launcher = launchClientAgainstMiniMR( + //config includes RM binding info + new YarnConfiguration(miniCluster.config), + //varargs list of command line params + [ + SliderActions.ACTION_INSTALL_PACKAGE, + Arguments.ARG_NAME, "hbase", + Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties", + Arguments.ARG_REPLACE_PKG, true + ], + ) + } catch (BadCommandArgumentsException e) { + log.info(e.message) + assert e.message.contains("Use --replacePkg true to overwrite") + } + } +}
