http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java index 64783b6..235d3da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java @@ -16,7 +16,7 @@ */ package org.apache.slider.common.tools; -import org.apache.slider.common.SliderExitCodes; +import org.apache.hadoop.yarn.service.conf.SliderExitCodes; import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.core.exceptions.SliderException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 2e1236d..fc57c82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -33,7 +33,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.registry.client.api.RegistryConstants; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ExitUtil; @@ -50,10 +49,10 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.slider.Slider; import org.apache.slider.api.RoleKeys; import org.apache.slider.api.types.ContainerInformation; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.SliderXmlConfKeys; -import org.apache.slider.common.params.Arguments; -import org.apache.slider.common.params.SliderActions; +import org.apache.hadoop.yarn.service.conf.SliderKeys; +import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys; +import org.apache.hadoop.yarn.service.client.params.Arguments; +import org.apache.hadoop.yarn.service.client.params.SliderActions; import org.apache.slider.core.exceptions.BadClusterStateException; import org.apache.slider.core.exceptions.BadCommandArgumentsException; import org.apache.slider.core.exceptions.BadConfigException; @@ -111,7 +110,7 @@ import java.util.zip.GZIPOutputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR; +import static org.apache.hadoop.yarn.service.conf.SliderKeys.COMPONENT_SEPARATOR; /** * These are slider-specific Util methods @@ -373,6 +372,13 @@ public final class SliderUtils { private static final PatternValidator clusternamePattern = new PatternValidator("[a-z][a-z0-9_-]*"); + private static final PatternValidator compNamePattern + = new PatternValidator("[a-z][a-z0-9-]*"); + + public static void validateCompName(String compName) { + compNamePattern.validate(compName); + } + /** * Normalize a cluster name then verify that it is valid * @param name proposed cluster name http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java index 7f3134a..1430c5a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java @@ -18,7 +18,7 @@ package org.apache.slider.core.exceptions; -import org.apache.slider.common.SliderExitCodes; +import org.apache.hadoop.yarn.service.conf.SliderExitCodes; import org.apache.slider.core.main.ServiceLaunchException; public class SliderException extends ServiceLaunchException implements http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java index 8f61bf4..55ffbf7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java @@ -19,30 +19,17 @@ package org.apache.slider.core.launch; import com.google.common.base.Preconditions; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LogAggregationContext; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.util.Records; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.api.RoleKeys; -import org.apache.slider.common.SliderKeys; +import org.apache.hadoop.yarn.service.conf.SliderKeys; import org.apache.slider.common.tools.CoreFileSystem; import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.MapOperations; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.reflect.Method; -import java.lang.reflect.InvocationTargetException; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -51,12 +38,12 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import static org.apache.slider.providers.docker.DockerKeys.DEFAULT_DOCKER_NETWORK; +import static org.apache.hadoop.yarn.service.provider.docker.DockerKeys.DEFAULT_DOCKER_NETWORK; /** * Launcher of applications: base class */ -public abstract class AbstractLauncher extends Configured { +public class AbstractLauncher { private static final Logger log = LoggerFactory.getLogger(AbstractLauncher.class); public static final String CLASSPATH = "CLASSPATH"; @@ -68,7 +55,6 @@ public abstract class AbstractLauncher extends Configured { * Env vars; set up at final launch stage */ protected final Map<String, String> envVars = new HashMap<>(); - protected final MapOperations env = new MapOperations("env", envVars); protected final ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class); protected final List<String> commands = new ArrayList<>(20); @@ -77,36 +63,24 @@ public abstract class AbstractLauncher extends Configured { private final Map<String, ByteBuffer> serviceData = new HashMap<>(); // security protected final Credentials credentials; - protected LogAggregationContext logAggregationContext; protected boolean yarnDockerMode = false; protected String dockerImage; protected String dockerNetwork = DEFAULT_DOCKER_NETWORK; protected String dockerHostname; - protected String yarnContainerMountPoints; protected String runPrivilegedContainer; /** * Create instance. - * @param conf configuration * @param coreFileSystem filesystem * @param credentials initial set of credentials -null is permitted */ - protected AbstractLauncher(Configuration conf, + public AbstractLauncher( CoreFileSystem coreFileSystem, Credentials credentials) { - super(conf); this.coreFileSystem = coreFileSystem; this.credentials = credentials != null ? credentials: new Credentials(); } - - /** - * Get the container. Until "completed", this isn't valid to launch. - * @return the container to launch - */ - public ContainerLaunchContext getContainerLaunchContext() { - return containerLaunchContext; - } public void setYarnDockerMode(boolean yarnDockerMode){ this.yarnDockerMode = yarnDockerMode; @@ -116,8 +90,8 @@ public abstract class AbstractLauncher extends Configured { * Get the env vars to work on * @return env vars */ - public MapOperations getEnv() { - return env; + public Map<String, String> getEnv() { + return envVars; } /** @@ -128,14 +102,6 @@ public abstract class AbstractLauncher extends Configured { return commands; } - /** - * Get the map of local resources. - * @return the live map of local resources. - */ - public Map<String, LocalResource> getLocalResources() { - return localResources; - } - public void addLocalResource(String subPath, LocalResource resource) { localResources.put(subPath, resource); } @@ -146,18 +112,6 @@ public abstract class AbstractLauncher extends Configured { } /** - * Add a set of local resources - * @param resourceMap map of name:resource to add - */ - public void addLocalResources(Map<String, LocalResource> resourceMap) { - localResources.putAll(resourceMap); - } - - public Map<String, ByteBuffer> getServiceData() { - return serviceData; - } - - /** * Accessor to the credentials * @return the credentials associated with this launcher */ @@ -165,36 +119,12 @@ public abstract class AbstractLauncher extends Configured { return credentials; } - /** - * Add a command line. It is converted to a single command before being - * added. - * @param cmd - */ - public void addCommandLine(CommandLineBuilder cmd) { - commands.add(cmd.build()); - } public void addCommand(String cmd) { commands.add(cmd); } /** - * Add a list of commands. Each element in the list becomes a single command - * @param commandList list of commands - */ - public void addCommands(List<String> commandList) { - commands.addAll(commandList); - } - - /** - * Get all commands as a string, separated by ";". This is for diagnostics - * @return a string description of the commands - */ - public String getCommandsAsString() { - return SliderUtils.join(getCommands(), "; "); - } - - /** * Complete the launch context (copy in env vars, etc). * @return the container to launch */ @@ -211,7 +141,7 @@ public abstract class AbstractLauncher extends Configured { log.debug(" \"{}\"=\"{}\"", envPair.getKey(), envPair.getValue()); } } - containerLaunchContext.setEnvironment(env); + containerLaunchContext.setEnvironment(envVars); //service data if (log.isDebugEnabled()) { @@ -281,120 +211,7 @@ public abstract class AbstractLauncher extends Configured { protected void propagateUsernameInInsecureCluster() throws IOException { //insecure cluster: propagate user name via env variable String userName = UserGroupInformation.getCurrentUser().getUserName(); - env.put(SliderKeys.HADOOP_USER_NAME, userName); - } - - /** - * Extract any resource requirements from this component's settings. - * All fields that are set will override the existing values -if - * unset that resource field will be left unchanged. - * - * Important: the configuration must already be fully resolved - * in order to pick up global options. - * @param resource resource to configure - * @param map map of options - */ - public void extractResourceRequirements(Resource resource, - Map<String, String> map) { - - if (map != null) { - MapOperations options = new MapOperations("", map); - resource.setMemory(options.getOptionInt(ResourceKeys.YARN_MEMORY, - resource.getMemory())); - resource.setVirtualCores(options.getOptionInt(ResourceKeys.YARN_CORES, - resource.getVirtualCores())); - } - } - - public void extractLogAggregationContext(Map<String, String> map) { - if (map != null) { - String logPatternSepStr = "\\|"; - String logPatternJoinStr = "|"; - MapOperations options = new MapOperations("", map); - - List<String> logIncludePatterns = new ArrayList<>(); - String includePatternExpression = options.getOption( - ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, "").trim(); - if (!includePatternExpression.isEmpty()) { - String[] includePatterns = includePatternExpression - .split(logPatternSepStr); - for (String includePattern : includePatterns) { - String trimmedIncludePattern = includePattern.trim(); - if (!trimmedIncludePattern.isEmpty()) { - logIncludePatterns.add(trimmedIncludePattern); - } - } - } - String logIncludePattern = StringUtils.join(logIncludePatterns, - logPatternJoinStr); - log.info("Log include patterns: {}", logIncludePattern); - - List<String> logExcludePatterns = new ArrayList<>(); - String excludePatternExpression = options.getOption( - ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, "").trim(); - if (!excludePatternExpression.isEmpty()) { - String[] excludePatterns = excludePatternExpression - .split(logPatternSepStr); - for (String excludePattern : excludePatterns) { - String trimmedExcludePattern = excludePattern.trim(); - if (!trimmedExcludePattern.isEmpty()) { - logExcludePatterns.add(trimmedExcludePattern); - } - } - } - String logExcludePattern = StringUtils.join(logExcludePatterns, - logPatternJoinStr); - log.info("Log exclude patterns: {}", logExcludePattern); - - // SLIDER-810/YARN-3154 - hadoop 2.7.0 onwards a new instance method has - // been added for log aggregation for LRS. Existing newInstance method's - // behavior has changed and is used for log aggregation only after the - // application has finished. This forces Slider users to move to hadoop - // 2.7.0+ just for log aggregation, which is not very desirable. So we - // decided to use reflection here to find out if the new 2.7.0 newInstance - // method is available. If yes, then we use it, so log aggregation will - // work in hadoop 2.7.0+ env. If no, then we fallback to the pre-2.7.0 - // newInstance method, which means log aggregation will work as expected - // in hadoop 2.6 as well. - // TODO: At some point, say 2-3 Slider releases down, when most users are - // running hadoop 2.7.0, we should get rid of the reflection code here. - try { - Method logAggregationContextMethod = LogAggregationContext.class - .getMethod("newInstance", String.class, String.class, String.class, - String.class); - // Need to set include/exclude patterns appropriately since by default - // rolled log aggregation is not done for any files, so defaults are - // - include pattern set to "" - // - exclude pattern set to "*" - // For Slider we want all logs to be uploaded if include/exclude - // patterns are left empty by the app owner in resources file - if (StringUtils.isEmpty(logIncludePattern) - && StringUtils.isEmpty(logExcludePattern)) { - logIncludePattern = ".*"; - logExcludePattern = ""; - } else if (StringUtils.isEmpty(logIncludePattern) - && StringUtils.isNotEmpty(logExcludePattern)) { - logIncludePattern = ".*"; - } else if (StringUtils.isNotEmpty(logIncludePattern) - && StringUtils.isEmpty(logExcludePattern)) { - logExcludePattern = ""; - } - log.debug("LogAggregationContext newInstance method for rolled logs " - + "include/exclude patterns is available"); - log.info("Modified log include patterns: {}", logIncludePattern); - log.info("Modified log exclude patterns: {}", logExcludePattern); - logAggregationContext = (LogAggregationContext) logAggregationContextMethod - .invoke(null, null, null, logIncludePattern, logExcludePattern); - } catch (NoSuchMethodException | SecurityException - | IllegalAccessException | IllegalArgumentException - | InvocationTargetException e) { - log.debug("LogAggregationContext newInstance method for rolled logs " - + "include/exclude patterns is not available - fallback to old one"); - log.debug(e.toString()); - logAggregationContext = LogAggregationContext.newInstance( - logIncludePattern, logExcludePattern); - } - } + envVars.put(SliderKeys.HADOOP_USER_NAME, userName); } /** @@ -413,57 +230,15 @@ public abstract class AbstractLauncher extends Configured { public void setEnv(String var, String value) { Preconditions.checkArgument(var != null, "null variable name"); Preconditions.checkArgument(value != null, "null value"); - env.put(var, value); + envVars.put(var, value); } - /** - * Set an environment variable if its value is non-null. - * @param var variable name - * @param value value (may be null) - */ - public void maybeSetEnv(String var, String value) { - if (value != null) { - setEnv(var, value); - } - } public void putEnv(Map<String, String> map) { - env.putAll(map); + envVars.putAll(map); } - public String[] dumpEnvToString() { - - List<String> nodeEnv = new ArrayList<>(); - - for (Map.Entry<String, String> entry : env.entrySet()) { - String envElt = String.format("%s=\"%s\"", - entry.getKey(), - entry.getValue()); - log.debug(envElt); - nodeEnv.add(envElt); - } - String[] envDescription = nodeEnv.toArray(new String[nodeEnv.size()]); - - return envDescription; - } - - /** - * Submit an entire directory - * @param srcDir src path in filesystem - * @param destRelativeDir relative path under destination local dir - * @throws IOException IO problems - */ - public void submitDirectory(Path srcDir, String destRelativeDir) - throws IOException { - //add the configuration resources - Map<String, LocalResource> confResources; - confResources = coreFileSystem.submitDirectory( - srcDir, - destRelativeDir); - addLocalResources(confResources); - } - public void setDockerImage(String dockerImage) { this.dockerImage = dockerImage; } @@ -476,14 +251,6 @@ public abstract class AbstractLauncher extends Configured { this.dockerHostname = dockerHostname; } - public void setYarnContainerMountPoints(String yarnContainerMountPoints) { - this.yarnContainerMountPoints = yarnContainerMountPoints; - } - - public void setRunPrivilegedContainer(String runPrivilegedContainer) { - this.runPrivilegedContainer = runPrivilegedContainer; - } - public void setRunPrivilegedContainer(boolean runPrivilegedContainer) { if (runPrivilegedContainer) { this.runPrivilegedContainer = Boolean.toString(true); http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java index d220928..7e164e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java @@ -38,35 +38,11 @@ import java.net.InetSocketAddress; public class ContainerLauncher extends AbstractLauncher { private static final Logger log = LoggerFactory.getLogger(ContainerLauncher.class); - // Allocated container - public final Container container; public ContainerLauncher(Configuration conf, CoreFileSystem coreFileSystem, Container container, Credentials credentials) { - super(conf, coreFileSystem, credentials); - this.container = container; + super(coreFileSystem, credentials); } - - /** - * This code is in the dist shell examples -it's been moved here - * so that if it is needed, it's still here - * @return a remote user with a token to access the container. - */ - public UserGroupInformation setupUGI() { - UserGroupInformation user = - UserGroupInformation.createRemoteUser(container.getId().toString()); - String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort(); - final InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr); - - org.apache.hadoop.yarn.api.records.Token containerToken = container.getContainerToken(); - if (containerToken != null) { - Token<ContainerTokenIdentifier> token = - ConverterUtils.convertFromYarn(containerToken, cmAddress); - user.addToken(token); - } - return user; - } - } http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java index 5357cc4..1fd49ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java @@ -37,7 +37,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; import org.apache.hadoop.yarn.util.ConverterUtils; -import org.apache.slider.common.SliderXmlConfKeys; +import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java index 37b36ea..ac8fca5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java @@ -20,7 +20,7 @@ package org.apache.slider.core.registry; import com.google.common.base.Preconditions; import org.apache.hadoop.registry.client.binding.RegistryUtils; -import org.apache.slider.common.SliderKeys; +import org.apache.hadoop.yarn.service.conf.SliderKeys; /** * Miscellaneous methods to assist slider registry work http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java deleted file mode 100644 index ea92ff7..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.slider.api.resource.Artifact; -import org.apache.slider.api.resource.ConfigFile; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.SliderException; -import org.codehaus.jettison.json.JSONObject; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Paths; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -public abstract class AbstractClientProvider { - - public AbstractClientProvider() { - } - - /** - * Generates a fixed format of application tags given one or more of - * application name, version and description. This allows subsequent query for - * an application with a name only, version only or description only or any - * combination of those as filters. - * - * @param appName name of the application - * @param appVersion version of the application - * @param appDescription brief description of the application - * @return - */ - public static final Set<String> createApplicationTags(String appName, - String appVersion, String appDescription) { - Set<String> tags = new HashSet<>(); - tags.add(SliderUtils.createNameTag(appName)); - if (appVersion != null) { - tags.add(SliderUtils.createVersionTag(appVersion)); - } - if (appDescription != null) { - tags.add(SliderUtils.createDescriptionTag(appDescription)); - } - return tags; - } - - /** - * Validate the artifact. - * @param artifact - */ - public abstract void validateArtifact(Artifact artifact, FileSystem - fileSystem) throws IOException; - - protected abstract void validateConfigFile(ConfigFile configFile, FileSystem - fileSystem) throws IOException; - - /** - * Validate the config files. - * @param configFiles config file list - * @param fs file system - */ - public void validateConfigFiles(List<ConfigFile> configFiles, - FileSystem fs) throws IOException { - Set<String> destFileSet = new HashSet<>(); - - for (ConfigFile file : configFiles) { - if (file.getType() == null) { - throw new IllegalArgumentException("File type is empty"); - } - - if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils - .isEmpty(file.getSrcFile())) { - throw new IllegalArgumentException( - "Src_file is empty for " + ConfigFile.TypeEnum.TEMPLATE); - - } - if (!StringUtils.isEmpty(file.getSrcFile())) { - Path p = new Path(file.getSrcFile()); - if (!fs.exists(p)) { - throw new IllegalArgumentException( - "Src_file does not exist for config file: " + file - .getSrcFile()); - } - } - - if (StringUtils.isEmpty(file.getDestFile())) { - throw new IllegalArgumentException("Dest_file is empty."); - } - - if (destFileSet.contains(file.getDestFile())) { - throw new IllegalArgumentException( - "Duplicated ConfigFile exists: " + file.getDestFile()); - } - destFileSet.add(file.getDestFile()); - - java.nio.file.Path destPath = Paths.get(file.getDestFile()); - if (!destPath.isAbsolute() && destPath.getNameCount() > 1) { - throw new IllegalArgumentException("Non-absolute dest_file has more " + - "than one path element"); - } - - // provider-specific validation - validateConfigFile(file, fs); - } - } - - /** - * Process client operations for applications such as install, configure. - * @param fileSystem - * @param registryOperations - * @param configuration - * @param operation - * @param clientInstallPath - * @param clientPackage - * @param clientConfig - * @param name - * @throws SliderException - */ - public void processClientOperation(SliderFileSystem fileSystem, - RegistryOperations registryOperations, - Configuration configuration, - String operation, - File clientInstallPath, - File clientPackage, - JSONObject clientConfig, - String name) - throws SliderException { - throw new SliderException("Provider does not support client operations."); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java index e0d9402..1e1b1b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java @@ -23,10 +23,13 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.service.conf.SliderKeys; +import org.apache.hadoop.yarn.service.provider.ProviderService; +import org.apache.hadoop.yarn.service.provider.ProviderUtils; +import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher; import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.Component; import org.apache.slider.api.resource.ContainerState; -import org.apache.slider.common.SliderKeys; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.exceptions.SliderException; @@ -35,7 +38,6 @@ import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.core.registry.docstore.PublishedConfiguration; import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher; import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,14 +46,14 @@ import java.io.IOException; import java.util.Map; import java.util.Map.Entry; -import static org.apache.slider.util.ServiceApiUtil.$; +import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$; public abstract class AbstractProviderService extends AbstractService implements ProviderService, SliderKeys { protected static final Logger log = LoggerFactory.getLogger(AbstractProviderService.class); - private static final ProviderUtils providerUtils = new ProviderUtils(log); + private static final ProviderUtils providerUtils = new ProviderUtils(); protected StateAccessForProviders amState; protected YarnRegistryViewForProviders yarnRegistry; private ServiceTimelinePublisher serviceTimelinePublisher; @@ -64,15 +66,6 @@ public abstract class AbstractProviderService extends AbstractService Application application, RoleInstance roleInstance, SliderFileSystem fileSystem) throws IOException; - @Override - public void setAMState(StateAccessForProviders stateAccessor) { - this.amState = stateAccessor; - } - - @Override - public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) { - this.yarnRegistry = yarnRegistry; - } public void buildContainerLaunchContext(ContainerLauncher launcher, Application application, Container container, ProviderRole providerRole, @@ -85,7 +78,7 @@ public abstract class AbstractProviderService extends AbstractService // Get pre-defined tokens Map<String, String> globalTokens = amState.getGlobalSubstitutionTokens(); Map<String, String> tokensForSubstitution = providerUtils - .initCompTokensForSubstitute(roleInstance); + .initCompTokensForSubstitute(null); tokensForSubstitution.putAll(globalTokens); // Set the environment variables in launcher launcher.putEnv(SliderUtils @@ -105,8 +98,6 @@ public abstract class AbstractProviderService extends AbstractService providerUtils.addComponentHostTokens(tokensForSubstitution, amState); // create config file on hdfs and add local resource - providerUtils.createConfigFileAndAddLocalResource(launcher, fileSystem, - component, tokensForSubstitution, roleInstance, amState); // substitute launch command String launchCommand = ProviderUtils @@ -141,15 +132,6 @@ public abstract class AbstractProviderService extends AbstractService return false; } - try { - providerUtils.updateServiceRecord(amState, yarnRegistry, - containerId.toString(), instance.role, status.getIPs(), status.getHost()); - } catch (IOException e) { - // could not write service record to ZK, log and retry - log.warn("Error updating container {} service record in registry, " + - "retrying", containerId, e); - return true; - } // TODO publish ip and host org.apache.slider.api.resource.Container container = instance.providerRole.component.getContainer(containerId.toString()); @@ -163,8 +145,4 @@ public abstract class AbstractProviderService extends AbstractService return false; } - @Override - public void setServiceTimelinePublisher(ServiceTimelinePublisher publisher) { - this.serviceTimelinePublisher = publisher; - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java deleted file mode 100644 index 004e81f..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.slider.api.resource.Artifact; -import org.apache.slider.api.resource.ConfigFile; - -import java.io.IOException; -import java.nio.file.Paths; - -public class DefaultClientProvider extends AbstractClientProvider { - - public DefaultClientProvider() { - } - - @Override - public void validateArtifact(Artifact artifact, FileSystem fileSystem) { - } - - @Override - protected void validateConfigFile(ConfigFile configFile, FileSystem - fileSystem) throws IOException { - // validate dest_file is not absolute - if (Paths.get(configFile.getDestFile()).isAbsolute()) { - throw new IllegalArgumentException( - "Dest_file must not be absolute path: " + configFile.getDestFile()); - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java deleted file mode 100644 index 09a1423..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers; - -public final class DefaultProviderFactory extends SliderProviderFactory { - private static final SliderProviderFactory FACTORY = new - DefaultProviderFactory(); - - private DefaultProviderFactory() {} - - private static class Client { - static final AbstractClientProvider PROVIDER = new DefaultClientProvider(); - } - - private static class Server { - static final ProviderService PROVIDER = new DefaultProviderService(); - } - - @Override - public AbstractClientProvider createClientProvider() { - return Client.PROVIDER; - } - - @Override - public ProviderService createServerProvider() { - return Server.PROVIDER; - } - - public static SliderProviderFactory getInstance() { - return FACTORY; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java deleted file mode 100644 index 8d2725c..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers; - -import org.apache.slider.api.resource.Application; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.core.launch.ContainerLauncher; -import org.apache.slider.server.appmaster.state.RoleInstance; - -import java.io.IOException; - -public class DefaultProviderService extends AbstractProviderService { - - protected DefaultProviderService() { - super(DefaultProviderService.class.getSimpleName()); - } - - @Override - public void processArtifact(ContainerLauncher launcher, Application - application, RoleInstance roleInstance, SliderFileSystem fileSystem) - throws IOException { - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java deleted file mode 100644 index edd313b..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers; - -import org.apache.hadoop.service.Service; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.slider.api.resource.Application; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.launch.ContainerLauncher; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher; -import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; - -import java.io.IOException; - -public interface ProviderService extends Service { - - /** - * Set up the entire container launch context - */ - void buildContainerLaunchContext(ContainerLauncher containerLauncher, - Application application, Container container, ProviderRole providerRole, - SliderFileSystem sliderFileSystem, RoleInstance roleInstance) - throws IOException, SliderException; - - - void setAMState(StateAccessForProviders stateAccessForProviders); - - /** - * Bind to the YARN registry - * @param yarnRegistry YARN registry - */ - void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry); - - /** - * Process container status - * @return true if status needs to be requested again, false otherwise - */ - boolean processContainerStatus(ContainerId containerId, - ContainerStatus status); - - /** - * Set service publisher. - * @param serviceTimelinePublisher service publisher. - */ - void setServiceTimelinePublisher( - ServiceTimelinePublisher serviceTimelinePublisher); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java deleted file mode 100644 index ecc521f..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java +++ /dev/null @@ -1,525 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers; - -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.registry.client.binding.RegistryPathUtils; -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; -import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.slider.api.ClusterNode; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.api.RoleKeys; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.Component; -import org.apache.slider.api.resource.ConfigFile; -import org.apache.slider.api.resource.Configuration; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.SliderXmlConfKeys; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.NoSuchNodeException; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.launch.ContainerLauncher; -import org.apache.slider.core.registry.docstore.ConfigFormat; -import org.apache.slider.core.registry.docstore.PublishedConfiguration; -import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; -import org.slf4j.Logger; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.regex.Pattern; - -import static org.apache.slider.api.ServiceApiConstants.*; -import static org.apache.slider.util.ServiceApiUtil.$; - -/** - * This is a factoring out of methods handy for providers. It's bonded to a log - * at construction time. - */ -public class ProviderUtils implements RoleKeys, SliderKeys { - - protected final Logger log; - - /** - * Create an instance - * @param log log directory to use -usually the provider - */ - - public ProviderUtils(Logger log) { - this.log = log; - } - - /** - * Add oneself to the classpath. This does not work - * on minicluster test runs where the JAR is not built up. - * @param providerResources map of provider resources to add these entries to - * @param providerClass provider to add - * @param jarName name of the jar to use - * @param sliderFileSystem target filesystem - * @param tempPath path in the cluster FS for temp files - * @param libdir relative directory to place resources - * @param miniClusterTestRun true if minicluster is being used - * @return true if the class was found in a JAR - * - * @throws FileNotFoundException if the JAR was not found and this is NOT - * a mini cluster test run - * @throws IOException IO problems - * @throws SliderException any Slider problem - */ - public static boolean addProviderJar( - Map<String, LocalResource> providerResources, - Class providerClass, - String jarName, - SliderFileSystem sliderFileSystem, - Path tempPath, - String libdir, - boolean miniClusterTestRun) throws - IOException, - SliderException { - try { - SliderUtils.putJar(providerResources, - sliderFileSystem, - providerClass, - tempPath, - libdir, - jarName); - return true; - } catch (FileNotFoundException e) { - if (miniClusterTestRun) { - return false; - } else { - throw e; - } - } - } - - /** - * Loads all dependency jars from the default path. - * @param providerResources map of provider resources to add these entries to - * @param sliderFileSystem target filesystem - * @param tempPath path in the cluster FS for temp files - * @param libDir relative directory to place resources - * @param libLocalSrcDir explicitly supplied local libs dir - * @throws IOException trouble copying to HDFS - * @throws SliderException trouble copying to HDFS - */ - public static void addAllDependencyJars( - Map<String, LocalResource> providerResources, - SliderFileSystem sliderFileSystem, - Path tempPath, - String libDir, - String libLocalSrcDir) - throws IOException, SliderException { - if (SliderUtils.isSet(libLocalSrcDir)) { - File file = new File(libLocalSrcDir); - if (!file.exists() || !file.isDirectory()) { - throw new BadCommandArgumentsException( - "Supplied lib src dir %s is not valid", libLocalSrcDir); - } - } - SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath, - libDir, libLocalSrcDir); - } - - public static String substituteStrWithTokens(String content, - Map<String, String> tokensForSubstitution) { - for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) { - content = - content.replaceAll(Pattern.quote(token.getKey()), token.getValue()); - } - return content; - } - - // configs will be substituted by corresponding env in tokenMap - public void substituteMapWithTokens(Map<String, String> configs, - Map<String, String> tokenMap) { - for (Map.Entry<String, String> entry : configs.entrySet()) { - String value = entry.getValue(); - if (tokenMap != null) { - for (Map.Entry<String, String> token : tokenMap.entrySet()) { - value = - value.replaceAll(Pattern.quote(token.getKey()), token.getValue()); - } - } - entry.setValue(value); - } - } - - /** - * Get resource requirements from a String value. If value isn't specified, - * use the default value. If value is greater than max, use the max value. - * @param val string value - * @param defVal default value - * @param maxVal maximum value - * @return int resource requirement - */ - public int getRoleResourceRequirement(String val, - int defVal, - int maxVal) { - if (val==null) { - val = Integer.toString(defVal); - } - Integer intVal; - if (ResourceKeys.YARN_RESOURCE_MAX.equals(val)) { - intVal = maxVal; - } else { - intVal = Integer.decode(val); - } - return intVal; - } - - - /** - * Localize the service keytabs for the application. - * @param launcher container launcher - * @param fileSystem file system - * @throws IOException trouble uploading to HDFS - */ - public void localizeServiceKeytabs(ContainerLauncher launcher, - SliderFileSystem fileSystem, Application application) throws IOException { - - Configuration conf = application.getConfiguration(); - String keytabPathOnHost = - conf.getProperty(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); - if (SliderUtils.isUnset(keytabPathOnHost)) { - String amKeytabName = - conf.getProperty(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); - String keytabDir = - conf.getProperty(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR); - // we need to localize the keytab files in the directory - Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null, - application.getName()); - boolean serviceKeytabsDeployed = false; - if (fileSystem.getFileSystem().exists(keytabDirPath)) { - FileStatus[] keytabs = fileSystem.getFileSystem().listStatus( - keytabDirPath); - LocalResource keytabRes; - for (FileStatus keytab : keytabs) { - if (!amKeytabName.equals(keytab.getPath().getName()) - && keytab.getPath().getName().endsWith(".keytab")) { - serviceKeytabsDeployed = true; - log.info("Localizing keytab {}", keytab.getPath().getName()); - keytabRes = fileSystem.createAmResource(keytab.getPath(), - LocalResourceType.FILE); - launcher.addLocalResource(KEYTAB_DIR + "/" + - keytab.getPath().getName(), - keytabRes); - } - } - } - if (!serviceKeytabsDeployed) { - log.warn("No service keytabs for the application have been localized. " - + "If the application requires keytabs for secure operation, " - + "please ensure that the required keytabs have been uploaded " - + "to the folder {}", keytabDirPath); - } - } - } - - // 1. Create all config files for a component on hdfs for localization - // 2. Add the config file to localResource - public synchronized void createConfigFileAndAddLocalResource( - ContainerLauncher launcher, SliderFileSystem fs, Component component, - Map<String, String> tokensForSubstitution, RoleInstance roleInstance, - StateAccessForProviders appState) throws IOException { - Path compDir = - new Path(new Path(fs.getAppDir(), "components"), component.getName()); - Path compInstanceDir = - new Path(compDir, roleInstance.getCompInstanceName()); - if (!fs.getFileSystem().exists(compInstanceDir)) { - fs.getFileSystem().mkdirs(compInstanceDir, - new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE)); - roleInstance.compInstanceDir = compInstanceDir; - log.info("Creating component instance dir: " + compInstanceDir); - } else { - log.info("Component instance conf dir already exists: " + compInstanceDir); - } - - log.info("Tokens substitution for component: " + roleInstance - .getCompInstanceName() + System.lineSeparator() - + tokensForSubstitution); - - for (ConfigFile originalFile : component.getConfiguration().getFiles()) { - ConfigFile configFile = originalFile.copy(); - String fileName = new Path(configFile.getDestFile()).getName(); - - // substitute file name - for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) { - configFile.setDestFile(configFile.getDestFile() - .replaceAll(Pattern.quote(token.getKey()), token.getValue())); - } - - Path remoteFile = new Path(compInstanceDir, fileName); - if (!fs.getFileSystem().exists(remoteFile)) { - log.info("Saving config file on hdfs for component " + roleInstance - .getCompInstanceName() + ": " + configFile); - - if (configFile.getSrcFile() != null) { - // Load config file template - switch (configFile.getType()) { - case HADOOP_XML: - // Hadoop_xml_template - resolveHadoopXmlTemplateAndSaveOnHdfs(fs.getFileSystem(), - tokensForSubstitution, configFile, remoteFile, appState); - break; - case TEMPLATE: - // plain-template - resolvePlainTemplateAndSaveOnHdfs(fs.getFileSystem(), - tokensForSubstitution, configFile, remoteFile, appState); - break; - default: - log.info("Not supporting loading src_file for " + configFile); - break; - } - } else { - // non-template - resolveNonTemplateConfigsAndSaveOnHdfs(fs, tokensForSubstitution, - roleInstance, configFile, fileName, remoteFile); - } - } - - // Add resource for localization - LocalResource configResource = - fs.createAmResource(remoteFile, LocalResourceType.FILE); - File destFile = new File(configFile.getDestFile()); - String symlink = APP_CONF_DIR + "/" + fileName; - if (destFile.isAbsolute()) { - launcher.addLocalResource(symlink, configResource, - configFile.getDestFile()); - log.info("Add config file for localization: " + symlink + " -> " - + configResource.getResource().getFile() + ", dest mount path: " - + configFile.getDestFile()); - } else { - launcher.addLocalResource(symlink, configResource); - log.info("Add config file for localization: " + symlink + " -> " - + configResource.getResource().getFile()); - } - } - } - - private void resolveNonTemplateConfigsAndSaveOnHdfs(SliderFileSystem fs, - Map<String, String> tokensForSubstitution, RoleInstance roleInstance, - ConfigFile configFile, String fileName, Path remoteFile) - throws IOException { - // substitute non-template configs - substituteMapWithTokens(configFile.getProps(), tokensForSubstitution); - - // write configs onto hdfs - PublishedConfiguration publishedConfiguration = - new PublishedConfiguration(fileName, - configFile.getProps().entrySet()); - if (!fs.getFileSystem().exists(remoteFile)) { - PublishedConfigurationOutputter configurationOutputter = - PublishedConfigurationOutputter.createOutputter( - ConfigFormat.resolve(configFile.getType().toString()), - publishedConfiguration); - try (FSDataOutputStream os = fs.getFileSystem().create(remoteFile)) { - configurationOutputter.save(os); - os.flush(); - } - } else { - log.info("Component instance = " + roleInstance.getCompInstanceName() - + ", config file already exists: " + remoteFile); - } - } - - // 1. substitute config template - only handle hadoop_xml format - // 2. save on hdfs - @SuppressWarnings("unchecked") - private void resolveHadoopXmlTemplateAndSaveOnHdfs(FileSystem fs, - Map<String, String> tokensForSubstitution, ConfigFile configFile, - Path remoteFile, StateAccessForProviders appState) throws IOException { - Map<String, String> conf; - try { - conf = (Map<String, String>) appState.getConfigFileCache() - .get(configFile); - } catch (ExecutionException e) { - log.info("Failed to load config file: " + configFile, e); - return; - } - // make a copy for substitution - org.apache.hadoop.conf.Configuration confCopy = - new org.apache.hadoop.conf.Configuration(false); - for (Map.Entry<String, String> entry : conf.entrySet()) { - confCopy.set(entry.getKey(), entry.getValue()); - } - // substitute properties - for (Map.Entry<String, String> entry : configFile.getProps().entrySet()) { - confCopy.set(entry.getKey(), entry.getValue()); - } - // substitute env variables - for (Map.Entry<String, String> entry : confCopy) { - String val = entry.getValue(); - if (val != null) { - for (Map.Entry<String, String> token : tokensForSubstitution - .entrySet()) { - val = val.replaceAll(Pattern.quote(token.getKey()), token.getValue()); - confCopy.set(entry.getKey(), val); - } - } - } - // save on hdfs - try (OutputStream output = fs.create(remoteFile)) { - confCopy.writeXml(output); - log.info("Reading config from: " + configFile.getSrcFile() - + ", writing to: " + remoteFile); - } - } - - // 1) read the template as a string - // 2) do token substitution - // 3) save on hdfs - private void resolvePlainTemplateAndSaveOnHdfs(FileSystem fs, - Map<String, String> tokensForSubstitution, ConfigFile configFile, - Path remoteFile, StateAccessForProviders appState) { - String content; - try { - content = (String) appState.getConfigFileCache().get(configFile); - } catch (ExecutionException e) { - log.info("Failed to load config file: " + configFile, e); - return; - } - // substitute tokens - content = substituteStrWithTokens(content, tokensForSubstitution); - - try (OutputStream output = fs.create(remoteFile)) { - org.apache.commons.io.IOUtils.write(content, output); - } catch (IOException e) { - log.info("Failed to create " + remoteFile); - } - } - - /** - * Get initial component token map to be substituted into config values. - * @param roleInstance role instance - * @return tokens to replace - */ - public Map<String, String> initCompTokensForSubstitute( - RoleInstance roleInstance) { - Map<String, String> tokens = new HashMap<>(); - tokens.put(COMPONENT_NAME, roleInstance.role); - tokens.put(COMPONENT_NAME_LC, roleInstance.role.toLowerCase()); - tokens.put(COMPONENT_INSTANCE_NAME, roleInstance.getCompInstanceName()); - tokens.put(CONTAINER_ID, roleInstance.getContainerId().toString()); - tokens.put(COMPONENT_ID, String.valueOf(roleInstance.componentId)); - return tokens; - } - - /** - * Add ROLE_HOST tokens for substitution into config values. - * @param tokens existing tokens - * @param amState access to AM state - */ - public void addComponentHostTokens(Map<String, String> tokens, - StateAccessForProviders amState) { - if (amState == null) { - return; - } - for (Map.Entry<String, Map<String, ClusterNode>> entry : - amState.getRoleClusterNodeMapping().entrySet()) { - String tokenName = entry.getKey().toUpperCase(Locale.ENGLISH) + "_HOST"; - String hosts = StringUtils .join(",", - getHostsList(entry.getValue().values(), true)); - tokens.put($(tokenName), hosts); - } - } - - /** - * Return a list of hosts based on current ClusterNodes. - * @param values cluster nodes - * @param hostOnly whether host or host/server name will be added to list - * @return list of hosts - */ - public Iterable<String> getHostsList(Collection<ClusterNode> values, - boolean hostOnly) { - List<String> hosts = new ArrayList<>(); - for (ClusterNode cn : values) { - hosts.add(hostOnly ? cn.host : cn.host + "/" + cn.name); - } - return hosts; - } - - /** - * Update ServiceRecord in Registry with IP and hostname. - * @param amState access to AM state - * @param yarnRegistry acces to YARN registry - * @param containerId container ID - * @param roleName component name - * @param ip list of IPs - * @param hostname hostname - */ - public void updateServiceRecord(StateAccessForProviders amState, - YarnRegistryViewForProviders yarnRegistry, - String containerId, String roleName, List<String> ip, String hostname) - throws IOException { - try { - RoleInstance role = null; - if(ip != null && !ip.isEmpty()){ - role = amState.getOwnedContainer(containerId); - role.ip = ip.get(0); - } - if(hostname != null && !hostname.isEmpty()){ - role = amState.getOwnedContainer(containerId); - role.hostname = hostname; - } - if (role != null) { - // create and publish updated service record (including hostname & ip) - ServiceRecord record = new ServiceRecord(); - record.set(YarnRegistryAttributes.YARN_ID, containerId); - record.description = role.getCompInstanceName(); - record.set(YarnRegistryAttributes.YARN_PERSISTENCE, - PersistencePolicies.CONTAINER); - // TODO: use constants from YarnRegistryAttributes - if (role.ip != null) { - record.set("yarn:ip", role.ip); - } - if (role.hostname != null) { - record.set("yarn:hostname", role.hostname); - } - yarnRegistry.putComponent( - RegistryPathUtils.encodeYarnID(containerId), record); - } - } catch (NoSuchNodeException e) { - // ignore - there is nothing to do if we don't find a container - log.warn("Owned container {} not found - {}", containerId, e); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java deleted file mode 100644 index 5ecc374..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers; - -import org.apache.slider.api.resource.Artifact; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.providers.docker.DockerProviderFactory; -import org.apache.slider.providers.tarball.TarballProviderFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Base class for factories. - */ -public abstract class SliderProviderFactory { - protected static final Logger LOG = - LoggerFactory.getLogger(SliderProviderFactory.class); - - protected SliderProviderFactory() {} - - public abstract AbstractClientProvider createClientProvider(); - - public abstract ProviderService createServerProvider(); - - public static synchronized ProviderService getProviderService(Artifact - artifact) { - return createSliderProviderFactory(artifact).createServerProvider(); - } - - public static synchronized AbstractClientProvider getClientProvider(Artifact - artifact) { - return createSliderProviderFactory(artifact).createClientProvider(); - } - - /** - * Create a provider for a specific application - * @param artifact artifact - * @return provider factory - * @throws SliderException on any instantiation problem - */ - public static synchronized SliderProviderFactory createSliderProviderFactory( - Artifact artifact) { - if (artifact == null || artifact.getType() == null) { - LOG.debug("Loading service provider type default"); - return DefaultProviderFactory.getInstance(); - } - LOG.debug("Loading service provider type {}", artifact.getType()); - switch (artifact.getType()) { - // TODO add handling for custom types? - // TODO handle application - case DOCKER: - return DockerProviderFactory.getInstance(); - case TARBALL: - return TarballProviderFactory.getInstance(); - default: - throw new IllegalArgumentException(String.format("Resolution error, " + - "%s should not be passed to createSliderProviderFactory", - artifact.getType())); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java deleted file mode 100644 index 4773327..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.docker; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.fs.FileSystem; -import org.apache.slider.api.resource.Artifact; -import org.apache.slider.api.resource.ConfigFile; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.providers.AbstractClientProvider; -import org.apache.slider.util.RestApiErrorMessages; - -import java.io.IOException; - -public class DockerClientProvider extends AbstractClientProvider - implements SliderKeys { - - public DockerClientProvider() { - super(); - } - - @Override - public void validateArtifact(Artifact artifact, FileSystem fileSystem) { - if (artifact == null) { - throw new IllegalArgumentException( - RestApiErrorMessages.ERROR_ARTIFACT_INVALID); - } - if (StringUtils.isEmpty(artifact.getId())) { - throw new IllegalArgumentException( - RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID); - } - } - - @Override - protected void validateConfigFile(ConfigFile configFile, FileSystem - fileSystem) throws IOException { - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java deleted file mode 100644 index 7413ebb..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.docker; - -public interface DockerKeys { - String PROVIDER_DOCKER = "docker"; - String DOCKER_PREFIX = "docker."; - String DOCKER_IMAGE = DOCKER_PREFIX + "image"; - String DOCKER_NETWORK = DOCKER_PREFIX + "network"; - String DOCKER_USE_PRIVILEGED = DOCKER_PREFIX + "usePrivileged"; - String DOCKER_START_COMMAND = DOCKER_PREFIX + "startCommand"; - - String DEFAULT_DOCKER_NETWORK = "bridge"; - Boolean DEFAULT_DOCKER_USE_PRIVILEGED = false; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java deleted file mode 100644 index 6977e41..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.docker; - -import org.apache.slider.providers.AbstractClientProvider; -import org.apache.slider.providers.ProviderService; -import org.apache.slider.providers.SliderProviderFactory; - -public class DockerProviderFactory extends SliderProviderFactory { - private static final SliderProviderFactory FACTORY = new - DockerProviderFactory(); - - private DockerProviderFactory() { - } - - private static class Client { - static final AbstractClientProvider PROVIDER = new DockerClientProvider(); - } - - private static class Server { - static final ProviderService PROVIDER = new DockerProviderService(); - } - - @Override - public AbstractClientProvider createClientProvider() { - return Client.PROVIDER; - } - - @Override - public ProviderService createServerProvider() { - return Server.PROVIDER; - } - - public static SliderProviderFactory getInstance() { - return FACTORY; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java deleted file mode 100644 index 73783af..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.docker; - -import org.apache.hadoop.registry.client.api.RegistryConstants; -import org.apache.hadoop.registry.client.binding.RegistryUtils; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.Component; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.core.launch.ContainerLauncher; -import org.apache.slider.providers.AbstractProviderService; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.text.MessageFormat; - -public class DockerProviderService extends AbstractProviderService - implements DockerKeys { - - protected static final Logger log = - LoggerFactory.getLogger(DockerProviderService.class); - - protected DockerProviderService() { - super(DockerProviderService.class.getSimpleName()); - } - - public void processArtifact(ContainerLauncher launcher, Application - application, RoleInstance roleInstance, SliderFileSystem fileSystem) - throws IOException { - Component component = roleInstance.providerRole.component; - launcher.setYarnDockerMode(true); - launcher.setDockerImage(component.getArtifact().getId()); - launcher.setDockerNetwork(component.getConfiguration() - .getProperty(DOCKER_NETWORK, DEFAULT_DOCKER_NETWORK)); - String domain = getConfig().get(RegistryConstants.KEY_DNS_DOMAIN); - String hostname; - if (domain == null || domain.isEmpty()) { - hostname = MessageFormat.format("{0}.{1}.{2}", roleInstance - .getCompInstanceName(), application.getName(), RegistryUtils - .currentUser()); - } else { - hostname = MessageFormat.format("{0}.{1}.{2}.{3}", roleInstance - .getCompInstanceName(), application.getName(), RegistryUtils - .currentUser(), domain); - } - launcher.setDockerHostname(hostname); - launcher.setRunPrivilegedContainer(component.getRunPrivilegedContainer()); - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org