http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java new file mode 100644 index 0000000..c87c3b4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java @@ -0,0 +1,449 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.utils; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.registry.client.api.RegistryConstants; +import org.apache.hadoop.registry.client.binding.RegistryUtils; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Artifact; +import org.apache.slider.api.resource.Component; +import org.apache.slider.api.resource.Configuration; +import org.apache.slider.api.resource.Resource; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.persist.JsonSerDeser; +import org.apache.hadoop.yarn.service.provider.AbstractClientProvider; +import org.apache.hadoop.yarn.service.provider.ProviderFactory; +import org.apache.slider.server.servicemonitor.MonitorUtils; +import org.apache.slider.server.services.utility.PatternValidator; +import org.apache.slider.util.RestApiConstants; +import org.apache.slider.util.RestApiErrorMessages; +import org.codehaus.jackson.map.PropertyNamingStrategy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ServiceApiUtil { + private static final Logger LOG = + LoggerFactory.getLogger(ServiceApiUtil.class); + public static JsonSerDeser<Application> jsonSerDeser = + new JsonSerDeser<>(Application.class, + PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); + private static final PatternValidator compNamePattern + = new PatternValidator("[a-z][a-z0-9-]*"); + + @VisibleForTesting + public static void setJsonSerDeser(JsonSerDeser jsd) { + jsonSerDeser = jsd; + } + + @VisibleForTesting + public static void validateAndResolveApplication(Application application, + SliderFileSystem fs, org.apache.hadoop.conf.Configuration conf) throws + IOException { + boolean dnsEnabled = conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED, + RegistryConstants.DEFAULT_DNS_ENABLED); + if (dnsEnabled && RegistryUtils.currentUser().length() > RegistryConstants + .MAX_FQDN_LABEL_LENGTH) { + throw new IllegalArgumentException(RestApiErrorMessages + .ERROR_USER_NAME_INVALID); + } + if (StringUtils.isEmpty(application.getName())) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID); + } + if (!SliderUtils.isClusternameValid(application.getName()) || (dnsEnabled + && application.getName().length() > RegistryConstants + .MAX_FQDN_LABEL_LENGTH)) { + throw new IllegalArgumentException(String.format( + RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID_FORMAT, + application.getName())); + } + + // If the application has no components do top-level checks + if (!hasComponent(application)) { + // If artifact is of type APPLICATION, read other application components + if (application.getArtifact() != null && application.getArtifact() + .getType() == Artifact.TypeEnum.APPLICATION) { + if (StringUtils.isEmpty(application.getArtifact().getId())) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID); + } + Application otherApplication = loadApplication(fs, + application.getArtifact().getId()); + application.setComponents(otherApplication.getComponents()); + application.setArtifact(null); + SliderUtils.mergeMapsIgnoreDuplicateKeys(application.getQuicklinks(), + otherApplication.getQuicklinks()); + } else { + // Since it is a simple app with no components, create a default + // component + Component comp = createDefaultComponent(application); + validateComponent(comp, fs.getFileSystem()); + application.getComponents().add(comp); + if (application.getLifetime() == null) { + application.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME); + } + return; + } + } + + // Validate there are no component name collisions (collisions are not + // currently supported) and add any components from external applications + // TODO allow name collisions? see AppState#roles + // TODO or add prefix to external component names? + Configuration globalConf = application.getConfiguration(); + Set<String> componentNames = new HashSet<>(); + List<Component> componentsToRemove = new ArrayList<>(); + List<Component> componentsToAdd = new ArrayList<>(); + for (Component comp : application.getComponents()) { + int maxCompLength = RegistryConstants.MAX_FQDN_LABEL_LENGTH; + if (comp.getUniqueComponentSupport()) { + maxCompLength = maxCompLength - Long.toString(Long.MAX_VALUE).length(); + } + if (dnsEnabled && comp.getName().length() > maxCompLength) { + throw new IllegalArgumentException(String.format(RestApiErrorMessages + .ERROR_COMPONENT_NAME_INVALID, maxCompLength, comp.getName())); + } + if (componentNames.contains(comp.getName())) { + throw new IllegalArgumentException("Component name collision: " + + comp.getName()); + } + // If artifact is of type APPLICATION (which cannot be filled from + // global), read external application and add its components to this + // application + if (comp.getArtifact() != null && comp.getArtifact().getType() == + Artifact.TypeEnum.APPLICATION) { + if (StringUtils.isEmpty(comp.getArtifact().getId())) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID); + } + LOG.info("Marking {} for removal", comp.getName()); + componentsToRemove.add(comp); + List<Component> externalComponents = getApplicationComponents(fs, + comp.getArtifact().getId()); + for (Component c : externalComponents) { + Component override = application.getComponent(c.getName()); + if (override != null && override.getArtifact() == null) { + // allow properties from external components to be overridden / + // augmented by properties in this component, except for artifact + // which must be read from external component + override.mergeFrom(c); + LOG.info("Merging external component {} from external {}", c + .getName(), comp.getName()); + } else { + if (componentNames.contains(c.getName())) { + throw new IllegalArgumentException("Component name collision: " + + c.getName()); + } + componentNames.add(c.getName()); + componentsToAdd.add(c); + LOG.info("Adding component {} from external {}", c.getName(), + comp.getName()); + } + } + } else { + // otherwise handle as a normal component + componentNames.add(comp.getName()); + // configuration + comp.getConfiguration().mergeFrom(globalConf); + } + } + application.getComponents().removeAll(componentsToRemove); + application.getComponents().addAll(componentsToAdd); + + // Validate components and let global values take effect if component level + // values are not provided + Artifact globalArtifact = application.getArtifact(); + Resource globalResource = application.getResource(); + Long globalNumberOfContainers = application.getNumberOfContainers(); + String globalLaunchCommand = application.getLaunchCommand(); + for (Component comp : application.getComponents()) { + // fill in global artifact unless it is type APPLICATION + if (comp.getArtifact() == null && application.getArtifact() != null + && application.getArtifact().getType() != Artifact.TypeEnum + .APPLICATION) { + comp.setArtifact(globalArtifact); + } + // fill in global resource + if (comp.getResource() == null) { + comp.setResource(globalResource); + } + // fill in global container count + if (comp.getNumberOfContainers() == null) { + comp.setNumberOfContainers(globalNumberOfContainers); + } + // fill in global launch command + if (comp.getLaunchCommand() == null) { + comp.setLaunchCommand(globalLaunchCommand); + } + // validate dependency existence + if (comp.getDependencies() != null) { + for (String dependency : comp.getDependencies()) { + if (!componentNames.contains(dependency)) { + throw new IllegalArgumentException(String.format( + RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, dependency, + comp.getName())); + } + } + } + validateComponent(comp, fs.getFileSystem()); + } + + // validate dependency tree + sortByDependencies(application.getComponents()); + + // Application lifetime if not specified, is set to unlimited lifetime + if (application.getLifetime() == null) { + application.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME); + } + } + + public static void validateComponent(Component comp, FileSystem fs) + throws IOException { + validateCompName(comp.getName()); + + AbstractClientProvider compClientProvider = ProviderFactory + .getClientProvider(comp.getArtifact()); + compClientProvider.validateArtifact(comp.getArtifact(), fs); + + if (comp.getLaunchCommand() == null && (comp.getArtifact() == null || comp + .getArtifact().getType() != Artifact.TypeEnum.DOCKER)) { + throw new IllegalArgumentException(RestApiErrorMessages + .ERROR_ABSENT_LAUNCH_COMMAND); + } + + validateApplicationResource(comp.getResource(), comp); + + if (comp.getNumberOfContainers() == null + || comp.getNumberOfContainers() < 0) { + throw new IllegalArgumentException(String.format( + RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID + + ": " + comp.getNumberOfContainers(), comp.getName())); + } + compClientProvider.validateConfigFiles(comp.getConfiguration() + .getFiles(), fs); + + MonitorUtils.getProbe(comp.getReadinessCheck()); + } + + // Check component name format and transform to lower case. + public static void validateCompName(String compName) { + if (StringUtils.isEmpty(compName)) { + throw new IllegalArgumentException("Component name can not be empty"); + } + // validate component name + if (compName.contains("_")) { + throw new IllegalArgumentException( + "Invalid format for component name: " + compName + + ", can not use '_' as DNS hostname does not allow underscore. Use '-' Instead. "); + } + compNamePattern.validate(compName); + } + + @VisibleForTesting + public static List<Component> getApplicationComponents(SliderFileSystem + fs, String appName) throws IOException { + return loadApplication(fs, appName).getComponents(); + } + + public static Application loadApplication(SliderFileSystem fs, String + appName) throws IOException { + Path appJson = getAppJsonPath(fs, appName); + LOG.info("Loading application definition from " + appJson); + return jsonSerDeser.load(fs.getFileSystem(), appJson); + } + + public static Application loadApplicationFrom(SliderFileSystem fs, + Path appDefPath) throws IOException { + LOG.info("Loading application definition from " + appDefPath); + return jsonSerDeser.load(fs.getFileSystem(), appDefPath); + } + + public static Path getAppJsonPath(SliderFileSystem fs, String appName) { + Path appDir = fs.buildClusterDirPath(appName); + Path appJson = new Path(appDir, appName + ".json"); + return appJson; + } + + private static void validateApplicationResource(Resource resource, + Component comp) { + // Only apps/components of type APPLICATION can skip resource requirement + if (resource == null) { + throw new IllegalArgumentException( + comp == null ? RestApiErrorMessages.ERROR_RESOURCE_INVALID : String + .format(RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID, + comp.getName())); + } + // One and only one of profile OR cpus & memory can be specified. Specifying + // both raises validation error. + if (StringUtils.isNotEmpty(resource.getProfile()) && ( + resource.getCpus() != null || StringUtils + .isNotEmpty(resource.getMemory()))) { + throw new IllegalArgumentException(comp == null ? + RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED : + String.format( + RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED, + comp.getName())); + } + // Currently resource profile is not supported yet, so we will raise + // validation error if only resource profile is specified + if (StringUtils.isNotEmpty(resource.getProfile())) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET); + } + + String memory = resource.getMemory(); + Integer cpus = resource.getCpus(); + if (StringUtils.isEmpty(memory)) { + throw new IllegalArgumentException( + comp == null ? RestApiErrorMessages.ERROR_RESOURCE_MEMORY_INVALID : + String.format( + RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID, + comp.getName())); + } + if (cpus == null) { + throw new IllegalArgumentException( + comp == null ? RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID : + String.format( + RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID, + comp.getName())); + } + if (cpus <= 0) { + throw new IllegalArgumentException(comp == null ? + RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID_RANGE : String + .format( + RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE, + comp.getName())); + } + } + + // check if comp mem size exceeds cluster limit + public static void validateCompResourceSize( + org.apache.hadoop.yarn.api.records.Resource maxResource, + Application application) throws YarnException { + for (Component component : application.getComponents()) { + // only handle mem now. + long mem = Long.parseLong(component.getResource().getMemory()); + if (mem > maxResource.getMemorySize()) { + throw new YarnException( + "Component " + component.getName() + " memory size (" + mem + + ") is larger than configured max container memory size (" + + maxResource.getMemorySize() + ")"); + } + } + } + + public static boolean hasComponent(Application application) { + if (application.getComponents() == null || application.getComponents() + .isEmpty()) { + return false; + } + return true; + } + + public static Component createDefaultComponent(Application app) { + Component comp = new Component(); + comp.setName(RestApiConstants.DEFAULT_COMPONENT_NAME); + comp.setArtifact(app.getArtifact()); + comp.setResource(app.getResource()); + comp.setNumberOfContainers(app.getNumberOfContainers()); + comp.setLaunchCommand(app.getLaunchCommand()); + comp.setConfiguration(app.getConfiguration()); + return comp; + } + + public static Collection<Component> sortByDependencies(List<Component> + components) { + Map<String, Component> sortedComponents = + sortByDependencies(components, null); + return sortedComponents.values(); + } + + /** + * Each internal call of sortByDependencies will identify all of the + * components with the same dependency depth (the lowest depth that has not + * been processed yet) and add them to the sortedComponents list, preserving + * their original ordering in the components list. + * + * So the first time it is called, all components with no dependencies + * (depth 0) will be identified. The next time it is called, all components + * that have dependencies only on the the depth 0 components will be + * identified (depth 1). This will be repeated until all components have + * been added to the sortedComponents list. If no new components are + * identified but the sortedComponents list is not complete, an error is + * thrown. + */ + private static Map<String, Component> sortByDependencies(List<Component> + components, Map<String, Component> sortedComponents) { + if (sortedComponents == null) { + sortedComponents = new LinkedHashMap<>(); + } + + Map<String, Component> componentsToAdd = new LinkedHashMap<>(); + List<Component> componentsSkipped = new ArrayList<>(); + for (Component component : components) { + String name = component.getName(); + if (sortedComponents.containsKey(name)) { + continue; + } + boolean dependenciesAlreadySorted = true; + if (!SliderUtils.isEmpty(component.getDependencies())) { + for (String dependency : component.getDependencies()) { + if (!sortedComponents.containsKey(dependency)) { + dependenciesAlreadySorted = false; + break; + } + } + } + if (dependenciesAlreadySorted) { + componentsToAdd.put(name, component); + } else { + componentsSkipped.add(component); + } + } + + if (componentsToAdd.size() == 0) { + throw new IllegalArgumentException(String.format(RestApiErrorMessages + .ERROR_DEPENDENCY_CYCLE, componentsSkipped)); + } + sortedComponents.putAll(componentsToAdd); + if (sortedComponents.size() == components.size()) { + return sortedComponents; + } + return sortByDependencies(components, sortedComponents); + } + + public static String $(String s) { + return "${" + s +"}"; + } +}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java index da87e3a..fa21211 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java @@ -21,7 +21,7 @@ package org.apache.slider.api; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import static org.apache.slider.util.ServiceApiUtil.$; +import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$; /** * This class defines constants that can be used in input spec for http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java index 448d4ba..aaf2f88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java @@ -22,7 +22,7 @@ import org.apache.hadoop.ipc.VersionedProtocol; import org.apache.hadoop.security.KerberosInfo; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.slider.api.proto.Messages; -import org.apache.slider.common.SliderXmlConfKeys; +import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys; import java.io.IOException; http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java index 4b7e59b..626efb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java @@ -226,7 +226,7 @@ public class Application extends BaseResource { /** * Life time (in seconds) of the application from the time it reaches the - * STARTED state (after which it is automatically destroyed by YARN). For + * RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For * unlimited lifetime do not set a lifetime value. **/ public Application lifetime(Long lifetime) { @@ -234,7 +234,7 @@ public class Application extends BaseResource { return this; } - @ApiModelProperty(example = "null", value = "Life time (in seconds) of the application from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.") + @ApiModelProperty(example = "null", value = "Life time (in seconds) of the application from the time it reaches the RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.") @JsonProperty("lifetime") public Long getLifetime() { return lifetime; http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java index 229e288..c15f82c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java @@ -213,6 +213,9 @@ public class Component implements Serializable { this.containers.add(container); } + public void removeContainer(Container container) { + containers.remove(container); + } public Container getContainer(String id) { for (Container container : containers) { if (container.getId().equals(id)) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ContainerState.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ContainerState.java index e79f6e0..cd1ef4a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ContainerState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ContainerState.java @@ -21,5 +21,5 @@ package org.apache.slider.api.resource; * The current state of the container of an application. **/ public enum ContainerState { - INIT, READY; + RUNNING_BUT_UNREADY, READY, STOPPED } http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java index c255369..314dfbb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java @@ -100,6 +100,13 @@ public class Resource extends BaseResource implements Cloneable { this.memory = memory; } + public long getMemoryMB() { + if (this.memory == null) { + return 0; + } + return Long.valueOf(memory); + } + @Override public boolean equals(java.lang.Object o) { if (this == o) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java index c3ccb1d..b28257f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java @@ -24,7 +24,7 @@ import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.hadoop.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.registry.client.exceptions.NoRecordException; import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.slider.common.SliderKeys; +import org.apache.hadoop.yarn.service.conf.SliderKeys; import org.apache.slider.core.exceptions.BadCommandArgumentsException; import org.apache.slider.core.exceptions.NotFoundException; import org.apache.slider.core.exceptions.SliderException; http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index e261a8d..7712191 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -29,7 +29,9 @@ import org.apache.hadoop.fs.PathNotFoundException; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto; import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.registry.client.api.RegistryConstants; import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.hadoop.registry.client.binding.RegistryPathUtils; @@ -60,9 +62,11 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Times; +import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.slider.api.SliderClusterProtocol; import org.apache.slider.api.proto.Messages; import org.apache.slider.api.resource.Application; @@ -71,17 +75,17 @@ import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.api.types.NodeInformationList; import org.apache.slider.client.ipc.SliderClusterOperations; import org.apache.slider.common.Constants; -import org.apache.slider.common.SliderExitCodes; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.SliderXmlConfKeys; -import org.apache.slider.common.params.AbstractActionArgs; +import org.apache.hadoop.yarn.service.conf.SliderExitCodes; +import org.apache.hadoop.yarn.service.conf.SliderKeys; +import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys; +import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; import org.apache.slider.common.params.ActionAMSuicideArgs; import org.apache.slider.common.params.ActionClientArgs; -import org.apache.slider.common.params.ActionDependencyArgs; +import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs; import org.apache.slider.common.params.ActionDiagnosticArgs; import org.apache.slider.common.params.ActionExistsArgs; -import org.apache.slider.common.params.ActionFlexArgs; +import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs; import org.apache.slider.common.params.ActionFreezeArgs; import org.apache.slider.common.params.ActionKDiagArgs; import org.apache.slider.common.params.ActionKeytabArgs; @@ -96,9 +100,9 @@ import org.apache.slider.common.params.ActionStatusArgs; import org.apache.slider.common.params.ActionThawArgs; import org.apache.slider.common.params.ActionTokensArgs; import org.apache.slider.common.params.ActionUpgradeArgs; -import org.apache.slider.common.params.Arguments; -import org.apache.slider.common.params.ClientArgs; -import org.apache.slider.common.params.CommonArgs; +import org.apache.hadoop.yarn.service.client.params.Arguments; +import org.apache.hadoop.yarn.service.client.params.ClientArgs; +import org.apache.hadoop.yarn.service.client.params.CommonArgs; import org.apache.slider.common.tools.ConfigHelper; import org.apache.slider.common.tools.Duration; import org.apache.slider.common.tools.SliderFileSystem; @@ -131,12 +135,14 @@ import org.apache.slider.core.registry.docstore.PublishedExportsSet; import org.apache.slider.core.registry.retrieve.RegistryRetriever; import org.apache.slider.core.zk.BlockingZKWatcher; import org.apache.slider.core.zk.ZKIntegration; -import org.apache.slider.providers.AbstractClientProvider; -import org.apache.slider.providers.ProviderUtils; -import org.apache.slider.server.appmaster.SliderAppMaster; +import org.apache.hadoop.yarn.service.provider.AbstractClientProvider; +import org.apache.hadoop.yarn.service.provider.ProviderUtils; import org.apache.slider.server.appmaster.rpc.RpcBinder; +import org.apache.hadoop.yarn.service.ClientAMProtocol; +import org.apache.hadoop.yarn.service.client.ClientAMProxy; +import org.apache.hadoop.yarn.service.ServiceMaster; import org.apache.slider.server.services.utility.AbstractSliderLaunchedService; -import org.apache.slider.util.ServiceApiUtil; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs; @@ -153,6 +159,7 @@ import java.io.InterruptedIOException; import java.io.OutputStreamWriter; import java.io.PrintStream; import java.io.PrintWriter; +import java.net.InetSocketAddress; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; @@ -170,9 +177,11 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.apache.hadoop.registry.client.binding.RegistryUtils.*; +import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*; import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG; -import static org.apache.slider.common.params.SliderActions.*; +import static org.apache.hadoop.yarn.service.client.params.SliderActions.*; import static org.apache.slider.common.tools.SliderUtils.*; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto; /** * Client service for Slider @@ -217,22 +226,24 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private SliderClusterOperations sliderClusterOperations; protected SliderFileSystem sliderFileSystem; - + private YarnRPC rpc; /** * Yarn client service */ private SliderYarnClientImpl yarnClient; private YarnAppListClient yarnAppListClient; - + ResourceCalculator calculator; /** * The YARN registry service */ @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized") private RegistryOperations registryOperations; - private static EnumSet<YarnApplicationState> terminatedStates = EnumSet - .of(YarnApplicationState.FINISHED, YarnApplicationState.FAILED, - YarnApplicationState.KILLED); + private static EnumSet<YarnApplicationState> terminatedStates = + EnumSet.of(FINISHED, FAILED, KILLED); + private static EnumSet<YarnApplicationState> waitingStates = + EnumSet.of(NEW, NEW_SAVING, SUBMITTED, RUNNING); + /** * Constructor */ @@ -277,9 +288,20 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe if (coreAction.getHadoopServicesRequired()) { initHadoopBinding(); } + rpc = YarnRPC.create(conf); super.serviceInit(conf); } + @Override + protected void serviceStart() throws Exception { + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + super.serviceStop(); + } + /** * Launched service execution. This runs {@link #exec()} * then catches some exceptions and converts them to exit codes @@ -668,9 +690,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe Path appRootDir = sliderFileSystem.buildClusterDirPath(app.getName()); deployedClusterName = appName; - YarnClientApplication yarnApp = yarnClient.createApplication(); + YarnClientApplication yarnApp = yarnClient.createApplication(); ApplicationSubmissionContext submissionContext = yarnApp.getApplicationSubmissionContext(); + ServiceApiUtil.validateCompResourceSize( + yarnApp.getNewApplicationResponse().getMaximumResourceCapability(), + app); + applicationId = submissionContext.getApplicationId(); submissionContext.setKeepContainersAcrossApplicationAttempts(true); if (app.getLifetime() > 0) { @@ -769,11 +795,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe CLI.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME); CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR); } - CLI.add(SliderAppMaster.SERVICE_CLASSNAME); + CLI.add(ServiceMaster.class.getCanonicalName()); CLI.add(ACTION_CREATE, appName); //TODO debugAM CLI.add(Arguments.ARG_DEBUG) - CLI.add(Arguments.ARG_CLUSTER_URI, appRootDir.toUri()); -// InetSocketAddress rmSchedulerAddress = getRmSchedulerAddress(conf); + CLI.add(Arguments.ARG_CLUSTER_URI, new Path(appRootDir, appName + ".json")); + // InetSocketAddress rmSchedulerAddress = getRmSchedulerAddress(conf); // String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress); // CLI.add(Arguments.ARG_RM_ADDR, rmAddr); // pass the registry binding @@ -825,12 +851,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return env; } - private Path addJarResource(String appName, + protected Path addJarResource(String appName, Map<String, LocalResource> localResources) throws IOException, SliderException { Path libPath = sliderFileSystem.buildClusterDirPath(appName); ProviderUtils - .addProviderJar(localResources, SliderAppMaster.class, SLIDER_JAR, + .addProviderJar(localResources, ServiceMaster.class, SLIDER_JAR, sliderFileSystem, libPath, "lib", false); Path dependencyLibTarGzip = sliderFileSystem.getDependencyTarGzip(); if (sliderFileSystem.isFile(dependencyLibTarGzip)) { @@ -1162,8 +1188,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe public String updateLifetime(String appName, long lifetime) throws YarnException, IOException { - EnumSet<YarnApplicationState> appStates = EnumSet.range( - YarnApplicationState.NEW, YarnApplicationState.RUNNING); + EnumSet<YarnApplicationState> appStates = EnumSet.range(NEW, RUNNING); ApplicationReport report = findInstance(appName, appStates); if (report == null) { throw new YarnException("Application not found for " + appName); @@ -1381,14 +1406,14 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe YarnApplicationState min, max; if (live) { - min = YarnApplicationState.NEW; - max = YarnApplicationState.RUNNING; + min = NEW; + max = RUNNING; } else if (!state.isEmpty()) { YarnApplicationState stateVal = extractYarnApplicationState(state); min = max = stateVal; } else { - min = YarnApplicationState.NEW; - max = YarnApplicationState.KILLED; + min = NEW; + max = KILLED; } // get the complete list of persistent instances Map<String, Path> persistentInstances = sliderFileSystem.listPersistentInstances(); @@ -1478,14 +1503,14 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } return stateVal; } - + /** * Is an application active: accepted or running * @param report the application report * @return true if it is running or scheduled to run. */ public boolean isApplicationActive(ApplicationReport report) { - return report.getYarnApplicationState() == YarnApplicationState.RUNNING + return report.getYarnApplicationState() == RUNNING || report.getYarnApplicationState() == YarnApplicationState.ACCEPTED; } @@ -1547,8 +1572,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // the app exists, check that it is not in any terminated state YarnApplicationState appstate = instance.getYarnApplicationState(); log.debug(" current app state = {}", appstate); - inDesiredState = - appstate.ordinal() < YarnApplicationState.FINISHED.ordinal(); + inDesiredState = appstate.ordinal() < FINISHED.ordinal(); } else { // scan for instance in single --state state state = state.toUpperCase(Locale.ENGLISH); @@ -1851,6 +1875,72 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return clusterOperations.getApplication(); } + private ClientAMProtocol connectToAM(String appName) + throws IOException, YarnException { + if (applicationId == null) { + Application persistedApp = ServiceApiUtil.loadApplication(sliderFileSystem, + appName); + if (persistedApp == null) { + throw new YarnException("Application " + appName + + " doesn't exist on hdfs. Please check if the app exists in RM"); + } + applicationId = ApplicationId.fromString(persistedApp.getId()); + } + // Wait until app becomes running. + long startTime = System.currentTimeMillis(); + int pollCount = 0; + ApplicationReport appReport = null; + while (true) { + appReport = yarnClient.getApplicationReport(applicationId); + YarnApplicationState state = appReport.getYarnApplicationState(); + if (state == RUNNING) { + break; + } + if (terminatedStates.contains(state)) { + throw new YarnException( + "Failed to getStatus " + applicationId + ": " + appReport + .getDiagnostics()); + } + long elapsedMillis = System.currentTimeMillis() - startTime; + // if over 5 min, quit + if (elapsedMillis >= 300000) { + throw new YarnException( + "Timed out while waiting for application " + applicationId + + " to be running"); + } + + if (++pollCount % 10 == 0) { + log.info("Waiting for application {} to be running, current state is {}", + applicationId, state); + } + try { + Thread.sleep(3000); + } catch (InterruptedException ie) { + String msg = + "Interrupted while waiting for application " + applicationId + + " to be running."; + throw new YarnException(msg, ie); + } + } + + // Make the connection + InetSocketAddress address = NetUtils + .createSocketAddrForHost(appReport.getHost(), appReport.getRpcPort()); + return ClientAMProxy + .createProxy(getConfig(), ClientAMProtocol.class, + UserGroupInformation.getCurrentUser(), rpc, address); + } + + public Application getStatus(String appName) + throws IOException, YarnException { + ClientAMProtocol proxy = connectToAM(appName); + GetStatusResponseProto response = + proxy.getStatus(GetStatusRequestProto.newBuilder().build()); + Application app = jsonSerDeser.fromJson(response.getStatus()); + return app; + } + + /** * Bond to a running cluster * @param clustername cluster name @@ -2160,7 +2250,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe builder.append(entry.getKey()).append("=") .append(entry.getValue()).append("\n"); } - + println(builder.toString()); // then the config @@ -2470,7 +2560,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe serviceclassPath(currentUser(), SliderKeys.APP_TYPE)); return recordMap; } - + /** * List instances in the registry * @return the instance IDs @@ -2686,8 +2776,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe @VisibleForTesting public ApplicationReport monitorAppToRunning(Duration duration) throws YarnException, IOException { - return yarnClient.monitorAppToState(applicationId, YarnApplicationState - .RUNNING, duration); + return yarnClient.monitorAppToState(applicationId, RUNNING, duration); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java index 7477c05..f1bf2ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java @@ -26,9 +26,9 @@ import org.apache.slider.api.types.NodeInformationList; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; import org.apache.slider.common.params.ActionAMSuicideArgs; import org.apache.slider.common.params.ActionClientArgs; -import org.apache.slider.common.params.ActionDependencyArgs; +import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs; import org.apache.slider.common.params.ActionDiagnosticArgs; -import org.apache.slider.common.params.ActionFlexArgs; +import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs; import org.apache.slider.common.params.ActionFreezeArgs; import org.apache.slider.common.params.ActionKeytabArgs; import org.apache.slider.common.params.ActionNodesArgs; http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java index 306bd99..e1b578c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java @@ -20,44 +20,29 @@ package org.apache.slider.client; import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; -import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.util.ConverterUtils; -import org.apache.hadoop.yarn.util.Records; import org.apache.slider.api.types.NodeInformation; import org.apache.slider.api.types.NodeInformationList; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.params.ActionNodesArgs; -import org.apache.slider.common.tools.CoreFileSystem; +import org.apache.hadoop.yarn.service.conf.SliderKeys; import org.apache.slider.common.tools.Duration; -import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.exceptions.BadCommandArgumentsException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.net.BindException; -import java.net.InetSocketAddress; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; //TODO, Remove this class and YarnAppListClient http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java deleted file mode 100644 index 5758f79..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common; - -import org.apache.slider.core.main.LauncherExitCodes; - -public interface SliderExitCodes extends LauncherExitCodes { - - /** - * starting point for exit codes; not an exception itself - */ - int _EXIT_CODE_BASE = 64; - - /** - * service entered the failed state: {@value} - */ - int EXIT_YARN_SERVICE_FAILED = 65; - - /** - * service was killed: {@value} - */ - int EXIT_YARN_SERVICE_KILLED = 66; - - /** - * timeout on monitoring client: {@value} - */ - int EXIT_TIMED_OUT = 67; - - /** - * service finished with an error: {@value} - */ - int EXIT_YARN_SERVICE_FINISHED_WITH_ERROR = 68; - - /** - * the application instance is unknown: {@value} - */ - int EXIT_UNKNOWN_INSTANCE = 69; - - /** - * the application instance is in the wrong state for that operation: {@value} - */ - int EXIT_BAD_STATE = 70; - - /** - * A spawned master process failed - */ - int EXIT_PROCESS_FAILED = 71; - - /** - * The instance failed -too many containers were - * failing or some other threshold was reached - */ - int EXIT_DEPLOYMENT_FAILED = 72; - - /** - * The application is live -and the requested operation - * does not work if the cluster is running - */ - int EXIT_APPLICATION_IN_USE = 73; - - /** - * There already is an application instance of that name - * when an attempt is made to create a new instance - */ - int EXIT_INSTANCE_EXISTS = 75; - - /** - * Exit code when the configurations in valid/incomplete: {@value} - */ - int EXIT_BAD_CONFIGURATION = 77; - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java deleted file mode 100644 index 865562e..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common; - -/** - * Keys and various constants for Slider - */ -public interface SliderKeys extends SliderXmlConfKeys { - - /** - * This is the name of the slider appmaster in configurations :{@value} - */ - String COMPONENT_AM = "slider-appmaster"; - - /** - * Slider role is "special":{@value} - */ - int ROLE_AM_PRIORITY_INDEX = 0; - - - /** - * The path under which cluster and temp data are stored - * {@value} - */ - String SLIDER_BASE_DIRECTORY = ".slider"; - - /** - * The paths under which Slider AM dependency libraries are stored - */ - String SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK = "slider_dep"; - String SLIDER_DEPENDENCY_HDP_PARENT_DIR = "/hdp"; - String SLIDER_DEPENDENCY_DIR = "/apps/%s/slider"; - String SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME = "slider-dep"; - String SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT = ".tar.gz"; - String SLIDER_DEPENDENCY_DIR_PERMISSIONS = "755"; - - /** - * - */ - String HDP_VERSION_PROP_NAME = "HDP_VERSION"; - - /** - * name of the relative path to expaned an image into: {@value}. - * The title of this path is to help people understand it when - * they see it in their error messages - */ - String LOCAL_TARBALL_INSTALL_SUBDIR = "expandedarchive"; - - - /** - * Application type for YARN {@value} - */ - String APP_TYPE = "org-apache-slider"; - - /** - * A component type for an external app that has been predefined using the - * slider build command - */ - String COMPONENT_SEPARATOR = "-"; - - /** - * A component type for a client component - */ - String COMPONENT_TYPE_CLIENT = "client"; - - /** - * Key for application version. - */ - String APP_VERSION_UNKNOWN = "awaiting heartbeat..."; - - /** - * Keys for application container specific properties, like release timeout - */ - String APP_CONTAINER_RELEASE_TIMEOUT = "site.global.app_container.release_timeout_secs"; - - /** - * Subdirectories of HDFS cluster dir. - */ - String DATA_DIR_NAME = "data"; - String HISTORY_DIR_NAME = "history"; - String HISTORY_FILENAME_SUFFIX = "json"; - String HISTORY_FILENAME_PREFIX = "rolehistory-"; - String KEYTAB_DIR = "keytabs"; - String RESOURCE_DIR = "resources"; - - /** - * Filename pattern is required to save in strict temporal order. - * Important: older files must sort less-than newer files when using - * case-sensitive name sort. - */ - String HISTORY_FILENAME_CREATION_PATTERN = HISTORY_FILENAME_PREFIX +"%016x."+ - HISTORY_FILENAME_SUFFIX; - /** - * The posix regexp used to locate this - */ - String HISTORY_FILENAME_MATCH_PATTERN = HISTORY_FILENAME_PREFIX +"[0-9a-f]+\\."+ - HISTORY_FILENAME_SUFFIX; - /** - * The posix regexp used to locate this - */ - String HISTORY_FILENAME_GLOB_PATTERN = HISTORY_FILENAME_PREFIX +"*."+ - HISTORY_FILENAME_SUFFIX; - - String CLUSTER_DIRECTORY = "cluster"; - - /** - * JVM property to define the slider lib directory; - * this is set by the slider script: {@value} - */ - String PROPERTY_LIB_DIR = "slider.libdir"; - - /** - * name of generated dir for this conf: {@value} - */ - String SUBMITTED_CONF_DIR = "conf"; - - /** - * Slider AM log4j file name : {@value} - */ - String LOG4J_SERVER_PROP_FILENAME = "slideram-log4j.properties"; - - /** - * Log4j sysprop to name the resource :{@value} - */ - String SYSPROP_LOG4J_CONFIGURATION = "log4j.configuration"; - - /** - * sysprop for Slider AM log4j directory :{@value} - */ - String SYSPROP_LOG_DIR = "LOG_DIR"; - - /** - * name of the Slider client resource - * loaded when the service is loaded. - */ - String SLIDER_CLIENT_XML = "slider-client.xml"; - - /** - * The name of the resource to put on the classpath - */ - String SLIDER_SERVER_XML = "slider-server.xml"; - - String TMP_DIR_PREFIX = "tmp"; - - /** - * Store the default app definition, e.g. metainfo file or content of a folder - */ - String APP_DEF_DIR = "appdef"; - /** - * Store additional app defs - co-processors - */ - String ADDONS_DIR = "addons"; - - String SLIDER_JAR = "slider-core.jar"; - - String STDOUT_AM = "slider-out.txt"; - String STDERR_AM = "slider-err.txt"; - - String HADOOP_USER_NAME = "HADOOP_USER_NAME"; - - /** - * Name of the AM filter to use: {@value} - */ - String AM_FILTER_NAME = - "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer"; - - String YARN_CONTAINER_PATH = "/node/container/"; - - String APP_CONF_DIR = "conf"; - - String APP_LIB_DIR = "lib"; - - String OUT_FILE = "stdout.txt"; - String ERR_FILE = "stderr.txt"; - - String QUICK_LINKS = "quicklinks"; - - String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec"; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java deleted file mode 100644 index e881edf..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common; - -/** - * These are the keys that can be added to <code>conf/slider-client.xml</code>. - */ -public interface SliderXmlConfKeys { - String PREFIX_PROVIDER = "slider.provider"; - /** - * pattern to identify a provider - * {@value} - */ - String KEY_PROVIDER = PREFIX_PROVIDER + ".%s"; - - /** - * conf option set to point to where the config came from - * {@value} - */ - String KEY_TEMPLATE_ORIGIN = "slider.template.origin"; - - /** - * Original name for the default FS. This is still - * expected by applications deployed - */ - String FS_DEFAULT_NAME_CLASSIC = "fs.default.name"; - - /** - * Slider principal - */ - String KEY_KERBEROS_PRINCIPAL = "slider.kerberos.principal"; - - /** - * Name of the property for ACLs for Slider AM. - * {@value} - */ - String KEY_PROTOCOL_ACL = "slider.security.protocol.acl"; - - /** - * Limit on restarts for the AM - * {@value} - */ - String KEY_AM_RESTART_LIMIT = "slider.yarn.restart.limit"; - - /** - * queue name, by default let YARN pick the queue - */ - String KEY_YARN_QUEUE = "slider.yarn.queue"; - String DEFAULT_YARN_QUEUE = null; - - /** - * default priority - */ - String KEY_YARN_QUEUE_PRIORITY = "slider.yarn.queue.priority"; - int DEFAULT_YARN_QUEUE_PRIORITY = 1; - - - String KEY_AM_RESOURCE_MEM = "slider.am.resource.memory"; - long DEFAULT_KEY_AM_RESOURCE_MEM = 1024; - - /** - * The slider base path: {@value} - * Defaults to HomeDir/.slider - */ - String KEY_SLIDER_BASE_PATH = "slider.base.path"; - - - /** - * Option for the permissions for the cluster directory itself: {@value} - */ - String CLUSTER_DIRECTORY_PERMISSIONS = - "slider.cluster.directory.permissions"; - - /** - * Default value for the permissions :{@value} - */ - String DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS = "750"; - - /** - * - * Option for the permissions for the data directory itself: {@value} - */ - String DATA_DIRECTORY_PERMISSIONS = "slider.data.directory.permissions"; - - /** - * Default value for the data directory permissions: {@value} - */ - String DEFAULT_DATA_DIRECTORY_PERMISSIONS = "750"; - - String IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH = - "ipc.client.fallback-to-simple-auth-allowed"; - String HADOOP_HTTP_FILTER_INITIALIZERS = - "hadoop.http.filter.initializers"; - String KEY_KEYSTORE_LOCATION = "ssl.server.keystore.location"; - String KEY_AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name"; - /** Declare that a keytab must be provided */ - String KEY_AM_LOGIN_KEYTAB_REQUIRED = "slider.am.login.keytab.required"; - String KEY_HDFS_KEYTAB_DIR = "slider.hdfs.keytab.dir"; - String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path"; - String KEY_KEYTAB_PRINCIPAL = "slider.keytab.principal.name"; - String KEY_SECURITY_ENABLED = "site.global.security_enabled"; - - /** - * Set to disable server-side checks for python, openssl &c. - * This should only be set for testing - */ - String KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED = - "slider.am.dependency.checks.disabled"; - - /** - * The path to the python executable utilized to launch the agent. - */ - String PYTHON_EXECUTABLE_PATH = "agent.python.exec.path"; - - /** - * Flag to enable the insecure AM filter: {@value} - */ - String X_DEV_INSECURE_WS = "slider.feature.ws.insecure"; - - /** - * Flag to indicate the insecure AM filter is enabled by default: {@value}. - */ - boolean X_DEV_INSECURE_DEFAULT = false; - - - /** - * Flag to indicate the insecure AM filter is required for - * complex REST Verbs: {@value}. - * When Slider switches to being Hadoop 2.7+ only, this flag - * can be set to false - */ - boolean X_DEV_INSECURE_REQUIRED = true; - - /** - * - */ - String KEY_IPC_CLIENT_RETRY_POLICY_ENABLED = - "slider.ipc.client.retry.enabled"; - boolean IPC_CLIENT_RETRY_POLICY_ENABLED_DEFAULT = true; - String KEY_IPC_CLIENT_RETRY_POLICY_SPEC = - "slider.ipc.client.retry.policy.spec"; - String IPC_CLIENT_RETRY_POLICY_SPEC_DEFAULT = - "10000,6,60000,10"; //t1,n1,t2,n2,... - - String KEY_AM_LAUNCH_ENV = "slider.am.launch.env"; - - /** - * From {@code DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY} - */ - String DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY = "dfs.namenode.kerberos.principal"; - - String DFS_DATANODE_KERBEROS_PRINCIPAL_KEY = "dfs.datanode.kerberos.principal"; - - //Delegation token related keys - String DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY - = "dfs.namenode.delegation.key.update-interval"; - long DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = 24 * 60 * 60 * - 1000; // 1 day - String DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY - = "dfs.namenode.delegation.token.renew-interval"; - long DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 24 * 60 * 60 * - 1000; // 1 day - String DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY - = "dfs.namenode.delegation.token.max-lifetime"; - long DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7 * 24 * 60 * 60 * - 1000; // 7 days - String DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY - = "dfs.namenode.delegation.token.always-use"; // for tests - boolean DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT = false; - String DFS_NAMENODE_KEYTAB_FILE_KEY = "dfs.namenode.keytab.file"; - String DFS_NAMENODE_DU_RESERVED_KEY = "dfs.namenode.resource.du.reserved"; - - String MAPREDUCE_JOB_CREDENTIALS_BINARY = "mapreduce.job.credentials.binary"; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java deleted file mode 100644 index 6dd61fa..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.params; - -import com.beust.jcommander.Parameter; -import org.apache.hadoop.fs.Path; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.ErrorStrings; -import org.apache.slider.core.exceptions.UsageException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; - -/** - * Base args for all actions - */ -public abstract class AbstractActionArgs extends ArgOps implements Arguments { - protected static final Logger log = - LoggerFactory.getLogger(AbstractActionArgs.class); - - - protected AbstractActionArgs() { - } - - /** - * URI/binding to the filesystem - */ - @Parameter(names = {ARG_FILESYSTEM, ARG_FILESYSTEM_LONG}, - description = "Filesystem Binding") - public String filesystemBinding; - - @Parameter(names = {ARG_BASE_PATH}, - description = "Slider base path on the filesystem", - converter = PathArgumentConverter.class) - public Path basePath; - - /** - * This is the default parameter - */ - @Parameter - public final List<String> parameters = new ArrayList<>(); - - /** - * get the name: relies on arg 1 being the cluster name in all operations - * @return the name argument, null if there is none - */ - public String getClusterName() { - return (parameters.isEmpty()) ? null : parameters.get(0); - } - - /** - -D name=value - - Define an configuration option which overrides any options in - the configuration XML files of the image or in the image configuration - directory. The values will be persisted. - Configuration options are only passed to the cluster when creating or reconfiguring a cluster. - - */ - - @Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions") - public final List<String> definitions = new ArrayList<>(); - - /** - * System properties - */ - @Parameter(names = {ARG_SYSPROP}, arity = 1, - description = "system properties in the form name value" + - " These are set after the JVM is started.") - public final List<String> sysprops = new ArrayList<>(0); - - - @Parameter(names = {ARG_MANAGER_SHORT, ARG_MANAGER}, - description = "Binding (usually hostname:port) of the YARN resource manager") - public String manager; - - - @Parameter(names = ARG_DEBUG, description = "Debug mode") - public boolean debug = false; - - @Parameter(names = {ARG_HELP}, description = "Help", help = true) - public boolean help = false; - - /** - * Get the min #of params expected - * @return the min number of params in the {@link #parameters} field - */ - public int getMinParams() { - return 1; - } - - /** - * Get the name of the action - * @return the action name - */ - public abstract String getActionName() ; - - /** - * Get the max #of params expected - * @return the number of params in the {@link #parameters} field; - */ - public int getMaxParams() { - return getMinParams(); - } - - public void validate() throws BadCommandArgumentsException, UsageException { - - int minArgs = getMinParams(); - int actionArgSize = parameters.size(); - if (minArgs > actionArgSize) { - throw new BadCommandArgumentsException( - ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS + getActionName() + - " Expected minimum " + minArgs + " but got " + actionArgSize); - } - int maxArgs = getMaxParams(); - if (maxArgs == -1) { - maxArgs = minArgs; - } - if (actionArgSize > maxArgs) { - String message = String.format("%s for action %s: limit is %d but saw %d: ", - ErrorStrings.ERROR_TOO_MANY_ARGUMENTS, - getActionName(), maxArgs, - actionArgSize); - - log.error(message); - int index = 1; - StringBuilder buf = new StringBuilder(message); - for (String actionArg : parameters) { - log.error("[{}] \"{}\"", index++, actionArg); - buf.append(" \"" + actionArg + "\" "); - } - throw new BadCommandArgumentsException(buf.toString()); - } - } - - @Override - public String toString() { - return super.toString() + ": " + getActionName(); - } - - /** - * Override point: - * Flag to indicate that core hadoop API services are needed (HDFS, YARN, etc) - * âand that validation of the client state should take place. - * - * @return a flag to indicate that the core hadoop services will be needed. - */ - public boolean getHadoopServicesRequired() { - return true; - } - - /** - * Flag to disable secure login. - * This MUST only be set if the action is bypassing security or setting - * it itself - * @return true if login at slider client init time is to be skipped - */ - public boolean disableSecureLogin() { - return false; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractArgsDelegate.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractArgsDelegate.java index 23ba414..ec88ca1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractArgsDelegate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractArgsDelegate.java @@ -18,6 +18,8 @@ package org.apache.slider.common.params; +import org.apache.hadoop.yarn.service.client.params.ArgOps; +import org.apache.hadoop.yarn.service.client.params.Arguments; /** * Base class for all the delegates http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java index c983a63..57c27e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java @@ -21,6 +21,8 @@ package org.apache.slider.common.params; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParametersDelegate; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs; +import org.apache.hadoop.yarn.service.client.params.ComponentArgsDelegate; import org.apache.slider.core.exceptions.BadCommandArgumentsException; import java.io.File; http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java index 5b4cfdc..04ec9e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java @@ -20,8 +20,10 @@ package org.apache.slider.common.params; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs; +import org.apache.hadoop.yarn.service.client.params.SliderActions; -@Parameters(commandNames = {SliderActions.ACTION_AM_SUICIDE}, +@Parameters(commandNames = { SliderActions.ACTION_AM_SUICIDE}, commandDescription = SliderActions.DESCRIBE_ACTION_AM_SUICIDE) public class ActionAMSuicideArgs extends AbstractActionArgs { http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java deleted file mode 100644 index 57e4b02..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.common.params; - -import com.beust.jcommander.Parameters; - -@Parameters(commandNames = {SliderActions.ACTION_BUILD}, - commandDescription = SliderActions.DESCRIBE_ACTION_BUILD) - -public class ActionBuildArgs extends AbstractClusterBuildingActionArgs { - - @Override - public String getActionName() { - return SliderActions.ACTION_BUILD; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java index 09e2b62..8dfde36 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java @@ -20,10 +20,12 @@ package org.apache.slider.common.params; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs; +import org.apache.hadoop.yarn.service.client.params.SliderActions; import java.io.File; -@Parameters(commandNames = {SliderActions.ACTION_CLIENT}, +@Parameters(commandNames = { SliderActions.ACTION_CLIENT}, commandDescription = SliderActions.DESCRIBE_ACTION_CLIENT) public class ActionClientArgs extends AbstractActionArgs { http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java deleted file mode 100644 index 4cc1077..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.params; - -import com.beust.jcommander.Parameters; - -@Parameters(commandNames = {SliderActions.ACTION_CREATE}, - commandDescription = SliderActions.DESCRIBE_ACTION_CREATE) - -public class ActionCreateArgs extends AbstractClusterBuildingActionArgs { - - @Override - public String getActionName() { - return SliderActions.ACTION_CREATE; - } -} - http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionDependencyArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionDependencyArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionDependencyArgs.java deleted file mode 100644 index 87f9f0dc..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionDependencyArgs.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.common.params; - -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.UsageException; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; - -@Parameters(commandNames = { SliderActions.ACTION_DEPENDENCY }, - commandDescription = SliderActions.DESCRIBE_ACTION_DEPENDENCY) -public class ActionDependencyArgs extends AbstractActionArgs { - - @Override - public String getActionName() { - return SliderActions.ACTION_DEPENDENCY; - } - - @Parameter(names = { ARG_UPLOAD }, - description = "Upload AM and agent libraries to HDFS for this client") - public boolean upload; - - @Parameter(names = { ARG_OVERWRITE }, - description = "Overwrite current uploaded dependency libs") - public boolean overwrite = false; - - /** - * Get the min #of params expected - * - * @return the min number of params in the {@link #parameters} field - */ - public int getMinParams() { - return 0; - } - - @Override - public int getMaxParams() { - return 1; - } - - @Override - public void validate() throws BadCommandArgumentsException, UsageException { - super.validate(); - - if (!upload) { - throw new UsageException("Option " + ARG_UPLOAD + " is mandatory"); - } - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
