http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java deleted file mode 100644 index 405f690..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.utils; - -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.service.utils.ApplicationReportSerDeser; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.io.IOException; - -/** - * Serialized form of an application report which can be persisted - * and then parsed. It can not be converted back into a - * real YARN application report - * - * Useful for testing - */ - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) - -public class SerializedApplicationReport { - - public String applicationId; - public String applicationAttemptId; - public String name; - public String applicationType; - public String user; - public String queue; - public String host; - public Integer rpcPort; - public String state; - public String diagnostics; - public String url; - /** - * This value is non-null only when a report is generated from a submission context. - * The YARN {@link ApplicationReport} structure does not propagate this value - * from the RM. - */ - public Long submitTime; - public Long startTime; - public Long finishTime; - public String finalStatus; - public String origTrackingUrl; - public Float progress; - - public SerializedApplicationReport() { - } - - public SerializedApplicationReport(ApplicationReport report) { - this.applicationId = report.getApplicationId().toString(); - ApplicationAttemptId attemptId = report.getCurrentApplicationAttemptId(); - this.applicationAttemptId = attemptId != null ? attemptId.toString() : "N/A"; - this.name = report.getName(); - this.applicationType = report.getApplicationType(); - this.user = report.getUser(); - this.queue = report.getQueue(); - this.host = report.getHost(); - this.rpcPort = report.getRpcPort(); - this.state = report.getYarnApplicationState().toString(); - this.diagnostics = report.getDiagnostics(); - this.startTime = report.getStartTime(); - this.finishTime = report.getFinishTime(); - FinalApplicationStatus appStatus = report.getFinalApplicationStatus(); - this.finalStatus = appStatus == null ? "" : appStatus.toString(); - this.progress = report.getProgress(); - this.url = report.getTrackingUrl(); - this.origTrackingUrl= report.getOriginalTrackingUrl(); - } - - @Override - public String toString() { - try { - return ApplicationReportSerDeser.toString(this); - } catch (IOException e) { - return super.toString(); - } - } -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java deleted file mode 100644 index 21cb049..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java +++ /dev/null @@ -1,443 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.utils; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.registry.client.api.RegistryConstants; -import org.apache.hadoop.registry.client.binding.RegistryUtils; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.service.api.records.Application; -import org.apache.hadoop.yarn.service.api.records.Artifact; -import org.apache.hadoop.yarn.service.api.records.Component; -import org.apache.hadoop.yarn.service.api.records.Configuration; -import org.apache.hadoop.yarn.service.api.records.Resource; -import org.apache.hadoop.yarn.service.provider.AbstractClientProvider; -import org.apache.hadoop.yarn.service.provider.ProviderFactory; -import org.apache.hadoop.yarn.service.servicemonitor.probe.MonitorUtils; -import org.apache.hadoop.yarn.service.conf.RestApiConstants; -import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages; -import org.codehaus.jackson.map.PropertyNamingStrategy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class ServiceApiUtil { - private static final Logger LOG = - LoggerFactory.getLogger(ServiceApiUtil.class); - public static JsonSerDeser<Application> jsonSerDeser = - new JsonSerDeser<>(Application.class, - PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); - private static final PatternValidator compNamePattern - = new PatternValidator("[a-z][a-z0-9-]*"); - - @VisibleForTesting - public static void setJsonSerDeser(JsonSerDeser jsd) { - jsonSerDeser = jsd; - } - - @VisibleForTesting - public static void validateAndResolveApplication(Application application, - SliderFileSystem fs, org.apache.hadoop.conf.Configuration conf) throws - IOException { - boolean dnsEnabled = conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED, - RegistryConstants.DEFAULT_DNS_ENABLED); - if (dnsEnabled && RegistryUtils.currentUser().length() > RegistryConstants - .MAX_FQDN_LABEL_LENGTH) { - throw new IllegalArgumentException(RestApiErrorMessages - .ERROR_USER_NAME_INVALID); - } - if (StringUtils.isEmpty(application.getName())) { - throw new IllegalArgumentException( - RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID); - } - if (!SliderUtils.isClusternameValid(application.getName()) || (dnsEnabled - && application.getName().length() > RegistryConstants - .MAX_FQDN_LABEL_LENGTH)) { - throw new IllegalArgumentException(String.format( - RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID_FORMAT, - application.getName())); - } - - // If the application has no components do top-level checks - if (!hasComponent(application)) { - // If artifact is of type APPLICATION, read other application components - if (application.getArtifact() != null && application.getArtifact() - .getType() == Artifact.TypeEnum.APPLICATION) { - if (StringUtils.isEmpty(application.getArtifact().getId())) { - throw new IllegalArgumentException( - RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID); - } - Application otherApplication = loadApplication(fs, - application.getArtifact().getId()); - application.setComponents(otherApplication.getComponents()); - application.setArtifact(null); - SliderUtils.mergeMapsIgnoreDuplicateKeys(application.getQuicklinks(), - otherApplication.getQuicklinks()); - } else { - // Since it is a simple app with no components, create a default - // component - Component comp = createDefaultComponent(application); - validateComponent(comp, fs.getFileSystem()); - application.getComponents().add(comp); - if (application.getLifetime() == null) { - application.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME); - } - return; - } - } - - // Validate there are no component name collisions (collisions are not - // currently supported) and add any components from external applications - // TODO allow name collisions? see AppState#roles - // TODO or add prefix to external component names? - Configuration globalConf = application.getConfiguration(); - Set<String> componentNames = new HashSet<>(); - List<Component> componentsToRemove = new ArrayList<>(); - List<Component> componentsToAdd = new ArrayList<>(); - for (Component comp : application.getComponents()) { - int maxCompLength = RegistryConstants.MAX_FQDN_LABEL_LENGTH; - maxCompLength = maxCompLength - Long.toString(Long.MAX_VALUE).length(); - if (dnsEnabled && comp.getName().length() > maxCompLength) { - throw new IllegalArgumentException(String.format(RestApiErrorMessages - .ERROR_COMPONENT_NAME_INVALID, maxCompLength, comp.getName())); - } - if (componentNames.contains(comp.getName())) { - throw new IllegalArgumentException("Component name collision: " + - comp.getName()); - } - // If artifact is of type APPLICATION (which cannot be filled from - // global), read external application and add its components to this - // application - if (comp.getArtifact() != null && comp.getArtifact().getType() == - Artifact.TypeEnum.APPLICATION) { - if (StringUtils.isEmpty(comp.getArtifact().getId())) { - throw new IllegalArgumentException( - RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID); - } - LOG.info("Marking {} for removal", comp.getName()); - componentsToRemove.add(comp); - List<Component> externalComponents = getApplicationComponents(fs, - comp.getArtifact().getId()); - for (Component c : externalComponents) { - Component override = application.getComponent(c.getName()); - if (override != null && override.getArtifact() == null) { - // allow properties from external components to be overridden / - // augmented by properties in this component, except for artifact - // which must be read from external component - override.mergeFrom(c); - LOG.info("Merging external component {} from external {}", c - .getName(), comp.getName()); - } else { - if (componentNames.contains(c.getName())) { - throw new IllegalArgumentException("Component name collision: " + - c.getName()); - } - componentNames.add(c.getName()); - componentsToAdd.add(c); - LOG.info("Adding component {} from external {}", c.getName(), - comp.getName()); - } - } - } else { - // otherwise handle as a normal component - componentNames.add(comp.getName()); - // configuration - comp.getConfiguration().mergeFrom(globalConf); - } - } - application.getComponents().removeAll(componentsToRemove); - application.getComponents().addAll(componentsToAdd); - - // Validate components and let global values take effect if component level - // values are not provided - Artifact globalArtifact = application.getArtifact(); - Resource globalResource = application.getResource(); - Long globalNumberOfContainers = application.getNumberOfContainers(); - String globalLaunchCommand = application.getLaunchCommand(); - for (Component comp : application.getComponents()) { - // fill in global artifact unless it is type APPLICATION - if (comp.getArtifact() == null && application.getArtifact() != null - && application.getArtifact().getType() != Artifact.TypeEnum - .APPLICATION) { - comp.setArtifact(globalArtifact); - } - // fill in global resource - if (comp.getResource() == null) { - comp.setResource(globalResource); - } - // fill in global container count - if (comp.getNumberOfContainers() == null) { - comp.setNumberOfContainers(globalNumberOfContainers); - } - // fill in global launch command - if (comp.getLaunchCommand() == null) { - comp.setLaunchCommand(globalLaunchCommand); - } - // validate dependency existence - if (comp.getDependencies() != null) { - for (String dependency : comp.getDependencies()) { - if (!componentNames.contains(dependency)) { - throw new IllegalArgumentException(String.format( - RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, dependency, - comp.getName())); - } - } - } - validateComponent(comp, fs.getFileSystem()); - } - - // validate dependency tree - sortByDependencies(application.getComponents()); - - // Application lifetime if not specified, is set to unlimited lifetime - if (application.getLifetime() == null) { - application.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME); - } - } - - public static void validateComponent(Component comp, FileSystem fs) - throws IOException { - validateCompName(comp.getName()); - - AbstractClientProvider compClientProvider = ProviderFactory - .getClientProvider(comp.getArtifact()); - compClientProvider.validateArtifact(comp.getArtifact(), fs); - - if (comp.getLaunchCommand() == null && (comp.getArtifact() == null || comp - .getArtifact().getType() != Artifact.TypeEnum.DOCKER)) { - throw new IllegalArgumentException(RestApiErrorMessages - .ERROR_ABSENT_LAUNCH_COMMAND); - } - - validateApplicationResource(comp.getResource(), comp); - - if (comp.getNumberOfContainers() == null - || comp.getNumberOfContainers() < 0) { - throw new IllegalArgumentException(String.format( - RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID - + ": " + comp.getNumberOfContainers(), comp.getName())); - } - compClientProvider.validateConfigFiles(comp.getConfiguration() - .getFiles(), fs); - - MonitorUtils.getProbe(comp.getReadinessCheck()); - } - - // Check component name format and transform to lower case. - public static void validateCompName(String compName) { - if (StringUtils.isEmpty(compName)) { - throw new IllegalArgumentException("Component name can not be empty"); - } - // validate component name - if (compName.contains("_")) { - throw new IllegalArgumentException( - "Invalid format for component name: " + compName - + ", can not use '_' as DNS hostname does not allow underscore. Use '-' Instead. "); - } - compNamePattern.validate(compName); - } - - @VisibleForTesting - public static List<Component> getApplicationComponents(SliderFileSystem - fs, String appName) throws IOException { - return loadApplication(fs, appName).getComponents(); - } - - public static Application loadApplication(SliderFileSystem fs, String - appName) throws IOException { - Path appJson = getAppJsonPath(fs, appName); - LOG.info("Loading application definition from " + appJson); - return jsonSerDeser.load(fs.getFileSystem(), appJson); - } - - public static Application loadApplicationFrom(SliderFileSystem fs, - Path appDefPath) throws IOException { - LOG.info("Loading application definition from " + appDefPath); - return jsonSerDeser.load(fs.getFileSystem(), appDefPath); - } - - public static Path getAppJsonPath(SliderFileSystem fs, String appName) { - Path appDir = fs.buildClusterDirPath(appName); - Path appJson = new Path(appDir, appName + ".json"); - return appJson; - } - - private static void validateApplicationResource(Resource resource, - Component comp) { - // Only apps/components of type APPLICATION can skip resource requirement - if (resource == null) { - throw new IllegalArgumentException( - comp == null ? RestApiErrorMessages.ERROR_RESOURCE_INVALID : String - .format(RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID, - comp.getName())); - } - // One and only one of profile OR cpus & memory can be specified. Specifying - // both raises validation error. - if (StringUtils.isNotEmpty(resource.getProfile()) && ( - resource.getCpus() != null || StringUtils - .isNotEmpty(resource.getMemory()))) { - throw new IllegalArgumentException(comp == null ? - RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED : - String.format( - RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED, - comp.getName())); - } - // Currently resource profile is not supported yet, so we will raise - // validation error if only resource profile is specified - if (StringUtils.isNotEmpty(resource.getProfile())) { - throw new IllegalArgumentException( - RestApiErrorMessages.ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET); - } - - String memory = resource.getMemory(); - Integer cpus = resource.getCpus(); - if (StringUtils.isEmpty(memory)) { - throw new IllegalArgumentException( - comp == null ? RestApiErrorMessages.ERROR_RESOURCE_MEMORY_INVALID : - String.format( - RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID, - comp.getName())); - } - if (cpus == null) { - throw new IllegalArgumentException( - comp == null ? RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID : - String.format( - RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID, - comp.getName())); - } - if (cpus <= 0) { - throw new IllegalArgumentException(comp == null ? - RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID_RANGE : String - .format( - RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE, - comp.getName())); - } - } - - // check if comp mem size exceeds cluster limit - public static void validateCompResourceSize( - org.apache.hadoop.yarn.api.records.Resource maxResource, - Application application) throws YarnException { - for (Component component : application.getComponents()) { - // only handle mem now. - long mem = Long.parseLong(component.getResource().getMemory()); - if (mem > maxResource.getMemorySize()) { - throw new YarnException( - "Component " + component.getName() + " memory size (" + mem - + ") is larger than configured max container memory size (" - + maxResource.getMemorySize() + ")"); - } - } - } - - public static boolean hasComponent(Application application) { - if (application.getComponents() == null || application.getComponents() - .isEmpty()) { - return false; - } - return true; - } - - public static Component createDefaultComponent(Application app) { - Component comp = new Component(); - comp.setName(RestApiConstants.DEFAULT_COMPONENT_NAME); - comp.setArtifact(app.getArtifact()); - comp.setResource(app.getResource()); - comp.setNumberOfContainers(app.getNumberOfContainers()); - comp.setLaunchCommand(app.getLaunchCommand()); - comp.setConfiguration(app.getConfiguration()); - return comp; - } - - public static Collection<Component> sortByDependencies(List<Component> - components) { - Map<String, Component> sortedComponents = - sortByDependencies(components, null); - return sortedComponents.values(); - } - - /** - * Each internal call of sortByDependencies will identify all of the - * components with the same dependency depth (the lowest depth that has not - * been processed yet) and add them to the sortedComponents list, preserving - * their original ordering in the components list. - * - * So the first time it is called, all components with no dependencies - * (depth 0) will be identified. The next time it is called, all components - * that have dependencies only on the the depth 0 components will be - * identified (depth 1). This will be repeated until all components have - * been added to the sortedComponents list. If no new components are - * identified but the sortedComponents list is not complete, an error is - * thrown. - */ - private static Map<String, Component> sortByDependencies(List<Component> - components, Map<String, Component> sortedComponents) { - if (sortedComponents == null) { - sortedComponents = new LinkedHashMap<>(); - } - - Map<String, Component> componentsToAdd = new LinkedHashMap<>(); - List<Component> componentsSkipped = new ArrayList<>(); - for (Component component : components) { - String name = component.getName(); - if (sortedComponents.containsKey(name)) { - continue; - } - boolean dependenciesAlreadySorted = true; - if (!SliderUtils.isEmpty(component.getDependencies())) { - for (String dependency : component.getDependencies()) { - if (!sortedComponents.containsKey(dependency)) { - dependenciesAlreadySorted = false; - break; - } - } - } - if (dependenciesAlreadySorted) { - componentsToAdd.put(name, component); - } else { - componentsSkipped.add(component); - } - } - - if (componentsToAdd.size() == 0) { - throw new IllegalArgumentException(String.format(RestApiErrorMessages - .ERROR_DEPENDENCY_CYCLE, componentsSkipped)); - } - sortedComponents.putAll(componentsToAdd); - if (sortedComponents.size() == components.size()) { - return sortedComponents; - } - return sortByDependencies(components, sortedComponents); - } - - public static String $(String s) { - return "${" + s +"}"; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java deleted file mode 100644 index 7440b11..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.utils; - -import org.apache.hadoop.registry.client.binding.RegistryUtils; -import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; - - -public class ServiceRegistryUtils { - - /** - * Base path for services - */ - public static final String ZK_SERVICES = "services"; - - /** - * Base path for all Slider references - */ - public static final String ZK_SLIDER = "slider"; - public static final String ZK_USERS = "users"; - public static final String SVC_SLIDER = "/" + ZK_SERVICES + "/" + ZK_SLIDER; - public static final String SVC_SLIDER_USERS = SVC_SLIDER + "/" + ZK_USERS; - - /** - * Get the registry path for an instance under the user's home node - * @param instanceName application instance - * @return a path to the registry location for this application instance. - */ - public static String registryPathForInstance(String instanceName) { - return RegistryUtils.servicePath( - RegistryUtils.currentUser(), YarnServiceConstants.APP_TYPE, instanceName - ); - } - - /** - * Build the path to a cluster; exists once the cluster has come up. - * Even before that, a ZK watcher could wait for it. - * @param username user - * @param clustername name of the cluster - * @return a strin - */ - public static String mkClusterPath(String username, String clustername) { - return mkSliderUserPath(username) + "/" + clustername; - } - - /** - * Build the path to a cluster; exists once the cluster has come up. - * Even before that, a ZK watcher could wait for it. - * @param username user - * @return a string - */ - public static String mkSliderUserPath(String username) { - return SVC_SLIDER_USERS + "/" + username; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java deleted file mode 100644 index d6d664e..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.utils; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; - -import java.io.IOException; - -/** - * Extends Core Filesystem with operations to manipulate ClusterDescription - * persistent state - */ -public class SliderFileSystem extends CoreFileSystem { - - Path appDir = null; - - public SliderFileSystem(FileSystem fileSystem, - Configuration configuration) { - super(fileSystem, configuration); - } - - public SliderFileSystem(Configuration configuration) throws IOException { - super(configuration); - } - - public void setAppDir(Path appDir) { - this.appDir = appDir; - } - - public Path getAppDir() { - return this.appDir; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java deleted file mode 100644 index 415392a..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java +++ /dev/null @@ -1,1699 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.utils; - -import com.google.common.base.Preconditions; -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; -import org.apache.commons.lang.ArrayUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.GlobFilter; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.io.nativeio.NativeIO; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.ExitUtil; -import org.apache.hadoop.util.Shell; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.service.client.params.Arguments; -import org.apache.hadoop.yarn.service.client.params.SliderActions; -import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; -import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor; -import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException; -import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; -import org.apache.hadoop.yarn.service.exceptions.BadConfigException; -import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes; -import org.apache.hadoop.yarn.service.exceptions.SliderException; -import org.apache.zookeeper.server.util.KerberosUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.FilenameFilter; -import java.io.IOException; -import java.io.Serializable; -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.net.URL; -import java.net.URLDecoder; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.regex.Pattern; -import java.util.zip.GZIPOutputStream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -/** - * These are slider-specific Util methods - */ -public final class SliderUtils { - - private static final Logger log = LoggerFactory.getLogger(SliderUtils.class); - - /** - * Atomic bool to track whether or not process security has already been - * turned on (prevents re-entrancy) - */ - private static final AtomicBoolean processSecurityAlreadyInitialized = - new AtomicBoolean(false); - public static final String JAVA_SECURITY_KRB5_REALM = - "java.security.krb5.realm"; - public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc"; - - /** - * Winutils - */ - public static final String WINUTILS = "WINUTILS.EXE"; - /** - * name of openssl program - */ - public static final String OPENSSL = "openssl"; - - /** - * name of python program - */ - public static final String PYTHON = "python"; - - /** - * type of docker standalone application - */ - public static final String DOCKER = "docker"; - /** - * type of docker on yarn application - */ - public static final String DOCKER_YARN = "yarn_docker"; - - public static final int NODE_LIST_LIMIT = 10; - - private SliderUtils() { - } - - /** - * Implementation of set-ness, groovy definition of true/false for a string - * @param s string - * @return true iff the string is neither null nor empty - */ - public static boolean isUnset(String s) { - return s == null || s.isEmpty(); - } - - public static boolean isSet(String s) { - return !isUnset(s); - } - - public static boolean isEmpty(Collection l) { - return l == null || l.isEmpty(); - } - - /** - * Probe for a collection existing and not being empty - * @param l collection - * @return true if the reference is valid and it contains entries - */ - - public static boolean isNotEmpty(Collection l) { - return l != null && !l.isEmpty(); - } - - /** - * Probe for a map existing and not being empty - * @param m map - * @return true if the reference is valid and it contains map entries - */ - public static boolean isNotEmpty(Map m) { - return m != null && !m.isEmpty(); - } - - /* - * Validates whether num is an integer - * @param num - * @param msg the message to be shown in exception - */ - @SuppressWarnings("ResultOfMethodCallIgnored") - private static void validateNumber(String num, String msg) throws - BadConfigException { - try { - Integer.parseInt(num); - } catch (NumberFormatException nfe) { - throw new BadConfigException(msg + num); - } - } - - /* - * Translates the trailing JVM heapsize unit: g, G, m, M - * This assumes designated unit of 'm' - * @param heapsize - * @return heapsize in MB - */ - public static String translateTrailingHeapUnit(String heapsize) throws - BadConfigException { - String errMsg = "Bad heapsize: "; - if (heapsize.endsWith("m") || heapsize.endsWith("M")) { - String num = heapsize.substring(0, heapsize.length() - 1); - validateNumber(num, errMsg); - return num; - } - if (heapsize.endsWith("g") || heapsize.endsWith("G")) { - String num = heapsize.substring(0, heapsize.length() - 1) + "000"; - validateNumber(num, errMsg); - return num; - } - // check if specified heap size is a number - validateNumber(heapsize, errMsg); - return heapsize; - } - - /** - * recursive directory delete - * @param dir dir to delete - * @throws IOException on any problem - */ - public static void deleteDirectoryTree(File dir) throws IOException { - if (dir.exists()) { - if (dir.isDirectory()) { - log.info("Cleaning up {}", dir); - //delete the children - File[] files = dir.listFiles(); - if (files == null) { - throw new IOException("listfiles() failed for " + dir); - } - for (File file : files) { - log.info("deleting {}", file); - if (!file.delete()) { - log.warn("Unable to delete " + file); - } - } - if (!dir.delete()) { - log.warn("Unable to delete " + dir); - } - } else { - throw new IOException("Not a directory " + dir); - } - } else { - //not found, do nothing - log.debug("No output dir yet"); - } - } - - /** - * Find a containing JAR - * @param clazz class to find - * @return the file - * @throws IOException any IO problem, including the class not having a - * classloader - * @throws FileNotFoundException if the class did not resolve to a file - */ - public static File findContainingJarOrFail(Class clazz) throws IOException { - File localFile = SliderUtils.findContainingJar(clazz); - if (null == localFile) { - throw new FileNotFoundException("Could not find JAR containing " + clazz); - } - return localFile; - } - - - /** - * Find a containing JAR - * @param my_class class to find - * @return the file or null if it is not found - * @throws IOException any IO problem, including the class not having a - * classloader - */ - public static File findContainingJar(Class my_class) throws IOException { - ClassLoader loader = my_class.getClassLoader(); - if (loader == null) { - throw new IOException( - "Class " + my_class + " does not have a classloader!"); - } - String class_file = my_class.getName().replaceAll("\\.", "/") + ".class"; - Enumeration<URL> urlEnumeration = loader.getResources(class_file); - for (; urlEnumeration.hasMoreElements(); ) { - URL url = urlEnumeration.nextElement(); - if ("jar".equals(url.getProtocol())) { - String toReturn = url.getPath(); - if (toReturn.startsWith("file:")) { - toReturn = toReturn.substring("file:".length()); - } - // URLDecoder is a misnamed class, since it actually decodes - // x-www-form-urlencoded MIME type rather than actual - // URL encoding (which the file path has). Therefore it would - // decode +s to ' 's which is incorrect (spaces are actually - // either unencoded or encoded as "%20"). Replace +s first, so - // that they are kept sacred during the decoding process. - toReturn = toReturn.replaceAll("\\+", "%2B"); - toReturn = URLDecoder.decode(toReturn, "UTF-8"); - String jarFilePath = toReturn.replaceAll("!.*$", ""); - return new File(jarFilePath); - } else { - log.info("could not locate JAR containing {} URL={}", my_class, url); - } - } - return null; - } - - public static void checkPort(String hostname, int port, int connectTimeout) - throws IOException { - InetSocketAddress addr = new InetSocketAddress(hostname, port); - checkPort(hostname, addr, connectTimeout); - } - - @SuppressWarnings("SocketOpenedButNotSafelyClosed") - public static void checkPort(String name, - InetSocketAddress address, - int connectTimeout) - throws IOException { - try(Socket socket = new Socket()) { - socket.connect(address, connectTimeout); - } catch (Exception e) { - throw new IOException("Failed to connect to " + name - + " at " + address - + " after " + connectTimeout + "milliseconds" - + ": " + e, - e); - } - } - - public static void checkURL(String name, String url, int timeout) throws - IOException { - InetSocketAddress address = NetUtils.createSocketAddr(url); - checkPort(name, address, timeout); - } - - /** - * A required file - * @param role role of the file (for errors) - * @param filename the filename - * @throws ExitUtil.ExitException if the file is missing - * @return the file - */ - public static File requiredFile(String filename, String role) throws - IOException { - if (filename.isEmpty()) { - throw new ExitUtil.ExitException(-1, role + " file not defined"); - } - File file = new File(filename); - if (!file.exists()) { - throw new ExitUtil.ExitException(-1, - role + " file not found: " + - file.getCanonicalPath()); - } - return file; - } - - private static final PatternValidator clusternamePattern - = new PatternValidator("[a-z][a-z0-9_-]*"); - - private static final PatternValidator compNamePattern - = new PatternValidator("[a-z][a-z0-9-]*"); - - public static void validateCompName(String compName) { - compNamePattern.validate(compName); - } - - /** - * Normalize a cluster name then verify that it is valid - * @param name proposed cluster name - * @return true iff it is valid - */ - public static boolean isClusternameValid(String name) { - return name != null && clusternamePattern.matches(name); - } - - public static boolean oldIsClusternameValid(String name) { - if (name == null || name.isEmpty()) { - return false; - } - int first = name.charAt(0); - if (0 == (Character.getType(first) & Character.LOWERCASE_LETTER)) { - return false; - } - - for (int i = 0; i < name.length(); i++) { - int elt = (int) name.charAt(i); - int t = Character.getType(elt); - if (0 == (t & Character.LOWERCASE_LETTER) - && 0 == (t & Character.DECIMAL_DIGIT_NUMBER) - && elt != '-' - && elt != '_') { - return false; - } - if (!Character.isLetterOrDigit(elt) && elt != '-' && elt != '_') { - return false; - } - } - return true; - } - - /** - * Copy a directory to a new FS -both paths must be qualified. If - * a directory needs to be created, supplied permissions can override - * the default values. Existing directories are not touched - * @param conf conf file - * @param srcDirPath src dir - * @param destDirPath dest dir - * @param permission permission for the dest directory; null means "default" - * @return # of files copies - */ - @SuppressWarnings("deprecation") - public static int copyDirectory(Configuration conf, - Path srcDirPath, - Path destDirPath, - FsPermission permission) throws - IOException, - BadClusterStateException { - FileSystem srcFS = FileSystem.get(srcDirPath.toUri(), conf); - FileSystem destFS = FileSystem.get(destDirPath.toUri(), conf); - //list all paths in the src. - if (!srcFS.exists(srcDirPath)) { - throw new FileNotFoundException("Source dir not found " + srcDirPath); - } - if (!srcFS.isDirectory(srcDirPath)) { - throw new FileNotFoundException( - "Source dir not a directory " + srcDirPath); - } - GlobFilter dotFilter = new GlobFilter("[!.]*"); - FileStatus[] entries = srcFS.listStatus(srcDirPath, dotFilter); - int srcFileCount = entries.length; - if (srcFileCount == 0) { - return 0; - } - if (permission == null) { - permission = FsPermission.getDirDefault(); - } - if (!destFS.exists(destDirPath)) { - new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath, - permission); - } - Path[] sourcePaths = new Path[srcFileCount]; - for (int i = 0; i < srcFileCount; i++) { - FileStatus e = entries[i]; - Path srcFile = e.getPath(); - if (srcFS.isDirectory(srcFile)) { - String msg = "Configuration dir " + srcDirPath - + " contains a directory " + srcFile; - log.warn(msg); - throw new IOException(msg); - } - log.debug("copying src conf file {}", srcFile); - sourcePaths[i] = srcFile; - } - log.debug("Copying {} files from {} to dest {}", srcFileCount, - srcDirPath, - destDirPath); - FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf); - return srcFileCount; - } - - /** - * Copy a file to a new FS -both paths must be qualified. - * @param conf conf file - * @param srcFile src file - * @param destFile dest file - */ - @SuppressWarnings("deprecation") - public static void copy(Configuration conf, - Path srcFile, - Path destFile) throws - IOException, - BadClusterStateException { - FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf); - //list all paths in the src. - if (!srcFS.exists(srcFile)) { - throw new FileNotFoundException("Source file not found " + srcFile); - } - if (!srcFS.isFile(srcFile)) { - throw new FileNotFoundException( - "Source file not a file " + srcFile); - } - FileSystem destFS = FileSystem.get(destFile.toUri(), conf); - FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf); - } - - /** - * Take a collection, return a list containing the string value of every - * element in the collection. - * @param c collection - * @return a stringified list - */ - public static List<String> collectionToStringList(Collection c) { - List<String> l = new ArrayList<>(c.size()); - for (Object o : c) { - l.add(o.toString()); - } - return l; - } - - /** - * Join an collection of objects with a separator that appears after every - * instance in the list -including at the end - * @param collection collection to call toString() on each element - * @param separator separator string - * @return the joined entries - */ - public static String join(Collection collection, String separator) { - return join(collection, separator, true); - } - - /** - * Join an collection of objects with a separator that appears after every - * instance in the list -optionally at the end - * @param collection collection to call toString() on each element - * @param separator separator string - * @param trailing add a trailing entry or not - * @return the joined entries - */ - public static String join(Collection collection, - String separator, - boolean trailing) { - StringBuilder b = new StringBuilder(); - // fast return on empty collection - if (collection.isEmpty()) { - return trailing ? separator : ""; - } - for (Object o : collection) { - b.append(o); - b.append(separator); - } - int length = separator.length(); - String s = b.toString(); - return (trailing || s.isEmpty()) ? - s : (b.substring(0, b.length() - length)); - } - - /** - * Join an array of strings with a separator that appears after every - * instance in the list -including at the end - * @param collection strings - * @param separator separator string - * @return the joined entries - */ - public static String join(String[] collection, String separator) { - return join(collection, separator, true); - - - } - - /** - * Join an array of strings with a separator that appears after every - * instance in the list -optionally at the end - * @param collection strings - * @param separator separator string - * @param trailing add a trailing entry or not - * @return the joined entries - */ - public static String join(String[] collection, String separator, - boolean trailing) { - return join(Arrays.asList(collection), separator, trailing); - } - - /** - * Join an array of strings with a separator that appears after every - * instance in the list -except at the end - * @param collection strings - * @param separator separator string - * @return the list - */ - public static String joinWithInnerSeparator(String separator, - Object... collection) { - StringBuilder b = new StringBuilder(); - boolean first = true; - - for (Object o : collection) { - if (first) { - first = false; - } else { - b.append(separator); - } - b.append(o.toString()); - b.append(separator); - } - return b.toString(); - } - - /** - * Resolve a mandatory environment variable - * @param key env var - * @return the resolved value - * @throws BadClusterStateException - */ - public static String mandatoryEnvVariable(String key) throws - BadClusterStateException { - String v = System.getenv(key); - if (v == null) { - throw new BadClusterStateException("Missing Environment variable " + key); - } - return v; - } - - public static String appReportToString(ApplicationReport r, - String separator) { - StringBuilder builder = new StringBuilder(512); - builder.append("application ") - .append( - r.getName()) - .append("/") - .append(r.getApplicationType()) - .append(separator); - Set<String> tags = r.getApplicationTags(); - if (!tags.isEmpty()) { - for (String tag : tags) { - builder.append(tag).append(separator); - } - } - DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm"); - dateFormat.setTimeZone(TimeZone.getDefault()); - builder.append("state: ").append(r.getYarnApplicationState()); - String trackingUrl = r.getTrackingUrl(); - if (isSet(trackingUrl)) { - builder.append(separator).append("URL: ").append(trackingUrl); - } - builder.append(separator) - .append("Started: ") - .append(dateFormat.format(new Date(r.getStartTime()))); - long finishTime = r.getFinishTime(); - if (finishTime > 0) { - builder.append(separator) - .append("Finished: ") - .append(dateFormat.format(new Date(finishTime))); - } - String rpcHost = r.getHost(); - if (!isSet(rpcHost)) { - builder.append(separator) - .append("RPC :") - .append(rpcHost) - .append(':') - .append(r.getRpcPort()); - } - String diagnostics = r.getDiagnostics(); - if (!isSet(diagnostics)) { - builder.append(separator).append("Diagnostics :").append(diagnostics); - } - return builder.toString(); - } - - /** - * Filter a string value given a single filter - * - * @param value - * the string value to check - * @param filter - * a single string filter - * @return return true if value should be trapped, false if it should be let - * through - */ - public static boolean filter(String value, String filter) { - return !(StringUtils.isEmpty(filter) || filter.equals(value)); - } - - /** - * Filter a string value given a set of filters - * - * @param value - * the string value to check - * @param filters - * a set of string filters - * @return return true if value should be trapped, false if it should be let - * through - */ - public static boolean filter(String value, Set<String> filters) { - return !(filters.isEmpty() || filters.contains(value)); - } - - /** - * Sorts the given list of application reports, most recently started - * or finished instance first. - * - * @param instances list of instances - */ - public static void sortApplicationsByMostRecent(List<ApplicationReport> instances) { - Collections.sort(instances, new MostRecentlyStartedOrFinishedFirst()); - } - - /** - * Sorts the given list of application reports - * Finished instances are ordered by finished time and running/accepted instances are - * ordered by start time - * Finally Instance are order by finished instances coming after running instances - * - * @param instances list of instances - */ - public static void sortApplicationReport(List<ApplicationReport> instances) { - if (instances.size() <= 1) { - return; - } - List<ApplicationReport> nonLiveInstance = - new ArrayList<>(instances.size()); - List<ApplicationReport> liveInstance = - new ArrayList<>(instances.size()); - - for (ApplicationReport report : instances) { - if (report.getYarnApplicationState() == YarnApplicationState.RUNNING - || - report.getYarnApplicationState() == YarnApplicationState.ACCEPTED) { - liveInstance.add(report); - } else { - nonLiveInstance.add(report); - } - } - - if (liveInstance.size() > 1) { - Collections.sort(liveInstance, new MostRecentlyStartedAppFirst()); - } - if (nonLiveInstance.size() > 1) { - Collections.sort(nonLiveInstance, new MostRecentAppFinishFirst()); - } - instances.clear(); - instances.addAll(liveInstance); - instances.addAll(nonLiveInstance); - } - - /** - * Merge in one map to another -all entries in the second map are - * merged into the first -overwriting any duplicate keys. - * @param first first map -the updated one. - * @param second the map that is merged in - * @return the first map - */ - public static Map<String, String> mergeMap(Map<String, String> first, - Map<String, String> second) { - first.putAll(second); - return first; - } - - /** - * Merge a set of entries into a map. This will take the entryset of - * a map, or a Hadoop collection itself - * @param dest destination - * @param entries entries - * @return dest -with the entries merged in - */ - public static Map<String, String> mergeEntries(Map<String, String> dest, - Iterable<Map.Entry<String, String>> entries) { - for (Map.Entry<String, String> entry : entries) { - dest.put(entry.getKey(), entry.getValue()); - } - return dest; - } - - /** - * Generic map merge logic - * @param first first map - * @param second second map - * @param <T1> key type - * @param <T2> value type - * @return 'first' merged with the second - */ - public static <T1, T2> Map<T1, T2> mergeMaps(Map<T1, T2> first, - Map<T1, T2> second) { - first.putAll(second); - return first; - } - - /** - * Generic map merge logic - * @param first first map - * @param second second map - * @param <T1> key type - * @param <T2> value type - * @return 'first' merged with the second - */ - public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first, - Map<T1, T2> second) { - Preconditions.checkArgument(first != null, "Null 'first' value"); - Preconditions.checkArgument(second != null, "Null 'second' value"); - for (Map.Entry<T1, T2> entry : second.entrySet()) { - T1 key = entry.getKey(); - if (!first.containsKey(key)) { - first.put(key, entry.getValue()); - } - } - return first; - } - - /** - * Convert a map to a multi-line string for printing - * @param map map to stringify - * @return a string representation of the map - */ - public static String stringifyMap(Map<String, String> map) { - StringBuilder builder = new StringBuilder(); - for (Map.Entry<String, String> entry : map.entrySet()) { - builder.append(entry.getKey()) - .append("=\"") - .append(entry.getValue()) - .append("\"\n"); - - } - return builder.toString(); - } - - /** - * Parse an int value, replacing it with defval if undefined; - * @param errorKey key to use in exceptions - * @param defVal default value to use if the key is not in the map - * @param min min value or -1 for do not check - * @param max max value or -1 for do not check - * @return the int value the integer value - * @throws BadConfigException if the value could not be parsed - */ - public static int parseAndValidate(String errorKey, - String valS, - int defVal, - int min, int max) throws - BadConfigException { - if (valS == null) { - valS = Integer.toString(defVal); - } - String trim = valS.trim(); - int val; - try { - val = Integer.decode(trim); - } catch (NumberFormatException e) { - throw new BadConfigException("Failed to parse value of " - + errorKey + ": \"" + trim + "\""); - } - if (min >= 0 && val < min) { - throw new BadConfigException("Value of " - + errorKey + ": " + val + "" - + "is less than the minimum of " + min); - } - if (max >= 0 && val > max) { - throw new BadConfigException("Value of " - + errorKey + ": " + val + "" - + "is more than the maximum of " + max); - } - return val; - } - - public static InetSocketAddress getRmAddress(Configuration conf) { - return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS, - YarnConfiguration.DEFAULT_RM_ADDRESS, - YarnConfiguration.DEFAULT_RM_PORT); - } - - public static InetSocketAddress getRmSchedulerAddress(Configuration conf) { - return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, - YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, - YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); - } - - /** - * probe to see if the RM scheduler is defined - * @param conf config - * @return true if the RM scheduler address is set to - * something other than 0.0.0.0 - */ - public static boolean isRmSchedulerAddressDefined(Configuration conf) { - InetSocketAddress address = getRmSchedulerAddress(conf); - return isAddressDefined(address); - } - - /** - * probe to see if the address - * @param address network address - * @return true if the scheduler address is set to - * something other than 0.0.0.0 - */ - public static boolean isAddressDefined(InetSocketAddress address) { - if (address == null || address.getHostString() == null) { - return false; - } - return !(address.getHostString().equals("0.0.0.0")); - } - - public static void setRmAddress(Configuration conf, String rmAddr) { - conf.set(YarnConfiguration.RM_ADDRESS, rmAddr); - } - - public static void setRmSchedulerAddress(Configuration conf, String rmAddr) { - conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr); - } - - public static boolean hasAppFinished(ApplicationReport report) { - return report == null || - report.getYarnApplicationState().ordinal() >= - YarnApplicationState.FINISHED.ordinal(); - } - - public static String containerToString(Container container) { - if (container == null) { - return "null container"; - } - return String.format(Locale.ENGLISH, - "ContainerID=%s nodeID=%s http=%s priority=%s resource=%s", - container.getId(), - container.getNodeId(), - container.getNodeHttpAddress(), - container.getPriority(), - container.getResource()); - } - - /** - * convert an AM report to a string for diagnostics - * @param report the report - * @return the string value - */ - public static String reportToString(ApplicationReport report) { - if (report == null) { - return "Null application report"; - } - - return "App " + report.getName() + "/" + report.getApplicationType() + - "# " + - report.getApplicationId() + " user " + report.getUser() + - " is in state " + report.getYarnApplicationState() + - " RPC: " + report.getHost() + ":" + report.getRpcPort() + - " URL: " + report.getOriginalTrackingUrl(); - } - - /** - * Convert a YARN URL into a string value of a normal URL - * @param url URL - * @return string representatin - */ - public static String stringify(org.apache.hadoop.yarn.api.records.URL url) { - StringBuilder builder = new StringBuilder(); - builder.append(url.getScheme()).append("://"); - if (url.getHost() != null) { - builder.append(url.getHost()).append(":").append(url.getPort()); - } - builder.append(url.getFile()); - return builder.toString(); - } - - /** - * Get a random open port - * @return true if the port was available for listening on - */ - public static int getOpenPort() throws IOException { - ServerSocket socket = null; - try { - socket = new ServerSocket(0); - return socket.getLocalPort(); - } finally { - if (socket != null) { - socket.close(); - } - } - } - - /** - * See if a port is available for listening on by trying to listen - * on it and seeing if that works or fails. - * @param port port to listen to - * @return true if the port was available for listening on - */ - public static boolean isPortAvailable(int port) { - try { - ServerSocket socket = new ServerSocket(port); - socket.close(); - return true; - } catch (IOException e) { - return false; - } - } - - // Build env map: key -> value; - // value will be replaced by the corresponding value in tokenMap, if any. - public static Map<String, String> buildEnvMap( - org.apache.hadoop.yarn.service.api.records.Configuration conf, - Map<String,String> tokenMap) { - if (tokenMap == null) { - return conf.getEnv(); - } - Map<String, String> env = new HashMap<>(); - for (Map.Entry<String, String> entry : conf.getEnv().entrySet()) { - String key = entry.getKey(); - String val = entry.getValue(); - for (Map.Entry<String,String> token : tokenMap.entrySet()) { - val = val.replaceAll(Pattern.quote(token.getKey()), - token.getValue()); - } - env.put(key,val); - } - return env; - } - - /** - * Apply a set of command line options to a cluster role map - * @param clusterRoleMap cluster role map to merge onto - * @param commandOptions command opts - */ - public static void applyCommandLineRoleOptsToRoleMap( - Map<String, Map<String, String>> clusterRoleMap, - Map<String, Map<String, String>> commandOptions) { - for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) { - String key = entry.getKey(); - Map<String, String> optionMap = entry.getValue(); - Map<String, String> existingMap = clusterRoleMap.get(key); - if (existingMap == null) { - existingMap = new HashMap<String, String>(); - } - log.debug("Overwriting role options with command line values {}", - stringifyMap(optionMap)); - mergeMap(existingMap, optionMap); - //set or overwrite the role - clusterRoleMap.put(key, existingMap); - } - } - - /** - * verify that the supplied cluster name is valid - * @param clustername cluster name - * @throws BadCommandArgumentsException if it is invalid - */ - public static void validateClusterName(String clustername) - throws BadCommandArgumentsException { - if (!isClusternameValid(clustername)) { - throw new BadCommandArgumentsException( - "Illegal cluster name: " + clustername); - } - } - - /** - * Verify that a Kerberos principal has been set -if not fail - * with an error message that actually tells you what is missing - * @param conf configuration to look at - * @param principal key of principal - * @throws BadConfigException if the key is not set - */ - public static void verifyPrincipalSet(Configuration conf, - String principal) throws - BadConfigException { - String principalName = conf.get(principal); - if (principalName == null) { - throw new BadConfigException("Unset Kerberos principal : %s", - principal); - } - log.debug("Kerberos princial {}={}", principal, principalName); - } - - /** - * Flag to indicate whether the cluster is in secure mode - * @param conf configuration to look at - * @return true if the slider client/service should be in secure mode - */ - public static boolean isHadoopClusterSecure(Configuration conf) { - return SecurityUtil.getAuthenticationMethod(conf) != - UserGroupInformation.AuthenticationMethod.SIMPLE; - } - - /** - * Init security if the cluster configuration declares the cluster is secure - * @param conf configuration to look at - * @return true if the cluster is secure - * @throws IOException cluster is secure - * @throws SliderException the configuration/process is invalid - */ - public static boolean maybeInitSecurity(Configuration conf) throws - IOException, - SliderException { - boolean clusterSecure = isHadoopClusterSecure(conf); - if (clusterSecure) { - log.debug("Enabling security"); - initProcessSecurity(conf); - } - return clusterSecure; - } - - /** - * Turn on security. This is setup to only run once. - * @param conf configuration to build up security - * @return true if security was initialized in this call - * @throws IOException IO/Net problems - * @throws BadConfigException the configuration and system state are inconsistent - */ - public static boolean initProcessSecurity(Configuration conf) throws - IOException, - SliderException { - - if (processSecurityAlreadyInitialized.compareAndSet(true, true)) { - //security is already inited - return false; - } - - log.info("JVM initialized into secure mode with kerberos realm {}", - SliderUtils.getKerberosRealm()); - //this gets UGI to reset its previous world view (i.e simple auth) - //security - log.debug("java.security.krb5.realm={}", - System.getProperty(JAVA_SECURITY_KRB5_REALM, "")); - log.debug("java.security.krb5.kdc={}", - System.getProperty(JAVA_SECURITY_KRB5_KDC, "")); - log.debug("hadoop.security.authentication={}", - conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION)); - log.debug("hadoop.security.authorization={}", - conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION)); - UserGroupInformation.setConfiguration(conf); - UserGroupInformation authUser = UserGroupInformation.getCurrentUser(); - log.debug("Authenticating as {}", authUser); - log.debug("Login user is {}", UserGroupInformation.getLoginUser()); - if (!UserGroupInformation.isSecurityEnabled()) { - throw new SliderException(LauncherExitCodes.EXIT_UNAUTHORIZED, - "Although secure mode is enabled," + - "the application has already set up its user as an insecure entity %s", - authUser); - } - if (authUser.getAuthenticationMethod() == - UserGroupInformation.AuthenticationMethod.SIMPLE) { - throw new BadConfigException("Auth User is not Kerberized %s" + - " -security has already been set up with the wrong authentication method. " - + "This can occur if a file system has already been created prior to the loading of " - + "the security configuration.", - authUser); - - } - - SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL); - SliderUtils.verifyPrincipalSet(conf, "dfs.namenode.kerberos.principal"); - return true; - } - - /** - * Force an early login: This catches any auth problems early rather than - * in RPC operations - * @throws IOException if the login fails - */ - public static void forceLogin() throws IOException { - if (UserGroupInformation.isSecurityEnabled()) { - if (UserGroupInformation.isLoginKeytabBased()) { - UserGroupInformation.getLoginUser().reloginFromKeytab(); - } else { - UserGroupInformation.getLoginUser().reloginFromTicketCache(); - } - } - } - - public static String getLibDir() { - String[] libDirs = getLibDirs(); - if (libDirs == null || libDirs.length == 0) { - return null; - } - return libDirs[0]; - } - - public static String[] getLibDirs() { - String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR); - if (isUnset(libDirStr)) { - return ArrayUtils.EMPTY_STRING_ARRAY; - } - return StringUtils.split(libDirStr, ','); - } - - /** - * Submit a JAR containing a specific class and map it - * @param providerResources provider map to build up - * @param sliderFileSystem remote fs - * @param clazz class to look for - * @param libdir lib directory - * @param jarName <i>At the destination</i> - * @return the local resource ref - * @throws IOException trouble copying to HDFS - */ - public static LocalResource putJar(Map<String, LocalResource> providerResources, - SliderFileSystem sliderFileSystem, - Class clazz, - Path tempPath, - String libdir, - String jarName - ) - throws IOException, SliderException { - LocalResource res = sliderFileSystem.submitJarWithClass( - clazz, - tempPath, - libdir, - jarName); - providerResources.put(libdir + "/" + jarName, res); - return res; - } - - /** - * Submit a JAR containing and map it - * @param providerResources provider map to build up - * @param sliderFileSystem remote fs - * @param libDir lib directory - * @param srcPath copy jars from - */ - public static void putAllJars(Map<String, LocalResource> providerResources, - SliderFileSystem sliderFileSystem, - Path tempPath, - String libDir, - String srcPath) throws IOException, SliderException { - log.info("Loading all dependencies from {}", srcPath); - if (SliderUtils.isSet(srcPath)) { - File srcFolder = new File(srcPath); - FilenameFilter jarFilter = createJarFilter(); - File[] listOfJars = srcFolder.listFiles(jarFilter); - if (listOfJars == null || listOfJars.length == 0) { - return; - } - for (File jarFile : listOfJars) { - LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName()); - providerResources.put(libDir + "/" + jarFile.getName(), res); - } - } - } - - /** - * Accept all filenames ending with {@code .jar} - * @return a filename filter - */ - public static FilenameFilter createJarFilter() { - return new FilenameFilter() { - public boolean accept(File dir, String name) { - return name.toLowerCase(Locale.ENGLISH).endsWith(".jar"); - } - }; - } - - /** - * Submit the AM tar.gz containing all dependencies and map it - * @param providerResources provider map to build up - * @param sliderFileSystem remote fs - */ - public static void putAmTarGzipAndUpdate( - Map<String, LocalResource> providerResources, - SliderFileSystem sliderFileSystem - ) throws IOException, SliderException { - log.info("Loading all dependencies from {}{}", - YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME, - YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT); - sliderFileSystem.submitTarGzipAndUpdate(providerResources); - } - - public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) { - Map<String, Map<String, String>> dest = new HashMap<>(); - for (Map.Entry<String, Map<String, String>> entry : src.entrySet()) { - dest.put(entry.getKey(), stringMapClone(entry.getValue())); - } - return dest; - } - - public static Map<String, String> stringMapClone(Map<String, String> src) { - Map<String, String> dest = new HashMap<>(); - return mergeEntries(dest, src.entrySet()); - } - - /** - * List a directory in the local filesystem - * @param dir directory - * @return a listing, one to a line - */ - public static String listDir(File dir) { - if (dir == null) { - return ""; - } - String[] confDirEntries = dir.list(); - if (confDirEntries == null) { - return ""; - } - StringBuilder builder = new StringBuilder(); - for (String entry : confDirEntries) { - builder.append(entry).append("\n"); - } - return builder.toString(); - } - - /** - * Create a file:// path from a local file - * @param file file to point the path - * @return a new Path - */ - public static Path createLocalPath(File file) { - return new Path(file.toURI()); - } - - public static String getKerberosRealm() { - try { - return KerberosUtil.getDefaultRealm(); - } catch (Exception e) { - log.debug("introspection into JVM internals failed", e); - return "(unknown)"; - - } - } - - /** - * Build up the classpath for execution - * -behaves very differently on a mini test cluster vs a production - * production one. - * - * @param sliderConfDir relative path to the dir containing slider config - * options to put on the classpath -or null - * @param libdir directory containing the JAR files - * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use - * (and hence the current classpath should be used, not anything built up) - * @return a classpath - */ - public static ClasspathConstructor buildClasspath(String sliderConfDir, - String libdir, - SliderFileSystem sliderFileSystem, - boolean usingMiniMRCluster) { - - ClasspathConstructor classpath = new ClasspathConstructor(); - classpath.append(YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME); - - // add the runtime classpath needed for tests to work - if (usingMiniMRCluster) { - // for mini cluster we pass down the java CP properties - // and nothing else - classpath.appendAll(classpath.localJVMClasspath()); - } else { - if (sliderConfDir != null) { - classpath.addClassDirectory(sliderConfDir); - } - classpath.addLibDir(libdir); - if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) { - classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK); - } else { - log.info( - "For faster submission of apps, upload dependencies using cmd {} {}", - SliderActions.ACTION_DEPENDENCY, Arguments.ARG_UPLOAD); - } - classpath.addRemoteClasspathEnvVar(); - classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$()); - } - return classpath; - } - - /** - * Verify that a path refers to a directory. If not - * logs the parent dir then throws an exception - * @param dir the directory - * @param errorlog log for output on an error - * @throws FileNotFoundException if it is not a directory - */ - public static void verifyIsDir(File dir, Logger errorlog) throws - FileNotFoundException { - if (!dir.exists()) { - errorlog.warn("contents of {}: {}", dir, - listDir(dir.getParentFile())); - throw new FileNotFoundException(dir.toString()); - } - if (!dir.isDirectory()) { - errorlog.info("contents of {}: {}", dir, - listDir(dir.getParentFile())); - throw new FileNotFoundException( - "Not a directory: " + dir); - } - } - - /** - * Verify that a file exists - * @param file file - * @param errorlog log for output on an error - * @throws FileNotFoundException - */ - public static void verifyFileExists(File file, Logger errorlog) throws - FileNotFoundException { - if (!file.exists()) { - errorlog.warn("contents of {}: {}", file, - listDir(file.getParentFile())); - throw new FileNotFoundException(file.toString()); - } - if (!file.isFile()) { - throw new FileNotFoundException("Not a file: " + file.toString()); - } - } - - /** - * verify that a config option is set - * @param configuration config - * @param key key - * @return the value, in case it needs to be verified too - * @throws BadConfigException if the key is missing - */ - public static String verifyOptionSet(Configuration configuration, String key, - boolean allowEmpty) throws BadConfigException { - String val = configuration.get(key); - if (val == null) { - throw new BadConfigException( - "Required configuration option \"%s\" not defined ", key); - } - if (!allowEmpty && val.isEmpty()) { - throw new BadConfigException( - "Configuration option \"%s\" must not be empty", key); - } - return val; - } - - /** - * Verify that a keytab property is defined and refers to a non-empty file - * - * @param siteConf configuration - * @param prop property to look for - * @return the file referenced - * @throws BadConfigException on a failure - */ - public static File verifyKeytabExists(Configuration siteConf, - String prop) throws - BadConfigException { - String keytab = siteConf.get(prop); - if (keytab == null) { - throw new BadConfigException("Missing keytab property %s", - prop); - - } - File keytabFile = new File(keytab); - if (!keytabFile.exists()) { - throw new BadConfigException("Missing keytab file %s defined in %s", - keytabFile, - prop); - } - if (keytabFile.length() == 0 || !keytabFile.isFile()) { - throw new BadConfigException("Invalid keytab file %s defined in %s", - keytabFile, - prop); - } - return keytabFile; - } - - /** - * Add a subpath to an existing URL. This extends - * the path, inserting a / between all entries - * if needed. - * @param base base path/URL - * @param path subpath - * @return base+"/"+subpath - */ - public static String appendToURL(String base, String path) { - StringBuilder fullpath = new StringBuilder(base); - if (!base.endsWith("/")) { - fullpath.append("/"); - } - if (path.startsWith("/")) { - fullpath.append(path.substring(1)); - } else { - fullpath.append(path); - } - return fullpath.toString(); - } - - /** - * Truncate the given string to a maximum length provided - * with a pad (...) added to the end if expected size if more than 10. - * @param toTruncate string to truncate; may be null - * @param maxSize maximum size - * @return the truncated/padded string. - */ - public static String truncate(String toTruncate, int maxSize) { - if (toTruncate == null || maxSize < 1 - || toTruncate.length() <= maxSize) { - return toTruncate; - } - - String pad = "..."; - if (maxSize < 10) { - pad = ""; - } - return toTruncate.substring(0, maxSize - pad.length()).concat(pad); - } - - /** - * Given a source folder create zipped file - * - * @param srcFolder - * @param zipFile - * - * @throws IOException - */ - public static void zipFolder(File srcFolder, File zipFile) throws IOException { - log.info("Zipping folder {} to {}", srcFolder.getAbsolutePath(), zipFile.getAbsolutePath()); - List<String> files = new ArrayList<>(); - generateFileList(files, srcFolder, srcFolder, true); - - byte[] buffer = new byte[1024]; - - try (FileOutputStream fos = new FileOutputStream(zipFile)) { - try (ZipOutputStream zos = new ZipOutputStream(fos)) { - - for (String file : files) { - ZipEntry ze = new ZipEntry(file); - zos.putNextEntry(ze); - try (FileInputStream in = new FileInputStream(srcFolder + File.separator + file)) { - int len; - while ((len = in.read(buffer)) > 0) { - zos.write(buffer, 0, len); - } - } - } - } - } - } - - /** - * Given a source folder create a tar.gz file - * - * @param libDirs - * @param tarGzipFile - * - * @throws IOException - */ - public static void tarGzipFolder(String[] libDirs, File tarGzipFile, - FilenameFilter filter) throws IOException { - log.info("Tar-gzipping folders {} to {}", libDirs, - tarGzipFile.getAbsolutePath()); - - try(TarArchiveOutputStream taos = - new TarArchiveOutputStream(new GZIPOutputStream( - new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) { - for (String libDir : libDirs) { - File srcFolder = new File(libDir); - List<String> files = new ArrayList<>(); - generateFileList(files, srcFolder, srcFolder, true, filter); - for (String file : files) { - File srcFile = new File(srcFolder, file); - TarArchiveEntry tarEntry = new TarArchiveEntry( - srcFile, file); - taos.putArchiveEntry(tarEntry); - try(FileInputStream in = new FileInputStream(srcFile)) { - org.apache.commons.io.IOUtils.copy(in, taos); - } - taos.flush(); - taos.closeArchiveEntry(); - } - } - } - } - - private static void generateFileList(List<String> fileList, File node, - File rootFolder, Boolean relative) { - generateFileList(fileList, node, rootFolder, relative, null); - } - - private static void generateFileList(List<String> fileList, File node, - File rootFolder, Boolean relative, FilenameFilter filter) { - if (node.isFile()) { - String fileFullPath = node.toString(); - if (relative) { - fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1, - fileFullPath.length())); - } else { - fileList.add(fileFullPath); - } - } - - if (node.isDirectory()) { - String[] subNode = node.list(filter); - if (subNode == null || subNode.length == 0) { - return; - } - for (String filename : subNode) { - generateFileList(fileList, new File(node, filename), rootFolder, - relative, filter); - } - } - } - - /** - * Check for any needed libraries being present. On Unix none are needed; - * on windows they must be present - * @return true if all is well - */ - public static String checkForRequiredNativeLibraries() { - - if (!Shell.WINDOWS) { - return ""; - } - StringBuilder errorText = new StringBuilder(""); - if (!NativeIO.isAvailable()) { - errorText.append("No native IO library. "); - } - try { - String path = Shell.getQualifiedBinPath(WINUTILS); - log.debug("winutils is at {}", path); - } catch (IOException e) { - errorText.append("No " + WINUTILS); - log.warn("No winutils: {}", e, e); - } - try { - File target = new File("target"); - FileUtil.canRead(target); - } catch (UnsatisfiedLinkError e) { - log.warn("Failing to link to native IO methods: {}", e, e); - errorText.append("No native IO methods"); - } - return errorText.toString(); - } - - /** - * Strictly verify that windows utils is present. - * Checks go as far as opening the file and looking for - * the headers. - * @throws IOException on any problem reading the file - * @throws FileNotFoundException if the file is not considered valid - */ - public static void maybeVerifyWinUtilsValid() throws - IOException, - SliderException { - String errorText = SliderUtils.checkForRequiredNativeLibraries(); - if (!errorText.isEmpty()) { - throw new BadClusterStateException(errorText); - } - } - - /** - * Write bytes to a file - * @param outfile output file - * @param data data to write - * @throws IOException on any IO problem - */ - public static void write(File outfile, byte[] data) - throws IOException { - File parentDir = outfile.getCanonicalFile().getParentFile(); - if (parentDir == null) { - throw new IOException(outfile.getPath() + " has no parent dir"); - } - if (!parentDir.exists()) { - if(!parentDir.mkdirs()) { - throw new IOException("Failed to create parent directory " + parentDir); - } - } - SliderUtils.verifyIsDir(parentDir, log); - try(FileOutputStream out = new FileOutputStream(outfile)) { - out.write(data); - } - } - - /** - * Compare the times of two applications: most recent app comes first - * Specifically: the one whose start time value is greater. - */ - private static class MostRecentlyStartedAppFirst - implements Comparator<ApplicationReport>, Serializable { - @Override - public int compare(ApplicationReport r1, ApplicationReport r2) { - long x = r1.getStartTime(); - long y = r2.getStartTime(); - return compareTwoLongsReverse(x, y); - } - } - - /** - * Compare the times of two applications: most recent app comes first. - * "Recent"== the app whose start time <i>or finish time</i> is the greatest. - */ - private static class MostRecentlyStartedOrFinishedFirst - implements Comparator<ApplicationReport>, Serializable { - @Override - public int compare(ApplicationReport r1, ApplicationReport r2) { - long started1 = r1.getStartTime(); - long started2 = r2.getStartTime(); - long finished1 = r1.getFinishTime(); - long finished2 = r2.getFinishTime(); - long lastEvent1 = Math.max(started1, finished1); - long lastEvent2 = Math.max(started2, finished2); - return compareTwoLongsReverse(lastEvent1, lastEvent2); - } - } - - /** - * Compare the times of two applications: most recently finished app comes first - * Specifically: the one whose finish time value is greater. - */ - private static class MostRecentAppFinishFirst - implements Comparator<ApplicationReport>, Serializable { - @Override - public int compare(ApplicationReport r1, ApplicationReport r2) { - long x = r1.getFinishTime(); - long y = r2.getFinishTime(); - return compareTwoLongsReverse(x, y); - } - } - - /** - * Compare two long values for sorting. As the return value for - * comparators must be int, the simple value of <code>x-y</code> - * is inapplicable - * @param x x value - * @param y y value - * @return +ve if x is less than y, -ve if y is greater than x; 0 for equality - */ - public static int compareTwoLongsReverse(long x, long y) { - return (x < y) ? 1 : ((x == y) ? 0 : -1); - } - - public static String createNameTag(String name) { - return "Name: " + name; - } - - public static String createVersionTag(String version) { - return "Version: " + version; - } - - public static String createDescriptionTag(String description) { - return "Description: " + description; - } - - public static final String DAYS = ".days"; - public static final String HOURS = ".hours"; - public static final String MINUTES = ".minutes"; - public static final String SECONDS = ".seconds"; -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
