http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java new file mode 100644 index 0000000..c0712c3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java @@ -0,0 +1,680 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.utils; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.SaslPropertiesResolver; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.Shell; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.crypto.Cipher; +import java.io.Closeable; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.lang.reflect.InvocationTargetException; +import java.net.InetAddress; +import java.security.NoSuchAlgorithmException; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.regex.Pattern; + +import static org.apache.hadoop.security.UserGroupInformation.*; +import static org.apache.hadoop.security.authentication.util.KerberosUtil.*; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*; + +/** + * Kerberos diagnostics + * At some point this may move to hadoop core, so please keep use of slider + * methods and classes to ~0. + * + * This operation expands some of the diagnostic output of the security code, + * but not all. For completeness + * + * Set the environment variable {@code HADOOP_JAAS_DEBUG=true} + * Set the log level for {@code org.apache.hadoop.security=DEBUG} + */ +public class KerberosDiags implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(KerberosDiags.class); + public static final String KRB5_CCNAME = "KRB5CCNAME"; + public static final String JAVA_SECURITY_KRB5_CONF + = "java.security.krb5.conf"; + public static final String JAVA_SECURITY_KRB5_REALM + = "java.security.krb5.realm"; + public static final String SUN_SECURITY_KRB5_DEBUG + = "sun.security.krb5.debug"; + public static final String SUN_SECURITY_SPNEGO_DEBUG + = "sun.security.spnego.debug"; + public static final String SUN_SECURITY_JAAS_FILE + = "java.security.auth.login.config"; + public static final String KERBEROS_KINIT_COMMAND + = "hadoop.kerberos.kinit.command"; + public static final String HADOOP_AUTHENTICATION_IS_DISABLED + = "Hadoop authentication is disabled"; + public static final String UNSET = "(unset)"; + public static final String NO_DEFAULT_REALM = "Cannot locate default realm"; + + private final Configuration conf; + private final List<String> services; + private final PrintStream out; + private final File keytab; + private final String principal; + private final long minKeyLength; + private final boolean securityRequired; + + public static final String CAT_JVM = "JVM"; + public static final String CAT_JAAS = "JAAS"; + public static final String CAT_CONFIG = "CONFIG"; + public static final String CAT_LOGIN = "LOGIN"; + public static final String CAT_KERBEROS = "KERBEROS"; + public static final String CAT_SASL = "SASL"; + + @SuppressWarnings("IOResourceOpenedButNotSafelyClosed") + public KerberosDiags(Configuration conf, + PrintStream out, + List<String> services, + File keytab, + String principal, + long minKeyLength, + boolean securityRequired) { + this.conf = conf; + this.services = services; + this.keytab = keytab; + this.principal = principal; + this.out = out; + this.minKeyLength = minKeyLength; + this.securityRequired = securityRequired; + } + + @Override + public void close() throws IOException { + flush(); + } + + /** + * Execute diagnostics. + * <p> + * Things it would be nice if UGI made accessible + * <ol> + * <li>A way to enable JAAS debug programatically</li> + * <li>Access to the TGT</li> + * </ol> + * @return true if security was enabled and all probes were successful + * @throws KerberosDiagsFailure explicitly raised failure + * @throws Exception other security problems + */ + @SuppressWarnings("deprecation") + public boolean execute() throws Exception { + + title("Kerberos Diagnostics scan at %s", + new Date(System.currentTimeMillis())); + + // check that the machine has a name + println("Hostname: %s", + InetAddress.getLocalHost().getCanonicalHostName()); + + // Fail fast on a JVM without JCE installed. + validateKeyLength(); + + // look at realm + println("JVM Kerberos Login Module = %s", getKrb5LoginModuleName()); + printDefaultRealm(); + + title("System Properties"); + for (String prop : new String[]{ + JAVA_SECURITY_KRB5_CONF, + JAVA_SECURITY_KRB5_REALM, + SUN_SECURITY_KRB5_DEBUG, + SUN_SECURITY_SPNEGO_DEBUG, + SUN_SECURITY_JAAS_FILE + }) { + printSysprop(prop); + } + + title("Environment Variables"); + for (String env : new String[]{ + "HADOOP_JAAS_DEBUG", + KRB5_CCNAME, + "HADOOP_USER_NAME", + "HADOOP_PROXY_USER", + HADOOP_TOKEN_FILE_LOCATION, + }) { + printEnv(env); + } + + for (String prop : new String[]{ + KERBEROS_KINIT_COMMAND, + HADOOP_SECURITY_AUTHENTICATION, + HADOOP_SECURITY_AUTHORIZATION, + "hadoop.kerberos.min.seconds.before.relogin", // not in 2.6 + "hadoop.security.dns.interface", // not in 2.6 + "hadoop.security.dns.nameserver", // not in 2.6 + HADOOP_RPC_PROTECTION, + HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS, + HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX, + HADOOP_SECURITY_GROUP_MAPPING, + "hadoop.security.impersonation.provider.class", // not in 2.6 + "dfs.data.transfer.protection" // HDFS + }) { + printConfOpt(prop); + } + + // check that authentication is enabled + if (SecurityUtil.getAuthenticationMethod(conf) + .equals(AuthenticationMethod.SIMPLE)) { + println(HADOOP_AUTHENTICATION_IS_DISABLED); + failif(securityRequired, CAT_CONFIG, HADOOP_AUTHENTICATION_IS_DISABLED); + // no security, skip rest of test + return false; + } + + validateKrb5File(); + validateSasl(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS); + validateSasl("dfs.data.transfer.saslproperties.resolver.class"); + validateKinitExecutable(); + validateJAAS(); + // now the big test: login, then try again + boolean krb5Debug = getAndSet(SUN_SECURITY_KRB5_DEBUG); + boolean spnegoDebug = getAndSet(SUN_SECURITY_SPNEGO_DEBUG); + try { + title("Logging in"); + + if (keytab != null) { + dumpKeytab(keytab); + loginFromKeytab(); + } else { + UserGroupInformation loginUser = getLoginUser(); + dumpUGI("Log in user", loginUser); + validateUGI("Login user", loginUser); + println("Ticket based login: %b", isLoginTicketBased()); + println("Keytab based login: %b", isLoginKeytabBased()); + } + + return true; + } finally { + // restore original system properties + System.setProperty(SUN_SECURITY_KRB5_DEBUG, + Boolean.toString(krb5Debug)); + System.setProperty(SUN_SECURITY_SPNEGO_DEBUG, + Boolean.toString(spnegoDebug)); + } + } + + /** + * Fail fast on a JVM without JCE installed. + * + * This is a recurrent problem + * (that is: it keeps creeping back with JVM updates); + * a fast failure is the best tactic + * @throws NoSuchAlgorithmException + */ + + protected void validateKeyLength() throws NoSuchAlgorithmException { + int aesLen = Cipher.getMaxAllowedKeyLength("AES"); + println("Maximum AES encryption key length %d bits", aesLen); + failif (aesLen < minKeyLength, + CAT_JVM, + "Java Cryptography Extensions are not installed on this JVM." + +" Maximum supported key length %s - minimum required %d", + aesLen, minKeyLength); + } + + /** + * Get the default realm. + * <p> + * Not having a default realm may be harmless, so is noted at info. + * All other invocation failures are downgraded to warn, as + * follow-on actions may still work. + * failure to invoke the method via introspection is rejected, + * as it's a sign of JVM compatibility issues that may have other + * consequences + */ + protected void printDefaultRealm() { + try { + println("Default Realm = %s", + getDefaultRealm()); + } catch (ClassNotFoundException + | IllegalAccessException + | NoSuchMethodException e) { + + throw new KerberosDiagsFailure(CAT_JVM, e, + "Failed to invoke krb5.Config.getDefaultRealm: %s", e); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause() != null ? e.getCause() : e; + if (cause.toString().contains(NO_DEFAULT_REALM)) { + // exception raised if there is no default realm. This is not + // always a problem, so downgrade to a message. + println("Host has no default realm"); + LOG.debug(cause.toString(), cause); + } else { + println("Kerberos.getDefaultRealm() failed: %s\n%s", + cause, + org.apache.hadoop.util.StringUtils.stringifyException(cause)); + } + } + } + + /** + * Locate the krb5.conf file and dump it. + * No-op on windows. + * @throws IOException + */ + private void validateKrb5File() throws IOException { + if (!Shell.WINDOWS) { + title("Locating Kerberos configuration file"); + String krbPath = "/etc/krb5.conf"; + String jvmKrbPath = System.getProperty(JAVA_SECURITY_KRB5_CONF); + if (jvmKrbPath != null) { + println("Setting kerberos path from sysprop %s: %s", + JAVA_SECURITY_KRB5_CONF, jvmKrbPath); + krbPath = jvmKrbPath; + } + + String krb5name = System.getenv(KRB5_CCNAME); + if (krb5name != null) { + println("Setting kerberos path from environment variable %s: %s", + KRB5_CCNAME, krb5name); + krbPath = krb5name; + if (jvmKrbPath != null) { + println("Warning - both %s and %s were set - %s takes priority", + JAVA_SECURITY_KRB5_CONF, KRB5_CCNAME, KRB5_CCNAME); + } + } + + File krbFile = new File(krbPath); + println("Kerberos configuration file = %s", krbFile); + failif(!krbFile.exists(), + CAT_KERBEROS, + "Kerberos configuration file %s not found", krbFile); + dump(krbFile); + } + } + + /** + * Dump a keytab: list all principals. + * @param keytabFile the keytab file + * @throws IOException IO problems + */ + public void dumpKeytab(File keytabFile) throws IOException { + title("Examining keytab %s", keytabFile); + File kt = keytabFile.getCanonicalFile(); + failif(!kt.exists(), CAT_CONFIG, "Keytab not found: %s", kt); + failif(!kt.isFile(), CAT_CONFIG, "Keytab is not a valid file: %s", kt); + + String[] names = getPrincipalNames(keytabFile.getCanonicalPath(), + Pattern.compile(".*")); + println("keytab entry count: %d", names.length); + for (String name : names) { + println(" %s", name); + } + println("-----"); + } + + /** + * Log in from a keytab, dump the UGI, validate it, then try and log in again. + * That second-time login catches JVM/Hadoop compatibility problems. + * @throws IOException + */ + private void loginFromKeytab() throws IOException { + UserGroupInformation ugi; + String identity; + if (keytab != null) { + File kt = keytab.getCanonicalFile(); + println("Using keytab %s principal %s", kt, principal); + identity = principal; + + failif(StringUtils.isEmpty(principal), CAT_KERBEROS, + "No principal defined"); + ugi = loginUserFromKeytabAndReturnUGI(principal, kt.getPath()); + dumpUGI(identity, ugi); + validateUGI(principal, ugi); + + title("Attempting to log in from keytab again"); + // package scoped -hence the reason why this class must be in the + // hadoop.security package + setShouldRenewImmediatelyForTests(true); + // attempt a new login + ugi.reloginFromKeytab(); + } else { + println("No keytab: logging is as current user"); + } + } + + /** + * Dump a UGI. + * @param title title of this section + * @param ugi UGI to dump + * @throws IOException + */ + private void dumpUGI(String title, UserGroupInformation ugi) + throws IOException { + title(title); + println("UGI instance = %s", ugi); + println("Has kerberos credentials: %b", ugi.hasKerberosCredentials()); + println("Authentication method: %s", ugi.getAuthenticationMethod()); + println("Real Authentication method: %s", + ugi.getRealAuthenticationMethod()); + title("Group names"); + for (String name : ugi.getGroupNames()) { + println(name); + } + title("Credentials"); + Credentials credentials = ugi.getCredentials(); + List<Text> secretKeys = credentials.getAllSecretKeys(); + title("Secret keys"); + if (!secretKeys.isEmpty()) { + for (Text secret: secretKeys) { + println("%s", secret); + } + } else { + println("(none)"); + } + + dumpTokens(ugi); + } + + /** + * Validate the UGI: verify it is kerberized. + * @param messagePrefix message in exceptions + * @param user user to validate + */ + private void validateUGI(String messagePrefix, UserGroupInformation user) { + failif(!user.hasKerberosCredentials(), + CAT_LOGIN, "%s: No kerberos credentials for %s", messagePrefix, user); + failif(user.getAuthenticationMethod() == null, + CAT_LOGIN, "%s: Null AuthenticationMethod for %s", messagePrefix, user); + } + + /** + * A cursory look at the {@code kinit} executable. + * If it is an absolute path: it must exist with a size > 0. + * If it is just a command, it has to be on the path. There's no check + * for that -but the PATH is printed out. + */ + private void validateKinitExecutable() { + String kinit = conf.getTrimmed(KERBEROS_KINIT_COMMAND, ""); + if (!kinit.isEmpty()) { + File kinitPath = new File(kinit); + println("%s = %s", KERBEROS_KINIT_COMMAND, kinitPath); + if (kinitPath.isAbsolute()) { + failif(!kinitPath.exists(), CAT_KERBEROS, + "%s executable does not exist: %s", + KERBEROS_KINIT_COMMAND, kinitPath); + failif(!kinitPath.isFile(), CAT_KERBEROS, + "%s path does not refer to a file: %s", + KERBEROS_KINIT_COMMAND, kinitPath); + failif(kinitPath.length() == 0, CAT_KERBEROS, + "%s file is empty: %s", + KERBEROS_KINIT_COMMAND, kinitPath); + } else { + println("Executable %s is relative -must be on the PATH", kinit); + printEnv("PATH"); + } + } + } + + /** + * Try to load the SASL resolver. + * @param saslPropsResolverKey key for the SASL resolver + */ + private void validateSasl(String saslPropsResolverKey) { + title("Resolving SASL property %s", saslPropsResolverKey); + String saslPropsResolver = conf.getTrimmed(saslPropsResolverKey); + try { + Class<? extends SaslPropertiesResolver> resolverClass = conf.getClass( + saslPropsResolverKey, + SaslPropertiesResolver.class, SaslPropertiesResolver.class); + println("Resolver is %s", resolverClass); + } catch (RuntimeException e) { + throw new KerberosDiagsFailure(CAT_SASL, e, + "Failed to load %s class %s", + saslPropsResolverKey, saslPropsResolver); + } + } + + /** + * Validate any JAAS entry referenced in the {@link #SUN_SECURITY_JAAS_FILE} + * property. + */ + private void validateJAAS() { + String jaasFilename = System.getProperty(SUN_SECURITY_JAAS_FILE); + if (jaasFilename != null) { + title("JAAS"); + File jaasFile = new File(jaasFilename); + println("JAAS file is defined in %s: %s", + SUN_SECURITY_JAAS_FILE, jaasFile); + failif(!jaasFile.exists(), CAT_JAAS, + "JAAS file does not exist: %s", jaasFile); + failif(!jaasFile.isFile(), CAT_JAAS, + "Specified JAAS file is not a file: %s", jaasFile); + } + } + + /** + * Dump all tokens of a user + * @param user user + */ + public void dumpTokens(UserGroupInformation user) { + Collection<Token<? extends TokenIdentifier>> tokens + = user.getCredentials().getAllTokens(); + title("Token Count: %d", tokens.size()); + for (Token<? extends TokenIdentifier> token : tokens) { + println("Token %s", token.getKind()); + } + } + + /** + * Set the System property to true; return the old value for caching + * @param sysprop property + * @return the previous value + */ + private boolean getAndSet(String sysprop) { + boolean old = Boolean.getBoolean(sysprop); + System.setProperty(sysprop, "true"); + return old; + } + + /** + * Flush all active output channels, including {@Code System.err}, + * so as to stay in sync with any JRE log messages. + */ + private void flush() { + if (out != null) { + out.flush(); + } else { + System.out.flush(); + } + System.err.flush(); + } + + /** + * Format and print a line of output. + * This goes to any output file, or + * is logged at info. The output is flushed before and after, to + * try and stay in sync with JRE logging. + * @param format format string + * @param args any arguments + */ + @VisibleForTesting + public void println(String format, Object... args) { + println(format(format, args)); + } + + /** + * Print a line of output. This goes to any output file, or + * is logged at info. The output is flushed before and after, to + * try and stay in sync with JRE logging. + * @param msg message string + */ + @VisibleForTesting + private void println(String msg) { + flush(); + if (out != null) { + out.println(msg); + } else { + LOG.info(msg); + } + flush(); + } + + /** + * Print a title entry + * @param format format string + * @param args any arguments + */ + private void title(String format, Object... args) { + println(""); + println(""); + String msg = "== " + format(format, args) + " =="; + println(msg); + println(""); + } + + /** + * Print a system property, or {@link #UNSET} if unset. + * @param property property to print + */ + private void printSysprop(String property) { + println("%s = \"%s\"", property, + System.getProperty(property, UNSET)); + } + + /** + * Print a configuration option, or {@link #UNSET} if unset. + * @param option option to print + */ + private void printConfOpt(String option) { + println("%s = \"%s\"", option, conf.get(option, UNSET)); + } + + /** + * Print an environment variable's name and value; printing + * {@link #UNSET} if it is not set + * @param variable environment variable + */ + private void printEnv(String variable) { + String env = System.getenv(variable); + println("%s = \"%s\"", variable, env != null ? env : UNSET); + } + + /** + * Dump any file to standard out; add a trailing newline + * @param file file to dump + * @throws IOException IO problems + */ + public void dump(File file) throws IOException { + try (FileInputStream in = new FileInputStream(file)) { + for (String line : IOUtils.readLines(in)) { + println("%s", line); + } + } + println(""); + } + + /** + * Format and raise a failure + * + * @param category category for exception + * @param message string formatting message + * @param args any arguments for the formatting + * @throws KerberosDiagsFailure containing the formatted text + */ + private void fail(String category, String message, Object... args) + throws KerberosDiagsFailure { + throw new KerberosDiagsFailure(category, message, args); + } + + /** + * Conditional failure with string formatted arguments + * @param condition failure condition + * @param category category for exception + * @param message string formatting message + * @param args any arguments for the formatting + * @throws KerberosDiagsFailure containing the formatted text + * if the condition was met + */ + private void failif(boolean condition, + String category, + String message, + Object... args) + throws KerberosDiagsFailure { + if (condition) { + fail(category, message, args); + } + } + + /** + * Format a string, treating a call where there are no varags values + * as a string to pass through unformatted. + * @param message message, which is either a format string + args, or + * a general string + * @param args argument array + * @return a string for printing. + */ + public static String format(String message, Object... args) { + if (args.length == 0) { + return message; + } else { + return String.format(message, args); + } + } + + /** + * Diagnostics failures return the exit code 41, "unauthorized". + * + * They have a category, initially for testing: the category can be + * validated without having to match on the entire string. + */ + public static class KerberosDiagsFailure extends ExitUtil.ExitException { + private final String category; + + public KerberosDiagsFailure(String category, String message) { + super(41, category + ": " + message); + this.category = category; + } + + public KerberosDiagsFailure(String category, String message, Object... args) { + this(category, format(message, args)); + } + + public KerberosDiagsFailure(String category, Throwable throwable, + String message, Object... args) { + this(category, message, args); + initCause(throwable); + } + + public String getCategory() { + return category; + } + } +}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java new file mode 100644 index 0000000..6efa880 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PatternValidator.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.utils; + +import java.util.regex.Pattern; + +/** + * Utility class to validate strings against a predefined pattern. + */ +public class PatternValidator { + + public static final String E_INVALID_NAME = + "Invalid name %s does not match the pattern pattern %s "; + private final Pattern valid; + private final String pattern; + + public PatternValidator(String pattern) { + this.pattern = pattern; + valid = Pattern.compile(pattern); + } + + /** + * Validate the name -restricting it to the set defined in + * @param name name to validate + * @throws IllegalArgumentException if not a valid name + */ + public void validate(String name) { + if (!matches(name)) { + throw new IllegalArgumentException( + String.format(E_INVALID_NAME, name, pattern)); + } + } + + /** + * Query to see if the pattern matches + * @param name name to validate + * @return true if the string matches the pattern + */ + public boolean matches(String name) { + return valid.matcher(name).matches(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java new file mode 100644 index 0000000..2dbf37f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.service.utils; + +import org.apache.hadoop.yarn.service.conf.SliderExitCodes; +import org.apache.hadoop.yarn.service.exceptions.BadConfigException; +import org.apache.hadoop.yarn.service.exceptions.SliderException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * a scanner which can take an input string for a range or scan the lot. + */ +public class PortScanner { + private static Pattern NUMBER_RANGE = Pattern.compile("^(\\d+)\\s*-\\s*(\\d+)$"); + private static Pattern SINGLE_NUMBER = Pattern.compile("^\\d+$"); + + private List<Integer> remainingPortsToCheck; + + public PortScanner() { + } + + public void setPortRange(String input) throws BadConfigException { + // first split based on commas + Set<Integer> inputPorts= new TreeSet<Integer>(); + String[] ranges = input.split(","); + for ( String range : ranges ) { + if (range.trim().isEmpty()) { + continue; + } + Matcher m = SINGLE_NUMBER.matcher(range.trim()); + if (m.find()) { + inputPorts.add(Integer.parseInt(m.group())); + continue; + } + m = NUMBER_RANGE.matcher(range.trim()); + if (m.find()) { + String[] boundaryValues = m.group(0).split("-"); + int start = Integer.parseInt(boundaryValues[0].trim()); + int end = Integer.parseInt(boundaryValues[1].trim()); + if (end < start) { + throw new BadConfigException("End of port range is before start: " + + range + " in input: " + input); + } + for (int i = start; i < end + 1; i++) { + inputPorts.add(i); + } + continue; + } + throw new BadConfigException("Bad port range: " + range + " in input: " + + input); + } + if (inputPorts.size() == 0) { + throw new BadConfigException("No ports found in range: " + input); + } + this.remainingPortsToCheck = new ArrayList<Integer>(inputPorts); + } + + public List<Integer> getRemainingPortsToCheck() { + return remainingPortsToCheck; + } + + public int getAvailablePort() throws SliderException, IOException { + if (remainingPortsToCheck != null) { + return getAvailablePortViaPortArray(); + } else { + return SliderUtils.getOpenPort(); + } + } + + private int getAvailablePortViaPortArray() throws SliderException { + boolean found = false; + int availablePort = -1; + Iterator<Integer> portsToCheck = this.remainingPortsToCheck.iterator(); + while (portsToCheck.hasNext() && !found) { + int portToCheck = portsToCheck.next(); + found = SliderUtils.isPortAvailable(portToCheck); + if (found) { + availablePort = portToCheck; + portsToCheck.remove(); + } + } + + if (availablePort < 0) { + throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION, + "No available ports found in configured range {}", + remainingPortsToCheck); + } + + return availablePort; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java new file mode 100644 index 0000000..9d00b3c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfiguration.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.utils; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.service.api.records.ConfigFormat; +import org.apache.hadoop.yarn.service.exceptions.BadConfigException; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.SerializationConfig; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.io.IOException; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +/** + * JSON-serializable description of a published key-val configuration. + * + * The values themselves are not serialized in the external view; they have + * to be served up by the far end + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class PublishedConfiguration { + + public String description; + public long updated; + + public String updatedTime; + + public Map<String, String> entries = new HashMap<>(); + + public PublishedConfiguration() { + } + + /** + * build an empty published configuration + * @param description configuration description + */ + public PublishedConfiguration(String description) { + this.description = description; + } + + /** + * Build a configuration from the entries + * @param description configuration description + * @param entries entries to put + */ + public PublishedConfiguration(String description, + Iterable<Map.Entry<String, String>> entries) { + this.description = description; + putValues(entries); + } + + /** + * Build a published configuration, using the keys from keysource, + * but resolving the values from the value source, via Configuration.get() + * @param description configuration description + * @param keysource source of keys + * @param valuesource source of values + */ + public PublishedConfiguration(String description, + Iterable<Map.Entry<String, String>> keysource, + Configuration valuesource) { + this.description = description; + putValues(ConfigHelper.resolveConfiguration(keysource, valuesource)); + } + + + /** + * Is the configuration empty. This means either that it has not + * been given any values, or it is stripped down copy set down over the + * wire. + * @return true if it is empty + */ + public boolean isEmpty() { + return entries.isEmpty(); + } + + + public void setUpdated(long updated) { + this.updated = updated; + this.updatedTime = new Date(updated).toString(); + } + + public long getUpdated() { + return updated; + } + + /** + * Set the values from an iterable (this includes a Hadoop Configuration + * and Java properties object). + * Any existing value set is discarded + * @param entries entries to put + */ + public void putValues(Iterable<Map.Entry<String, String>> entries) { + this.entries = new HashMap<String, String>(); + for (Map.Entry<String, String> entry : entries) { + this.entries.put(entry.getKey(), entry.getValue()); + } + + } + + /** + * Convert to Hadoop XML + * @return the configuration as a Hadoop Configuratin + */ + public Configuration asConfiguration() { + Configuration conf = new Configuration(false); + try { + ConfigHelper.addConfigMap(conf, entries, ""); + } catch (BadConfigException e) { + // triggered on a null value; switch to a runtime (and discard the stack) + throw new RuntimeException(e.toString()); + } + return conf; + } + + public String asConfigurationXML() throws IOException { + return ConfigHelper.toXml(asConfiguration()); + } + + /** + * Convert values to properties + * @return a property file + */ + public Properties asProperties() { + Properties props = new Properties(); + props.putAll(entries); + return props; + } + + /** + * Return the values as json string + * @return the JSON representation + * @throws IOException marshalling failure + */ + public String asJson() throws IOException { + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true); + String json = mapper.writeValueAsString(entries); + return json; + } + + + /** + * This makes a copy without the nested content -so is suitable + * for returning as part of the list of a parent's values + * @return the copy + */ + public PublishedConfiguration shallowCopy() { + PublishedConfiguration that = new PublishedConfiguration(); + that.description = this.description; + that.updated = this.updated; + that.updatedTime = this.updatedTime; + return that; + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("PublishedConfiguration{"); + sb.append("description='").append(description).append('\''); + sb.append(" entries = ").append(entries.size()); + sb.append('}'); + return sb.toString(); + } + + /** + * Create an outputter for a given format + * @param format format to use + * @return an instance of output + */ + public PublishedConfigurationOutputter createOutputter(ConfigFormat format) { + return PublishedConfigurationOutputter.createOutputter(format, this); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java new file mode 100644 index 0000000..88ecf2c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/PublishedConfigurationOutputter.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.utils; + +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.service.api.records.ConfigFormat; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.DumperOptions.FlowStyle; +import org.yaml.snakeyaml.Yaml; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.io.StringWriter; +import java.util.Properties; + +/** + * Output a published configuration + */ +public abstract class PublishedConfigurationOutputter { + + private static final String COMMENTS = "Generated by Apache Slider"; + + protected final PublishedConfiguration owner; + + protected PublishedConfigurationOutputter(PublishedConfiguration owner) { + this.owner = owner; + } + + /** + * Save the config to a destination file, in the format of this outputter + * @param dest destination file + * @throws IOException + */ +/* JDK7 + public void save(File dest) throws IOException { + try(FileOutputStream out = new FileOutputStream(dest)) { + save(out); + out.close(); + } + } +*/ + public void save(File dest) throws IOException { + FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8); + } + + /** + * Save the content. The default saves the asString() value + * to the output stream + * @param out output stream + * @throws IOException + */ + public void save(OutputStream out) throws IOException { + IOUtils.write(asString(), out, Charsets.UTF_8); + } + /** + * Convert to a string + * @return the string form + * @throws IOException + */ + public abstract String asString() throws IOException; + + /** + * Create an outputter for the chosen format + * @param format format enumeration + * @param owner owning config + * @return the outputter + */ + + public static PublishedConfigurationOutputter createOutputter(ConfigFormat format, + PublishedConfiguration owner) { + Preconditions.checkNotNull(owner); + switch (format) { + case XML: + case HADOOP_XML: + return new XmlOutputter(owner); + case PROPERTIES: + return new PropertiesOutputter(owner); + case JSON: + return new JsonOutputter(owner); + case ENV: + return new EnvOutputter(owner); + case TEMPLATE: + return new TemplateOutputter(owner); + case YAML: + return new YamlOutputter(owner); + default: + throw new RuntimeException("Unsupported format :" + format); + } + } + + public static class XmlOutputter extends PublishedConfigurationOutputter { + + + private final Configuration configuration; + + public XmlOutputter(PublishedConfiguration owner) { + super(owner); + configuration = owner.asConfiguration(); + } + + @Override + public void save(OutputStream out) throws IOException { + configuration.writeXml(out); + } + + @Override + public String asString() throws IOException { + return ConfigHelper.toXml(configuration); + } + + public Configuration getConfiguration() { + return configuration; + } + } + + public static class PropertiesOutputter extends PublishedConfigurationOutputter { + + private final Properties properties; + + public PropertiesOutputter(PublishedConfiguration owner) { + super(owner); + properties = owner.asProperties(); + } + + @Override + public void save(OutputStream out) throws IOException { + properties.store(out, COMMENTS); + } + + + public String asString() throws IOException { + StringWriter sw = new StringWriter(); + properties.store(sw, COMMENTS); + return sw.toString(); + } + } + + + public static class JsonOutputter extends PublishedConfigurationOutputter { + + public JsonOutputter(PublishedConfiguration owner) { + super(owner); + } + + @Override + public String asString() throws IOException { + return owner.asJson(); + } + } + + + public static class EnvOutputter extends PublishedConfigurationOutputter { + + public EnvOutputter(PublishedConfiguration owner) { + super(owner); + } + + @Override + public String asString() throws IOException { + if (!owner.entries.containsKey("content")) { + throw new IOException("Configuration has no content field and cannot " + + "be retrieved as type 'env'"); + } + String content = owner.entries.get("content"); + return ConfigUtils.replaceProps(owner.entries, content); + } + } + + public static class TemplateOutputter extends EnvOutputter { + public TemplateOutputter(PublishedConfiguration owner) { + super(owner); + } + } + + public static class YamlOutputter extends PublishedConfigurationOutputter { + + private final Yaml yaml; + + public YamlOutputter(PublishedConfiguration owner) { + super(owner); + DumperOptions options = new DumperOptions(); + options.setDefaultFlowStyle(FlowStyle.BLOCK); + yaml = new Yaml(options); + } + + public String asString() throws IOException { + return yaml.dump(owner.entries); + } + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java new file mode 100644 index 0000000..405f690 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.utils; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.service.utils.ApplicationReportSerDeser; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import java.io.IOException; + +/** + * Serialized form of an application report which can be persisted + * and then parsed. It can not be converted back into a + * real YARN application report + * + * Useful for testing + */ + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) + +public class SerializedApplicationReport { + + public String applicationId; + public String applicationAttemptId; + public String name; + public String applicationType; + public String user; + public String queue; + public String host; + public Integer rpcPort; + public String state; + public String diagnostics; + public String url; + /** + * This value is non-null only when a report is generated from a submission context. + * The YARN {@link ApplicationReport} structure does not propagate this value + * from the RM. + */ + public Long submitTime; + public Long startTime; + public Long finishTime; + public String finalStatus; + public String origTrackingUrl; + public Float progress; + + public SerializedApplicationReport() { + } + + public SerializedApplicationReport(ApplicationReport report) { + this.applicationId = report.getApplicationId().toString(); + ApplicationAttemptId attemptId = report.getCurrentApplicationAttemptId(); + this.applicationAttemptId = attemptId != null ? attemptId.toString() : "N/A"; + this.name = report.getName(); + this.applicationType = report.getApplicationType(); + this.user = report.getUser(); + this.queue = report.getQueue(); + this.host = report.getHost(); + this.rpcPort = report.getRpcPort(); + this.state = report.getYarnApplicationState().toString(); + this.diagnostics = report.getDiagnostics(); + this.startTime = report.getStartTime(); + this.finishTime = report.getFinishTime(); + FinalApplicationStatus appStatus = report.getFinalApplicationStatus(); + this.finalStatus = appStatus == null ? "" : appStatus.toString(); + this.progress = report.getProgress(); + this.url = report.getTrackingUrl(); + this.origTrackingUrl= report.getOriginalTrackingUrl(); + } + + @Override + public String toString() { + try { + return ApplicationReportSerDeser.toString(this); + } catch (IOException e) { + return super.toString(); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java index c87c3b4..21cb049 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java @@ -25,20 +25,16 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.registry.client.api.RegistryConstants; import org.apache.hadoop.registry.client.binding.RegistryUtils; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.Artifact; -import org.apache.slider.api.resource.Component; -import org.apache.slider.api.resource.Configuration; -import org.apache.slider.api.resource.Resource; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.persist.JsonSerDeser; +import org.apache.hadoop.yarn.service.api.records.Application; +import org.apache.hadoop.yarn.service.api.records.Artifact; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.Configuration; +import org.apache.hadoop.yarn.service.api.records.Resource; import org.apache.hadoop.yarn.service.provider.AbstractClientProvider; import org.apache.hadoop.yarn.service.provider.ProviderFactory; -import org.apache.slider.server.servicemonitor.MonitorUtils; -import org.apache.slider.server.services.utility.PatternValidator; -import org.apache.slider.util.RestApiConstants; -import org.apache.slider.util.RestApiErrorMessages; +import org.apache.hadoop.yarn.service.servicemonitor.probe.MonitorUtils; +import org.apache.hadoop.yarn.service.conf.RestApiConstants; +import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages; import org.codehaus.jackson.map.PropertyNamingStrategy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -127,9 +123,7 @@ public class ServiceApiUtil { List<Component> componentsToAdd = new ArrayList<>(); for (Component comp : application.getComponents()) { int maxCompLength = RegistryConstants.MAX_FQDN_LABEL_LENGTH; - if (comp.getUniqueComponentSupport()) { - maxCompLength = maxCompLength - Long.toString(Long.MAX_VALUE).length(); - } + maxCompLength = maxCompLength - Long.toString(Long.MAX_VALUE).length(); if (dnsEnabled && comp.getName().length() > maxCompLength) { throw new IllegalArgumentException(String.format(RestApiErrorMessages .ERROR_COMPONENT_NAME_INVALID, maxCompLength, comp.getName())); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java new file mode 100644 index 0000000..7440b11 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceRegistryUtils.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.utils; + +import org.apache.hadoop.registry.client.binding.RegistryUtils; +import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; + + +public class ServiceRegistryUtils { + + /** + * Base path for services + */ + public static final String ZK_SERVICES = "services"; + + /** + * Base path for all Slider references + */ + public static final String ZK_SLIDER = "slider"; + public static final String ZK_USERS = "users"; + public static final String SVC_SLIDER = "/" + ZK_SERVICES + "/" + ZK_SLIDER; + public static final String SVC_SLIDER_USERS = SVC_SLIDER + "/" + ZK_USERS; + + /** + * Get the registry path for an instance under the user's home node + * @param instanceName application instance + * @return a path to the registry location for this application instance. + */ + public static String registryPathForInstance(String instanceName) { + return RegistryUtils.servicePath( + RegistryUtils.currentUser(), YarnServiceConstants.APP_TYPE, instanceName + ); + } + + /** + * Build the path to a cluster; exists once the cluster has come up. + * Even before that, a ZK watcher could wait for it. + * @param username user + * @param clustername name of the cluster + * @return a strin + */ + public static String mkClusterPath(String username, String clustername) { + return mkSliderUserPath(username) + "/" + clustername; + } + + /** + * Build the path to a cluster; exists once the cluster has come up. + * Even before that, a ZK watcher could wait for it. + * @param username user + * @return a string + */ + public static String mkSliderUserPath(String username) { + return SVC_SLIDER_USERS + "/" + username; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3aff2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java new file mode 100644 index 0000000..d6d664e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderFileSystem.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.utils; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.IOException; + +/** + * Extends Core Filesystem with operations to manipulate ClusterDescription + * persistent state + */ +public class SliderFileSystem extends CoreFileSystem { + + Path appDir = null; + + public SliderFileSystem(FileSystem fileSystem, + Configuration configuration) { + super(fileSystem, configuration); + } + + public SliderFileSystem(Configuration configuration) throws IOException { + super(configuration); + } + + public void setAppDir(Path appDir) { + this.appDir = appDir; + } + + public Path getAppDir() { + return this.appDir; + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
