http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b7fcb09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties deleted file mode 100644 index 333859e..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This is the log4j configuration for Slider Application Master - -# Log rotation based on size (100KB) with a max of 10 backup files -log4j.rootLogger=INFO, amlog -log4j.threshhold=ALL -log4j.appender.amlog=org.apache.log4j.RollingFileAppender -log4j.appender.amlog.layout=org.apache.log4j.PatternLayout -log4j.appender.amlog.File=${LOG_DIR}/slider.log -log4j.appender.amlog.MaxFileSize=1MB -log4j.appender.amlog.MaxBackupIndex=10 - -# log layout skips stack-trace creation operations by avoiding line numbers and method -log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n - -# debug edition is much more expensive -#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n - -# configure stderr -# set the conversion pattern of stderr -# Print the date in ISO 8601 format -log4j.appender.stderr=org.apache.log4j.ConsoleAppender -log4j.appender.stderr.Target=System.err -log4j.appender.stderr.layout=org.apache.log4j.PatternLayout -log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n - -log4j.appender.subprocess=org.apache.log4j.ConsoleAppender -log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout -log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n -#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess - -# for debugging Slider -#log4j.logger.org.apache.slider=DEBUG - -# uncomment to debug service lifecycle issues -#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG -#log4j.logger.org.apache.hadoop.yarn.service=DEBUG - -# uncomment for YARN operations -#log4j.logger.org.apache.hadoop.yarn.client=DEBUG - -# uncomment this to debug security problems -#log4j.logger.org.apache.hadoop.security=DEBUG - -#crank back on some noise -log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR -log4j.logger.org.apache.hadoop.hdfs=WARN -log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR - -log4j.logger.org.apache.zookeeper=WARN -log4j.logger.org.apache.curator.framework.state=ERROR -log4j.logger.org.apache.curator.framework.imps=WARN
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b7fcb09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties new file mode 100644 index 0000000..58c8e27 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is the log4j configuration for Slider Application Master + +# Log rotation based on size (256MB) with a max of 20 backup files +log4j.rootLogger=INFO, amlog +log4j.threshhold=ALL +log4j.appender.amlog=org.apache.log4j.RollingFileAppender +log4j.appender.amlog.layout=org.apache.log4j.PatternLayout +log4j.appender.amlog.File=${LOG_DIR}/serviceam.log +log4j.appender.amlog.MaxFileSize=256MB +log4j.appender.amlog.MaxBackupIndex=20 + +# log layout skips stack-trace creation operations by avoiding line numbers and method +log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n + +# debug edition is much more expensive +#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n + +# configure stderr +# set the conversion pattern of stderr +# Print the date in ISO 8601 format +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.Target=System.err +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n + +log4j.appender.subprocess=org.apache.log4j.ConsoleAppender +log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout +log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n + +# for debugging yarn-service framework +#log4j.logger.org.apache.hadoop.yarn.service=DEBUG + +# uncomment for YARN operations +#log4j.logger.org.apache.hadoop.yarn.client=DEBUG + +# uncomment this to debug security problems +#log4j.logger.org.apache.hadoop.security=DEBUG + +#crank back on some noise +log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR +log4j.logger.org.apache.hadoop.hdfs=WARN + +log4j.logger.org.apache.zookeeper=WARN +log4j.logger.org.apache.curator.framework.state=ERROR +log4j.logger.org.apache.curator.framework.imps=WARN http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b7fcb09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml index 02317e5..c8de037 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml @@ -59,8 +59,6 @@ <source> <directory>${basedir}/src/main/proto</directory> <includes> - <include>SliderClusterMessages.proto</include> - <include>SliderClusterProtocol.proto</include> <include>ClientAMProtocol.proto</include> </includes> </source> @@ -92,32 +90,11 @@ </plugin> <plugin> - <groupId>org.apache.avro</groupId> - <artifactId>avro-maven-plugin</artifactId> - <version>${avro.version}</version> - <executions> - <execution> - <phase>generate-sources</phase> - <goals> - <goal>schema</goal> - </goals> - <configuration> - <sourceDirectory>${project.basedir}/src/main/avro/ - </sourceDirectory> - <outputDirectory>${project.build.directory}/generated-sources/java - </outputDirectory> - </configuration> - </execution> - </executions> - </plugin> - - <plugin> <groupId>org.apache.rat</groupId> <artifactId>apache-rat-plugin</artifactId> <configuration> <excludes> <exclude>**/*.json</exclude> - <exclude>src/main/resources/webapps/slideram/.keep</exclude> </excludes> </configuration> </plugin> @@ -218,11 +195,6 @@ </dependency> <dependency> - <groupId>org.apache.avro</groupId> - <artifactId>avro</artifactId> - </dependency> - - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-compress</artifactId> </dependency> @@ -422,7 +394,7 @@ <attach>false</attach> <finalName>${project.artifactId}-${project.version}</finalName> <descriptorRefs> - <descriptorRef>hadoop-yarn-slider-dist</descriptorRef> + <descriptorRef>hadoop-yarn-services-dist</descriptorRef> </descriptorRefs> </configuration> </execution> http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b7fcb09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc deleted file mode 100644 index 3667c01..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc +++ /dev/null @@ -1,114 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -[ - - { - "type": "record", - "name": "NodeEntryRecord", - "namespace": "org.apache.slider.server.avro", - "fields": [ - { - "name": "host", - "type": "string" - }, - { - "name": "role", - "type": "int" - }, - { - "name": "active", - "type": "boolean" - }, - { - "name": "last_used", - "type": "long" - } - ] - }, - - { - "type": "record", - "name": "RoleHistoryHeader", - "namespace": "org.apache.slider.server.avro", - "fields": [ - { - "name": "version", - "type": "int" - }, - { - "name": "saved", - "type": "long" - }, - { - "name": "savedx", - "type": "string" - }, - { - "name": "savedate", - "type": "string", - "default": "" - }, - { - "name": "roles", - "type": "int" - } - ] - }, - { - "type": "record", - "name": "RoleHistoryMapping", - "namespace": "org.apache.slider.server.avro", - "fields": [ - { - "name": "rolemap", - "type": { - "type": "map", - "values": "int" - } - } - ] - }, - { - "type": "record", - "name": "RoleHistoryFooter", - "namespace": "org.apache.slider.server.avro", - "fields": [ - { - "name": "count", - "type": "long" - } - ] - }, - - { - "type": "record", - "name": "RoleHistoryRecord", - "namespace": "org.apache.slider.server.avro", - "fields": [ - { - "name": "entry", - "type": [ - "org.apache.slider.server.avro.NodeEntryRecord", - "org.apache.slider.server.avro.RoleHistoryHeader", - "org.apache.slider.server.avro.RoleHistoryFooter", - "org.apache.slider.server.avro.RoleHistoryMapping" - ] - } - ] - } - -] http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b7fcb09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java deleted file mode 100644 index 905d4b1..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java +++ /dev/null @@ -1,677 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.security; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.util.ExitUtil; -import org.apache.hadoop.util.Shell; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.crypto.Cipher; -import java.io.Closeable; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.io.PrintWriter; -import java.lang.reflect.InvocationTargetException; -import java.net.InetAddress; -import java.security.NoSuchAlgorithmException; -import java.util.Collection; -import java.util.Date; -import java.util.List; -import java.util.regex.Pattern; - -import static org.apache.hadoop.security.UserGroupInformation.*; -import static org.apache.hadoop.security.authentication.util.KerberosUtil.*; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*; - -/** - * Kerberos diagnostics - * At some point this may move to hadoop core, so please keep use of slider - * methods and classes to ~0. - * - * This operation expands some of the diagnostic output of the security code, - * but not all. For completeness - * - * Set the environment variable {@code HADOOP_JAAS_DEBUG=true} - * Set the log level for {@code org.apache.hadoop.security=DEBUG} - */ -public class KerberosDiags implements Closeable { - - private static final Logger LOG = LoggerFactory.getLogger(KerberosDiags.class); - public static final String KRB5_CCNAME = "KRB5CCNAME"; - public static final String JAVA_SECURITY_KRB5_CONF - = "java.security.krb5.conf"; - public static final String JAVA_SECURITY_KRB5_REALM - = "java.security.krb5.realm"; - public static final String SUN_SECURITY_KRB5_DEBUG - = "sun.security.krb5.debug"; - public static final String SUN_SECURITY_SPNEGO_DEBUG - = "sun.security.spnego.debug"; - public static final String SUN_SECURITY_JAAS_FILE - = "java.security.auth.login.config"; - public static final String KERBEROS_KINIT_COMMAND - = "hadoop.kerberos.kinit.command"; - public static final String HADOOP_AUTHENTICATION_IS_DISABLED - = "Hadoop authentication is disabled"; - public static final String UNSET = "(unset)"; - public static final String NO_DEFAULT_REALM = "Cannot locate default realm"; - - private final Configuration conf; - private final List<String> services; - private final PrintStream out; - private final File keytab; - private final String principal; - private final long minKeyLength; - private final boolean securityRequired; - - public static final String CAT_JVM = "JVM"; - public static final String CAT_JAAS = "JAAS"; - public static final String CAT_CONFIG = "CONFIG"; - public static final String CAT_LOGIN = "LOGIN"; - public static final String CAT_KERBEROS = "KERBEROS"; - public static final String CAT_SASL = "SASL"; - - @SuppressWarnings("IOResourceOpenedButNotSafelyClosed") - public KerberosDiags(Configuration conf, - PrintStream out, - List<String> services, - File keytab, - String principal, - long minKeyLength, - boolean securityRequired) { - this.conf = conf; - this.services = services; - this.keytab = keytab; - this.principal = principal; - this.out = out; - this.minKeyLength = minKeyLength; - this.securityRequired = securityRequired; - } - - @Override - public void close() throws IOException { - flush(); - } - - /** - * Execute diagnostics. - * <p> - * Things it would be nice if UGI made accessible - * <ol> - * <li>A way to enable JAAS debug programatically</li> - * <li>Access to the TGT</li> - * </ol> - * @return true if security was enabled and all probes were successful - * @throws KerberosDiagsFailure explicitly raised failure - * @throws Exception other security problems - */ - @SuppressWarnings("deprecation") - public boolean execute() throws Exception { - - title("Kerberos Diagnostics scan at %s", - new Date(System.currentTimeMillis())); - - // check that the machine has a name - println("Hostname: %s", - InetAddress.getLocalHost().getCanonicalHostName()); - - // Fail fast on a JVM without JCE installed. - validateKeyLength(); - - // look at realm - println("JVM Kerberos Login Module = %s", getKrb5LoginModuleName()); - printDefaultRealm(); - - title("System Properties"); - for (String prop : new String[]{ - JAVA_SECURITY_KRB5_CONF, - JAVA_SECURITY_KRB5_REALM, - SUN_SECURITY_KRB5_DEBUG, - SUN_SECURITY_SPNEGO_DEBUG, - SUN_SECURITY_JAAS_FILE - }) { - printSysprop(prop); - } - - title("Environment Variables"); - for (String env : new String[]{ - "HADOOP_JAAS_DEBUG", - KRB5_CCNAME, - HADOOP_USER_NAME, - HADOOP_PROXY_USER, - HADOOP_TOKEN_FILE_LOCATION, - }) { - printEnv(env); - } - - for (String prop : new String[]{ - KERBEROS_KINIT_COMMAND, - HADOOP_SECURITY_AUTHENTICATION, - HADOOP_SECURITY_AUTHORIZATION, - "hadoop.kerberos.min.seconds.before.relogin", // not in 2.6 - "hadoop.security.dns.interface", // not in 2.6 - "hadoop.security.dns.nameserver", // not in 2.6 - HADOOP_RPC_PROTECTION, - HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS, - HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX, - HADOOP_SECURITY_GROUP_MAPPING, - "hadoop.security.impersonation.provider.class", // not in 2.6 - "dfs.data.transfer.protection" // HDFS - }) { - printConfOpt(prop); - } - - // check that authentication is enabled - if (SecurityUtil.getAuthenticationMethod(conf) - .equals(AuthenticationMethod.SIMPLE)) { - println(HADOOP_AUTHENTICATION_IS_DISABLED); - failif(securityRequired, CAT_CONFIG, HADOOP_AUTHENTICATION_IS_DISABLED); - // no security, skip rest of test - return false; - } - - validateKrb5File(); - validateSasl(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS); - validateSasl("dfs.data.transfer.saslproperties.resolver.class"); - validateKinitExecutable(); - validateJAAS(); - // now the big test: login, then try again - boolean krb5Debug = getAndSet(SUN_SECURITY_KRB5_DEBUG); - boolean spnegoDebug = getAndSet(SUN_SECURITY_SPNEGO_DEBUG); - try { - title("Logging in"); - - if (keytab != null) { - dumpKeytab(keytab); - loginFromKeytab(); - } else { - UserGroupInformation loginUser = getLoginUser(); - dumpUGI("Log in user", loginUser); - validateUGI("Login user", loginUser); - println("Ticket based login: %b", isLoginTicketBased()); - println("Keytab based login: %b", isLoginKeytabBased()); - } - - return true; - } finally { - // restore original system properties - System.setProperty(SUN_SECURITY_KRB5_DEBUG, - Boolean.toString(krb5Debug)); - System.setProperty(SUN_SECURITY_SPNEGO_DEBUG, - Boolean.toString(spnegoDebug)); - } - } - - /** - * Fail fast on a JVM without JCE installed. - * - * This is a recurrent problem - * (that is: it keeps creeping back with JVM updates); - * a fast failure is the best tactic - * @throws NoSuchAlgorithmException - */ - - protected void validateKeyLength() throws NoSuchAlgorithmException { - int aesLen = Cipher.getMaxAllowedKeyLength("AES"); - println("Maximum AES encryption key length %d bits", aesLen); - failif (aesLen < minKeyLength, - CAT_JVM, - "Java Cryptography Extensions are not installed on this JVM." - +" Maximum supported key length %s - minimum required %d", - aesLen, minKeyLength); - } - - /** - * Get the default realm. - * <p> - * Not having a default realm may be harmless, so is noted at info. - * All other invocation failures are downgraded to warn, as - * follow-on actions may still work. - * failure to invoke the method via introspection is rejected, - * as it's a sign of JVM compatibility issues that may have other - * consequences - */ - protected void printDefaultRealm() { - try { - println("Default Realm = %s", - getDefaultRealm()); - } catch (ClassNotFoundException - | IllegalAccessException - | NoSuchMethodException e) { - - throw new KerberosDiagsFailure(CAT_JVM, e, - "Failed to invoke krb5.Config.getDefaultRealm: %s", e); - } catch (InvocationTargetException e) { - Throwable cause = e.getCause() != null ? e.getCause() : e; - if (cause.toString().contains(NO_DEFAULT_REALM)) { - // exception raised if there is no default realm. This is not - // always a problem, so downgrade to a message. - println("Host has no default realm"); - LOG.debug(cause.toString(), cause); - } else { - println("Kerberos.getDefaultRealm() failed: %s\n%s", - cause, - org.apache.hadoop.util.StringUtils.stringifyException(cause)); - } - } - } - - /** - * Locate the krb5.conf file and dump it. - * No-op on windows. - * @throws IOException - */ - private void validateKrb5File() throws IOException { - if (!Shell.WINDOWS) { - title("Locating Kerberos configuration file"); - String krbPath = "/etc/krb5.conf"; - String jvmKrbPath = System.getProperty(JAVA_SECURITY_KRB5_CONF); - if (jvmKrbPath != null) { - println("Setting kerberos path from sysprop %s: %s", - JAVA_SECURITY_KRB5_CONF, jvmKrbPath); - krbPath = jvmKrbPath; - } - - String krb5name = System.getenv(KRB5_CCNAME); - if (krb5name != null) { - println("Setting kerberos path from environment variable %s: %s", - KRB5_CCNAME, krb5name); - krbPath = krb5name; - if (jvmKrbPath != null) { - println("Warning - both %s and %s were set - %s takes priority", - JAVA_SECURITY_KRB5_CONF, KRB5_CCNAME, KRB5_CCNAME); - } - } - - File krbFile = new File(krbPath); - println("Kerberos configuration file = %s", krbFile); - failif(!krbFile.exists(), - CAT_KERBEROS, - "Kerberos configuration file %s not found", krbFile); - dump(krbFile); - } - } - - /** - * Dump a keytab: list all principals. - * @param keytabFile the keytab file - * @throws IOException IO problems - */ - public void dumpKeytab(File keytabFile) throws IOException { - title("Examining keytab %s", keytabFile); - File kt = keytabFile.getCanonicalFile(); - failif(!kt.exists(), CAT_CONFIG, "Keytab not found: %s", kt); - failif(!kt.isFile(), CAT_CONFIG, "Keytab is not a valid file: %s", kt); - - String[] names = getPrincipalNames(keytabFile.getCanonicalPath(), - Pattern.compile(".*")); - println("keytab entry count: %d", names.length); - for (String name : names) { - println(" %s", name); - } - println("-----"); - } - - /** - * Log in from a keytab, dump the UGI, validate it, then try and log in again. - * That second-time login catches JVM/Hadoop compatibility problems. - * @throws IOException - */ - private void loginFromKeytab() throws IOException { - UserGroupInformation ugi; - String identity; - if (keytab != null) { - File kt = keytab.getCanonicalFile(); - println("Using keytab %s principal %s", kt, principal); - identity = principal; - - failif(StringUtils.isEmpty(principal), CAT_KERBEROS, - "No principal defined"); - ugi = loginUserFromKeytabAndReturnUGI(principal, kt.getPath()); - dumpUGI(identity, ugi); - validateUGI(principal, ugi); - - title("Attempting to log in from keytab again"); - // package scoped -hence the reason why this class must be in the - // hadoop.security package - setShouldRenewImmediatelyForTests(true); - // attempt a new login - ugi.reloginFromKeytab(); - } else { - println("No keytab: logging is as current user"); - } - } - - /** - * Dump a UGI. - * @param title title of this section - * @param ugi UGI to dump - * @throws IOException - */ - private void dumpUGI(String title, UserGroupInformation ugi) - throws IOException { - title(title); - println("UGI instance = %s", ugi); - println("Has kerberos credentials: %b", ugi.hasKerberosCredentials()); - println("Authentication method: %s", ugi.getAuthenticationMethod()); - println("Real Authentication method: %s", - ugi.getRealAuthenticationMethod()); - title("Group names"); - for (String name : ugi.getGroupNames()) { - println(name); - } - title("Credentials"); - Credentials credentials = ugi.getCredentials(); - List<Text> secretKeys = credentials.getAllSecretKeys(); - title("Secret keys"); - if (!secretKeys.isEmpty()) { - for (Text secret: secretKeys) { - println("%s", secret); - } - } else { - println("(none)"); - } - - dumpTokens(ugi); - } - - /** - * Validate the UGI: verify it is kerberized. - * @param messagePrefix message in exceptions - * @param user user to validate - */ - private void validateUGI(String messagePrefix, UserGroupInformation user) { - failif(!user.hasKerberosCredentials(), - CAT_LOGIN, "%s: No kerberos credentials for %s", messagePrefix, user); - failif(user.getAuthenticationMethod() == null, - CAT_LOGIN, "%s: Null AuthenticationMethod for %s", messagePrefix, user); - } - - /** - * A cursory look at the {@code kinit} executable. - * If it is an absolute path: it must exist with a size > 0. - * If it is just a command, it has to be on the path. There's no check - * for that -but the PATH is printed out. - */ - private void validateKinitExecutable() { - String kinit = conf.getTrimmed(KERBEROS_KINIT_COMMAND, ""); - if (!kinit.isEmpty()) { - File kinitPath = new File(kinit); - println("%s = %s", KERBEROS_KINIT_COMMAND, kinitPath); - if (kinitPath.isAbsolute()) { - failif(!kinitPath.exists(), CAT_KERBEROS, - "%s executable does not exist: %s", - KERBEROS_KINIT_COMMAND, kinitPath); - failif(!kinitPath.isFile(), CAT_KERBEROS, - "%s path does not refer to a file: %s", - KERBEROS_KINIT_COMMAND, kinitPath); - failif(kinitPath.length() == 0, CAT_KERBEROS, - "%s file is empty: %s", - KERBEROS_KINIT_COMMAND, kinitPath); - } else { - println("Executable %s is relative -must be on the PATH", kinit); - printEnv("PATH"); - } - } - } - - /** - * Try to load the SASL resolver. - * @param saslPropsResolverKey key for the SASL resolver - */ - private void validateSasl(String saslPropsResolverKey) { - title("Resolving SASL property %s", saslPropsResolverKey); - String saslPropsResolver = conf.getTrimmed(saslPropsResolverKey); - try { - Class<? extends SaslPropertiesResolver> resolverClass = conf.getClass( - saslPropsResolverKey, - SaslPropertiesResolver.class, SaslPropertiesResolver.class); - println("Resolver is %s", resolverClass); - } catch (RuntimeException e) { - throw new KerberosDiagsFailure(CAT_SASL, e, - "Failed to load %s class %s", - saslPropsResolverKey, saslPropsResolver); - } - } - - /** - * Validate any JAAS entry referenced in the {@link #SUN_SECURITY_JAAS_FILE} - * property. - */ - private void validateJAAS() { - String jaasFilename = System.getProperty(SUN_SECURITY_JAAS_FILE); - if (jaasFilename != null) { - title("JAAS"); - File jaasFile = new File(jaasFilename); - println("JAAS file is defined in %s: %s", - SUN_SECURITY_JAAS_FILE, jaasFile); - failif(!jaasFile.exists(), CAT_JAAS, - "JAAS file does not exist: %s", jaasFile); - failif(!jaasFile.isFile(), CAT_JAAS, - "Specified JAAS file is not a file: %s", jaasFile); - } - } - - /** - * Dump all tokens of a user - * @param user user - */ - public void dumpTokens(UserGroupInformation user) { - Collection<Token<? extends TokenIdentifier>> tokens - = user.getCredentials().getAllTokens(); - title("Token Count: %d", tokens.size()); - for (Token<? extends TokenIdentifier> token : tokens) { - println("Token %s", token.getKind()); - } - } - - /** - * Set the System property to true; return the old value for caching - * @param sysprop property - * @return the previous value - */ - private boolean getAndSet(String sysprop) { - boolean old = Boolean.getBoolean(sysprop); - System.setProperty(sysprop, "true"); - return old; - } - - /** - * Flush all active output channels, including {@Code System.err}, - * so as to stay in sync with any JRE log messages. - */ - private void flush() { - if (out != null) { - out.flush(); - } else { - System.out.flush(); - } - System.err.flush(); - } - - /** - * Format and print a line of output. - * This goes to any output file, or - * is logged at info. The output is flushed before and after, to - * try and stay in sync with JRE logging. - * @param format format string - * @param args any arguments - */ - @VisibleForTesting - public void println(String format, Object... args) { - println(format(format, args)); - } - - /** - * Print a line of output. This goes to any output file, or - * is logged at info. The output is flushed before and after, to - * try and stay in sync with JRE logging. - * @param msg message string - */ - @VisibleForTesting - private void println(String msg) { - flush(); - if (out != null) { - out.println(msg); - } else { - LOG.info(msg); - } - flush(); - } - - /** - * Print a title entry - * @param format format string - * @param args any arguments - */ - private void title(String format, Object... args) { - println(""); - println(""); - String msg = "== " + format(format, args) + " =="; - println(msg); - println(""); - } - - /** - * Print a system property, or {@link #UNSET} if unset. - * @param property property to print - */ - private void printSysprop(String property) { - println("%s = \"%s\"", property, - System.getProperty(property, UNSET)); - } - - /** - * Print a configuration option, or {@link #UNSET} if unset. - * @param option option to print - */ - private void printConfOpt(String option) { - println("%s = \"%s\"", option, conf.get(option, UNSET)); - } - - /** - * Print an environment variable's name and value; printing - * {@link #UNSET} if it is not set - * @param variable environment variable - */ - private void printEnv(String variable) { - String env = System.getenv(variable); - println("%s = \"%s\"", variable, env != null ? env : UNSET); - } - - /** - * Dump any file to standard out; add a trailing newline - * @param file file to dump - * @throws IOException IO problems - */ - public void dump(File file) throws IOException { - try (FileInputStream in = new FileInputStream(file)) { - for (String line : IOUtils.readLines(in)) { - println("%s", line); - } - } - println(""); - } - - /** - * Format and raise a failure - * - * @param category category for exception - * @param message string formatting message - * @param args any arguments for the formatting - * @throws KerberosDiagsFailure containing the formatted text - */ - private void fail(String category, String message, Object... args) - throws KerberosDiagsFailure { - throw new KerberosDiagsFailure(category, message, args); - } - - /** - * Conditional failure with string formatted arguments - * @param condition failure condition - * @param category category for exception - * @param message string formatting message - * @param args any arguments for the formatting - * @throws KerberosDiagsFailure containing the formatted text - * if the condition was met - */ - private void failif(boolean condition, - String category, - String message, - Object... args) - throws KerberosDiagsFailure { - if (condition) { - fail(category, message, args); - } - } - - /** - * Format a string, treating a call where there are no varags values - * as a string to pass through unformatted. - * @param message message, which is either a format string + args, or - * a general string - * @param args argument array - * @return a string for printing. - */ - public static String format(String message, Object... args) { - if (args.length == 0) { - return message; - } else { - return String.format(message, args); - } - } - - /** - * Diagnostics failures return the exit code 41, "unauthorized". - * - * They have a category, initially for testing: the category can be - * validated without having to match on the entire string. - */ - public static class KerberosDiagsFailure extends ExitUtil.ExitException { - private final String category; - - public KerberosDiagsFailure(String category, String message) { - super(41, category + ": " + message); - this.category = category; - } - - public KerberosDiagsFailure(String category, String message, Object... args) { - this(category, format(message, args)); - } - - public KerberosDiagsFailure(String category, Throwable throwable, - String message, Object... args) { - this(category, message, args); - initCause(throwable); - } - - public String getCategory() { - return category; - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b7fcb09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java index bbb4c44..4743f28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java @@ -29,7 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Set; -import static org.apache.slider.api.ResourceKeys.NODE_FAILURE_THRESHOLD; +import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.NODE_BLACKLIST_THRESHOLD; /** * This tracks the container failures per node. If the failure counter exceeds @@ -52,7 +52,7 @@ public class ContainerFailureTracker { this.context = context; this.component = component; maxFailurePerNode = component.getComponentSpec().getConfiguration() - .getPropertyInt(NODE_FAILURE_THRESHOLD, 3); + .getPropertyInt(NODE_BLACKLIST_THRESHOLD, 3); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b7fcb09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerLaunchService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerLaunchService.java deleted file mode 100644 index 2037a3b..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerLaunchService.java +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.service.compinstance.ComponentInstance; -import org.apache.hadoop.yarn.service.provider.ProviderService; -import org.apache.hadoop.yarn.service.provider.ProviderFactory; -import org.apache.slider.api.resource.Application; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.core.launch.AbstractLauncher; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -public class ContainerLaunchService extends AbstractService{ - - protected static final Logger LOG = - LoggerFactory.getLogger(ContainerLaunchService.class); - - private ExecutorService executorService; - private SliderFileSystem fs; - - public ContainerLaunchService(SliderFileSystem fs) { - super(ContainerLaunchService.class.getName()); - this.fs = fs; - } - - @Override - public void serviceInit(Configuration conf) throws Exception { - executorService = Executors.newCachedThreadPool(); - super.serviceInit(conf); - } - - @Override - protected void serviceStop() throws Exception { - if (executorService != null) { - executorService.shutdownNow(); - } - super.serviceStop(); - } - - public void launchCompInstance(Application application, - ComponentInstance instance, Container container) { - ContainerLauncher launcher = - new ContainerLauncher(application, instance, container); - executorService.execute(launcher); - } - - private class ContainerLauncher implements Runnable { - public final Container container; - public final Application application; - public ComponentInstance instance; - - public ContainerLauncher( - Application application, - ComponentInstance instance, Container container) { - this.container = container; - this.application = application; - this.instance = instance; - } - - @Override public void run() { - org.apache.slider.api.resource.Component compSpec = instance.getCompSpec(); - ProviderService provider = ProviderFactory.getProviderService( - compSpec.getArtifact()); - AbstractLauncher launcher = new AbstractLauncher(fs, null); - try { - provider.buildContainerLaunchContext(launcher, application, - instance, fs, getConfig()); - instance.getComponent().getScheduler().getNmClient() - .startContainerAsync(container, - launcher.completeContainerLaunch()); - } catch (Exception e) { - LOG.error(instance.getCompInstanceId() - + ": Failed to launch container. ", e); - - } - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b7fcb09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java index 80668a0..c7616af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java @@ -21,9 +21,9 @@ package org.apache.hadoop.yarn.service; import com.google.common.cache.LoadingCache; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.ConfigFile; -import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.hadoop.yarn.service.api.records.Application; +import org.apache.hadoop.yarn.service.api.records.ConfigFile; +import org.apache.hadoop.yarn.service.utils.SliderFileSystem; public class ServiceContext { public Application application = null; http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b7fcb09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java index c22dec4..d099f8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java @@ -35,10 +35,12 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; import org.apache.hadoop.yarn.service.client.params.SliderAMArgs; +import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; +import org.apache.hadoop.yarn.service.servicemonitor.ServiceMonitor; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.BadClusterStateException; +import org.apache.hadoop.yarn.service.utils.SliderFileSystem; +import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,6 +70,7 @@ public class ServiceMaster extends CompositeService { context = new ServiceContext(); Path appDir = getAppDir(); + context.serviceHdfsDir = appDir.toString(); SliderFileSystem fs = new SliderFileSystem(conf); context.fs = fs; fs.setAppDir(appDir); http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b7fcb09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMonitor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMonitor.java deleted file mode 100644 index bc37614..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMonitor.java +++ /dev/null @@ -1,149 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.service.component.Component; -import org.apache.hadoop.yarn.service.compinstance.ComponentInstance; -import org.apache.hadoop.yarn.service.conf.YarnServiceConf; -import org.apache.slider.api.InternalKeys; -import org.apache.hadoop.yarn.service.component.ComponentEvent; -import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent; -import org.apache.hadoop.yarn.service.component.ComponentState; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.server.servicemonitor.ProbeStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.RUNNING_BUT_UNREADY; -import static org.apache.hadoop.yarn.service.component.ComponentEventType.FLEX; -import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.BECOME_NOT_READY; -import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.BECOME_READY; -import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.READY; - -public class ServiceMonitor extends AbstractService { - - private static final Logger LOG = - LoggerFactory.getLogger(ServiceMonitor.class); - - public ScheduledExecutorService executorService; - private Map<ContainerId, ComponentInstance> liveInstances = null; - private ServiceContext context; - private Configuration conf; - - public ServiceMonitor(String name, ServiceContext context) { - super(name); - liveInstances = context.scheduler.getLiveInstances(); - this.context = context; - } - - @Override - public void serviceInit(Configuration conf) throws Exception { - executorService = Executors.newScheduledThreadPool(1); - this.conf = conf; - super.serviceInit(conf); - } - - @Override - public void serviceStart() throws Exception { - long readinessCheckInterval = - YarnServiceConf.getLong(InternalKeys.MONITOR_INTERVAL, - InternalKeys.DEFAULT_MONITOR_INTERVAL, - context.application.getConfiguration(), conf); - - executorService - .scheduleAtFixedRate(new ReadinessChecker(), readinessCheckInterval, - readinessCheckInterval, TimeUnit.SECONDS); - - long failureResetInterval = SliderUtils - .getTimeRange(context.application.getConfiguration(), - ResourceKeys.CONTAINER_FAILURE_WINDOW, - ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS, - ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS, - ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES, 0); - - executorService - .scheduleAtFixedRate(new ContainerFailureReset(), failureResetInterval, - failureResetInterval, TimeUnit.SECONDS); - } - - @Override - public void serviceStop() throws Exception { - if (executorService != null) { - executorService.shutdownNow(); - } - } - - private class ReadinessChecker implements Runnable { - - @Override - public void run() { - - // check if the comp instance are ready - for (Map.Entry<ContainerId, ComponentInstance> entry : liveInstances - .entrySet()) { - ComponentInstance instance = entry.getValue(); - - ProbeStatus status = instance.ping(); - if (status.isSuccess()) { - if (instance.getState() == RUNNING_BUT_UNREADY) { - // synchronously update the state. - instance.handle( - new ComponentInstanceEvent(entry.getKey(), BECOME_READY)); - } - } else { - if (instance.getState() == READY) { - instance.handle( - new ComponentInstanceEvent(entry.getKey(), BECOME_NOT_READY)); - } - } - } - - for (Component component : context.scheduler.getAllComponents() - .values()) { - // If comp hasn't started yet and its dependencies are satisfied - if (component.getState() == ComponentState.INIT && component - .areDependenciesReady()) { - LOG.info("[COMPONENT {}]: Dependencies satisfied, ramping up.", - component.getName()); - ComponentEvent event = new ComponentEvent(component.getName(), FLEX) - .setDesired(component.getComponentSpec().getNumberOfContainers()); - component.handle(event); - } - } - } - } - - private class ContainerFailureReset implements Runnable { - @Override - public void run() { - for (Component component : context.scheduler.getAllComponents().values()) { - component.resetCompFailureCount(); - } - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b7fcb09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index 590655f..8c968dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -54,26 +54,25 @@ import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants; +import org.apache.hadoop.yarn.service.api.records.Application; +import org.apache.hadoop.yarn.service.api.records.ConfigFile; import org.apache.hadoop.yarn.service.compinstance.ComponentInstance; import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent; import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType; import org.apache.hadoop.yarn.service.component.Component; import org.apache.hadoop.yarn.service.component.ComponentEvent; import org.apache.hadoop.yarn.service.component.ComponentEventType; -import org.apache.hadoop.yarn.service.conf.SliderKeys; +import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; +import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService; import org.apache.hadoop.yarn.service.metrics.ServiceMetrics; import org.apache.hadoop.yarn.service.provider.ProviderUtils; +import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders; import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink; import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; +import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils; import org.apache.hadoop.yarn.util.BoundedAppender; -import org.apache.slider.api.RoleKeys; -import org.apache.slider.api.ServiceApiConstants; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.ConfigFile; -import org.apache.slider.core.registry.info.CustomRegistryConstants; -import org.apache.slider.core.zk.ZKIntegration; -import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -93,8 +92,8 @@ import java.util.concurrent.TimeUnit; import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY; import static org.apache.hadoop.registry.client.api.RegistryConstants.*; +import static org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants.*; import static org.apache.hadoop.yarn.service.component.ComponentEventType.*; -import static org.apache.slider.api.ServiceApiConstants.*; /** * @@ -203,7 +202,7 @@ public class ServiceScheduler extends CompositeService { protected YarnRegistryViewForProviders createYarnRegistryOperations( ServiceContext context, RegistryOperations registryClient) { return new YarnRegistryViewForProviders(registryClient, - RegistryUtils.currentUser(), SliderKeys.APP_TYPE, app.getName(), + RegistryUtils.currentUser(), YarnServiceConstants.APP_TYPE, app.getName(), context.attemptId); } @@ -236,7 +235,8 @@ public class ServiceScheduler extends CompositeService { DefaultMetricsSystem.shutdown(); if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { - serviceTimelinePublisher.serviceAttemptUnregistered(context); + serviceTimelinePublisher + .serviceAttemptUnregistered(context, diagnostics.toString()); } // Cleanup each component instance. no need to release containers as // they will be automatically released by RM @@ -285,7 +285,7 @@ public class ServiceScheduler extends CompositeService { LOG.error("Failed to get user.", e); } globalTokens - .put(SERVICE_ZK_PATH, ZKIntegration.mkClusterPath(user, app.getName())); + .put(SERVICE_ZK_PATH, ServiceRegistryUtils.mkClusterPath(user, app.getName())); globalTokens.put(ServiceApiConstants.USER, user); String dnsDomain = getConfig().getTrimmed(KEY_DNS_DOMAIN); @@ -336,7 +336,7 @@ public class ServiceScheduler extends CompositeService { context.configCache = configFileCache; } - protected void registerServiceInstance(ApplicationAttemptId attemptId, + private void registerServiceInstance(ApplicationAttemptId attemptId, Application application) throws IOException { LOG.info("Registering " + attemptId + ", " + application.getName() + " into registry"); @@ -345,11 +345,11 @@ public class ServiceScheduler extends CompositeService { attemptId.getApplicationId().toString()); serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE, PersistencePolicies.APPLICATION); - serviceRecord.description = "Slider Application Master"; + serviceRecord.description = "Yarn Service Master"; serviceRecord.addExternalEndpoint(RegistryTypeUtils - .ipcEndpoint(CustomRegistryConstants.AM_IPC_PROTOCOL, - new InetSocketAddress(5000))); // FIXME + .ipcEndpoint("classpath:org.apache.hadoop.yarn.service.appmaster.ipc", + context.clientAMService.getBindAddress())); // set any provided attributes setUserProvidedServiceRecordAttributes(application.getConfiguration(), @@ -376,13 +376,13 @@ public class ServiceScheduler extends CompositeService { } }); if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { - serviceTimelinePublisher.serviceAttemptRegistered(app); + serviceTimelinePublisher.serviceAttemptRegistered(app, getConfig()); } } private void setUserProvidedServiceRecordAttributes( - org.apache.slider.api.resource.Configuration conf, ServiceRecord record) { - String prefix = RoleKeys.SERVICE_RECORD_ATTRIBUTE_PREFIX; + org.apache.hadoop.yarn.service.api.records.Configuration conf, ServiceRecord record) { + String prefix = "service.record.attribute"; for (Map.Entry<String, String> entry : conf.getProperties().entrySet()) { if (entry.getKey().startsWith(prefix)) { String key = entry.getKey().substring(prefix.length() + 1); @@ -395,10 +395,10 @@ public class ServiceScheduler extends CompositeService { long allocateId = 0; // sort components by dependencies - Collection<org.apache.slider.api.resource.Component> sortedComponents = + Collection<org.apache.hadoop.yarn.service.api.records.Component> sortedComponents = ServiceApiUtil.sortByDependencies(app.getComponents()); - for (org.apache.slider.api.resource.Component compSpec : sortedComponents) { + for (org.apache.hadoop.yarn.service.api.records.Component compSpec : sortedComponents) { Component component = new Component(compSpec, allocateId, context); componentsById.put(allocateId, component); componentsByName.put(component.getName(), component); @@ -517,7 +517,7 @@ public class ServiceScheduler extends CompositeService { @Override public float getProgress() { // get running containers over desired containers long total = 0; - for (org.apache.slider.api.resource.Component component : app + for (org.apache.hadoop.yarn.service.api.records.Component component : app .getComponents()) { total += component.getNumberOfContainers(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b7fcb09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/constants/ServiceApiConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/constants/ServiceApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/constants/ServiceApiConstants.java new file mode 100644 index 0000000..cf9e31f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/constants/ServiceApiConstants.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.api.constants; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$; + +/** + * This class defines constants that can be used in input spec for + * variable substitutions + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public interface ServiceApiConstants { + + // Constants for service + String SERVICE_NAME = $("SERVICE_NAME"); + + String SERVICE_NAME_LC = $("SERVICE_NAME.lc"); + + String USER = $("USER"); + + String DOMAIN = $("DOMAIN"); + + // Constants for component + String COMPONENT_NAME = $("COMPONENT_NAME"); + + String COMPONENT_NAME_LC = $("COMPONENT_NAME.lc"); + + String COMPONENT_INSTANCE_NAME = $("COMPONENT_INSTANCE_NAME"); + + // Constants for component instance + String COMPONENT_ID = $("COMPONENT_ID"); + + String CONTAINER_ID = $("CONTAINER_ID"); + + // Constants for default cluster ZK + String CLUSTER_ZK_QUORUM = $("CLUSTER_ZK_QUORUM"); + + // URI for the default cluster fs + String CLUSTER_FS_URI = $("CLUSTER_FS_URI"); + + // the host component of the cluster fs UI + String CLUSTER_FS_HOST = $("CLUSTER_FS_HOST"); + + // Path in zookeeper for a specific service + String SERVICE_ZK_PATH = $("SERVICE_ZK_PATH"); + + // Constants for service specific hdfs dir + String SERVICE_HDFS_DIR = $("SERVICE_HDFS_DIR"); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b7fcb09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Application.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Application.java new file mode 100644 index 0000000..f9e5154 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Application.java @@ -0,0 +1,466 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.api.records; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * An Application resource has the following attributes. + **/ +@InterfaceAudience.Public +@InterfaceStability.Unstable +@ApiModel(description = "An Application resource has the following attributes.") +@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00") +@XmlRootElement +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ "name", "state", "resource", "number_of_containers", + "lifetime", "containers" }) +public class Application extends BaseResource { + private static final long serialVersionUID = -4491694636566094885L; + + private String name = null; + private String id = null; + private Artifact artifact = null; + private Resource resource = null; + private String launchCommand = null; + private Date launchTime = null; + private Long numberOfContainers = null; + private Long numberOfRunningContainers = null; + private Long lifetime = null; + private PlacementPolicy placementPolicy = null; + private List<Component> components = new ArrayList<>(); + private Configuration configuration = new Configuration(); + private List<Container> containers = new ArrayList<>(); + private ApplicationState state = null; + private Map<String, String> quicklinks = new HashMap<>(); + private String queue = null; + + /** + * A unique application name. + **/ + public Application name(String name) { + this.name = name; + return this; + } + + @ApiModelProperty(example = "null", required = true, value = "A unique application name.") + @JsonProperty("name") + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + /** + * A unique application id. + **/ + public Application id(String id) { + this.id = id; + return this; + } + + @ApiModelProperty(example = "null", value = "A unique application id.") + @JsonProperty("id") + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + /** + * Artifact of single-component applications. Mandatory if components + * attribute is not specified. + **/ + public Application artifact(Artifact artifact) { + this.artifact = artifact; + return this; + } + + @ApiModelProperty(example = "null", value = "Artifact of single-component applications. Mandatory if components attribute is not specified.") + @JsonProperty("artifact") + public Artifact getArtifact() { + return artifact; + } + + public void setArtifact(Artifact artifact) { + this.artifact = artifact; + } + + /** + * Resource of single-component applications or the global default for + * multi-component applications. Mandatory if it is a single-component + * application and if cpus and memory are not specified at the Application + * level. + **/ + public Application resource(Resource resource) { + this.resource = resource; + return this; + } + + @ApiModelProperty(example = "null", value = "Resource of single-component applications or the global default for multi-component applications. Mandatory if it is a single-component application and if cpus and memory are not specified at the Application level.") + @JsonProperty("resource") + public Resource getResource() { + return resource; + } + + public void setResource(Resource resource) { + this.resource = resource; + } + + /** + * The custom launch command of an application component (optional). If not + * specified for applications with docker images say, it will default to the + * default start command of the image. If there is a single component in this + * application, you can specify this without the need to have a 'components' + * section. + **/ + public Application launchCommand(String launchCommand) { + this.launchCommand = launchCommand; + return this; + } + + @ApiModelProperty(example = "null", value = "The custom launch command of an application component (optional). If not specified for applications with docker images say, it will default to the default start command of the image. If there is a single component in this application, you can specify this without the need to have a 'components' section.") + @JsonProperty("launch_command") + public String getLaunchCommand() { + return launchCommand; + } + + @XmlElement(name = "launch_command") + public void setLaunchCommand(String launchCommand) { + this.launchCommand = launchCommand; + } + + /** + * The time when the application was created, e.g. 2016-03-16T01:01:49.000Z. + **/ + public Application launchTime(Date launchTime) { + this.launchTime = launchTime == null ? null : (Date) launchTime.clone(); + return this; + } + + @ApiModelProperty(example = "null", value = "The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.") + @JsonProperty("launch_time") + public Date getLaunchTime() { + return launchTime == null ? null : (Date) launchTime.clone(); + } + + @XmlElement(name = "launch_time") + public void setLaunchTime(Date launchTime) { + this.launchTime = launchTime == null ? null : (Date) launchTime.clone(); + } + + /** + * Number of containers for each app-component in the application. Each + * app-component can further override this app-level global default. + **/ + public Application numberOfContainers(Long numberOfContainers) { + this.numberOfContainers = numberOfContainers; + return this; + } + + @ApiModelProperty(example = "null", value = "Number of containers for each app-component in the application. Each app-component can further override this app-level global default.") + @JsonProperty("number_of_containers") + public Long getNumberOfContainers() { + return numberOfContainers; + } + + @XmlElement(name = "number_of_containers") + public void setNumberOfContainers(Long numberOfContainers) { + this.numberOfContainers = numberOfContainers; + } + + /** + * In get response this provides the total number of running containers for + * this application (across all components) at the time of request. Note, a + * subsequent request can return a different number as and when more + * containers get allocated until it reaches the total number of containers or + * if a flex request has been made between the two requests. + **/ + public Application numberOfRunningContainers(Long numberOfRunningContainers) { + this.numberOfRunningContainers = numberOfRunningContainers; + return this; + } + + @ApiModelProperty(example = "null", value = "In get response this provides the total number of running containers for this application (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.") + @JsonProperty("number_of_running_containers") + public Long getNumberOfRunningContainers() { + return numberOfRunningContainers; + } + + @XmlElement(name = "number_of_running_containers") + public void setNumberOfRunningContainers(Long numberOfRunningContainers) { + this.numberOfRunningContainers = numberOfRunningContainers; + } + + /** + * Life time (in seconds) of the application from the time it reaches the + * RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For + * unlimited lifetime do not set a lifetime value. + **/ + public Application lifetime(Long lifetime) { + this.lifetime = lifetime; + return this; + } + + @ApiModelProperty(example = "null", value = "Life time (in seconds) of the application from the time it reaches the RUNNING_BUT_UNREADY state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.") + @JsonProperty("lifetime") + public Long getLifetime() { + return lifetime; + } + + public void setLifetime(Long lifetime) { + this.lifetime = lifetime; + } + + /** + * Advanced scheduling and placement policies (optional). If not specified, it + * defaults to the default placement policy of the app owner. The design of + * placement policies are in the works. It is not very clear at this point, + * how policies in conjunction with labels be exposed to application owners. + * This is a placeholder for now. The advanced structure of this attribute + * will be determined by YARN-4902. + **/ + public Application placementPolicy(PlacementPolicy placementPolicy) { + this.placementPolicy = placementPolicy; + return this; + } + + @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the app owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to application owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.") + @JsonProperty("placement_policy") + public PlacementPolicy getPlacementPolicy() { + return placementPolicy; + } + + @XmlElement(name = "placement_policy") + public void setPlacementPolicy(PlacementPolicy placementPolicy) { + this.placementPolicy = placementPolicy; + } + + /** + * Components of an application. + **/ + public Application components(List<Component> components) { + this.components = components; + return this; + } + + @ApiModelProperty(example = "null", value = "Components of an application.") + @JsonProperty("components") + public List<Component> getComponents() { + return components; + } + + public void setComponents(List<Component> components) { + this.components = components; + } + + public void addComponent(Component component) { + components.add(component); + } + + public Component getComponent(String name) { + for (Component component : components) { + if (component.getName().equals(name)) { + return component; + } + } + return null; + } + + /** + * Config properties of an application. Configurations provided at the + * application/global level are available to all the components. Specific + * properties can be overridden at the component level. + **/ + public Application configuration(Configuration configuration) { + this.configuration = configuration; + return this; + } + + @ApiModelProperty(example = "null", value = "Config properties of an application. Configurations provided at the application/global level are available to all the components. Specific properties can be overridden at the component level.") + @JsonProperty("configuration") + public Configuration getConfiguration() { + return configuration; + } + + public void setConfiguration(Configuration configuration) { + this.configuration = configuration; + } + + /** + * Containers of a started application. Specifying a value for this attribute + * for the POST payload raises a validation error. This blob is available only + * in the GET response of a started application. + **/ + public Application containers(List<Container> containers) { + this.containers = containers; + return this; + } + + @ApiModelProperty(example = "null", value = "Containers of a started application. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application.") + @JsonProperty("containers") + public List<Container> getContainers() { + return containers; + } + + public void setContainers(List<Container> containers) { + this.containers = containers; + } + + public void addContainer(Container container) { + this.containers.add(container); + } + + /** + * State of the application. Specifying a value for this attribute for the + * POST payload raises a validation error. This attribute is available only in + * the GET response of a started application. + **/ + public Application state(ApplicationState state) { + this.state = state; + return this; + } + + @ApiModelProperty(example = "null", value = "State of the application. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started application.") + @JsonProperty("state") + public ApplicationState getState() { + return state; + } + + public void setState(ApplicationState state) { + this.state = state; + } + + /** + * A blob of key-value pairs of quicklinks to be exported for an application. + **/ + public Application quicklinks(Map<String, String> quicklinks) { + this.quicklinks = quicklinks; + return this; + } + + @ApiModelProperty(example = "null", value = "A blob of key-value pairs of quicklinks to be exported for an application.") + @JsonProperty("quicklinks") + public Map<String, String> getQuicklinks() { + return quicklinks; + } + + public void setQuicklinks(Map<String, String> quicklinks) { + this.quicklinks = quicklinks; + } + + /** + * The YARN queue that this application should be submitted to. + **/ + public Application queue(String queue) { + this.queue = queue; + return this; + } + + @ApiModelProperty(example = "null", value = "The YARN queue that this application should be submitted to.") + @JsonProperty("queue") + public String getQueue() { + return queue; + } + + public void setQueue(String queue) { + this.queue = queue; + } + + @Override + public boolean equals(java.lang.Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Application application = (Application) o; + return Objects.equals(this.name, application.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class Application {\n"); + + sb.append(" name: ").append(toIndentedString(name)).append("\n"); + sb.append(" id: ").append(toIndentedString(id)).append("\n"); + sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n"); + sb.append(" resource: ").append(toIndentedString(resource)).append("\n"); + sb.append(" launchCommand: ").append(toIndentedString(launchCommand)) + .append("\n"); + sb.append(" launchTime: ").append(toIndentedString(launchTime)) + .append("\n"); + sb.append(" numberOfContainers: ") + .append(toIndentedString(numberOfContainers)).append("\n"); + sb.append(" numberOfRunningContainers: ") + .append(toIndentedString(numberOfRunningContainers)).append("\n"); + sb.append(" lifetime: ").append(toIndentedString(lifetime)).append("\n"); + sb.append(" placementPolicy: ").append(toIndentedString(placementPolicy)) + .append("\n"); + sb.append(" components: ").append(toIndentedString(components)) + .append("\n"); + sb.append(" configuration: ").append(toIndentedString(configuration)) + .append("\n"); + sb.append(" containers: ").append(toIndentedString(containers)) + .append("\n"); + sb.append(" state: ").append(toIndentedString(state)).append("\n"); + sb.append(" quicklinks: ").append(toIndentedString(quicklinks)) + .append("\n"); + sb.append(" queue: ").append(toIndentedString(queue)).append("\n"); + sb.append("}"); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b7fcb09/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationState.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationState.java new file mode 100644 index 0000000..acef562 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationState.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.api.records; + +import io.swagger.annotations.ApiModel; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * The current state of an application. + **/ +@InterfaceAudience.Public +@InterfaceStability.Unstable +@ApiModel(description = "The current state of an application.") +@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00") +public enum ApplicationState { + ACCEPTED, STARTED, READY, STOPPED, FAILED; +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
