SLIDER-1035 check for SASL resolver, kinit command, move things up and down, flush stderr on every line too. Stopping work on this for now
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3c5cea48 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3c5cea48 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3c5cea48 Branch: refs/heads/develop Commit: 3c5cea48007aeeea9e7b2fd586573438a318109d Parents: 64525aa Author: Steve Loughran <ste...@apache.org> Authored: Fri Jan 8 13:55:22 2016 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Fri Jan 8 13:55:22 2016 +0000 ---------------------------------------------------------------------- .../apache/hadoop/security/KerberosDiags.java | 218 +++++++++++++------ .../server/appmaster/SliderAppMaster.java | 2 +- .../apache/slider/client/TestDiagnostics.groovy | 23 +- .../funtest/commands/KDiagCommandIT.groovy | 3 +- 4 files changed, 171 insertions(+), 75 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c5cea48/slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java b/slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java index 4c16a48..4b64e9c 100644 --- a/slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java +++ b/slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java @@ -65,6 +65,8 @@ public class KerberosDiags implements Closeable { = "sun.security.krb5.debug"; public static final String SUN_SECURITY_SPNEGO_DEBUG = "sun.security.spnego.debug"; + public static final String KERBEROS_KINIT_COMMAND + = "hadoop.kerberos.kinit.command"; private final Configuration conf; private final List<String> services; @@ -86,48 +88,7 @@ public class KerberosDiags implements Closeable { @Override public void close() throws IOException { - if (out != null) { - out.flush(); - } - } - - private void println(String format, Object... args) { - String msg = String.format(format, args); - if (out != null) { - out.println(msg); - out.flush(); - } else { - LOG.info(msg); - } - } - - private void title(String format, Object... args) { - println(""); - println(""); - println(format, args); - println(""); - } - - private void printSysprop(String key) { - println("%s = \"%s\"", key, System.getProperty(key, "(unset)")); - } - - private void printConfOpt(String key) { - println("%s = \"%s\"", key, conf.get(key, "(unset)")); - } - - private void printEnv(String key) { - String env = System.getenv(key); - println("%s = \"%s\"", key, env != null ? env : "(unset)"); - } - - private void dump(File file) throws IOException { - try(FileInputStream in = new FileInputStream(file)) { - for (String line: IOUtils.readLines(in)) { - println(line); - } - } - println(""); + flush(); } /** @@ -150,7 +111,7 @@ public class KerberosDiags implements Closeable { println("Maximum AES encryption key length %d", aesLen); failif (aesLen < 256, "Java Cryptography Extensions are not installed on this JVM." - +"Kerberos is not going to work."); + +" Kerberos will not work."); boolean securityDisabled = SecurityUtil.getAuthenticationMethod(conf) .equals(UserGroupInformation.AuthenticationMethod.SIMPLE); if (securityDisabled) { @@ -178,31 +139,54 @@ public class KerberosDiags implements Closeable { printEnv(env); } for (String prop : new String[]{ - "hadoop.kerberos.kinit.command", + KERBEROS_KINIT_COMMAND, HADOOP_SECURITY_AUTHENTICATION, HADOOP_SECURITY_AUTHORIZATION, + "hadoop.kerberos.min.seconds.before.relogin", // not in 2.6 "hadoop.security.dns.interface", // not in 2.6 "hadoop.security.dns.nameserver", // not in 2.6 - HADOOP_SSL_ENABLED_KEY, HADOOP_RPC_PROTECTION, HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS, HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX, HADOOP_SECURITY_GROUP_MAPPING, + "hadoop.security.impersonation.provider.class", // not in 2.6 + "dfs.data.transfer.protection" // HDFS + }) { printConfOpt(prop); } + validateKrb5File(); + validateSasl(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS); + validateSasl("dfs.data.transfer.saslproperties.resolver.class"); + validateKinit(); + + boolean krb5Debug = getAndSet(SUN_SECURITY_KRB5_DEBUG); + boolean spnegoDebug = getAndSet(SUN_SECURITY_SPNEGO_DEBUG); + + try { + title("Logging in"); + UserGroupInformation loginUser = getLoginUser(); + dumpUser("Log in user", loginUser); + println("Ticket based login: %b", isLoginTicketBased()); + println("Keytab based login: %b", isLoginKeytabBased()); + validateUser("Login user", loginUser); + loginFromKeytab(); + + return true; + } finally { + // restore original system properties + System.setProperty(SUN_SECURITY_KRB5_DEBUG, + Boolean.toString(krb5Debug)); + System.setProperty(SUN_SECURITY_SPNEGO_DEBUG, + Boolean.toString(spnegoDebug)); + } + } - System.setProperty(SUN_SECURITY_KRB5_DEBUG, "true"); - System.setProperty(SUN_SECURITY_SPNEGO_DEBUG, "true"); - - title("Logging in"); - UserGroupInformation loginUser = getLoginUser(); - dumpUser("Log in user", loginUser); - println("Ticket based login: %b", isLoginTicketBased()); - println("Keytab based login: %b", isLoginKeytabBased()); - validateUser("Login user", loginUser); - - // locate KDC and dump it + /** + * Locate the krb5.conf file and dump it. No-op on windows + * @throws IOException + */ + private void validateKrb5File() throws IOException { if (!Shell.WINDOWS) { title("Locating Kerberos configuration file"); String krbPath = "/etc/krb5.conf"; @@ -230,7 +214,9 @@ public class KerberosDiags implements Closeable { "Kerberos configuration file %s not found", krbFile); dump(krbFile); } + } + private void loginFromKeytab() throws IOException { UserGroupInformation ugi; String identity; if (keytab != null) { @@ -253,7 +239,6 @@ public class KerberosDiags implements Closeable { } else { println("No keytab: logging is as current user"); } - return true; } private void dumpUser(String message, UserGroupInformation ugi) @@ -280,16 +265,7 @@ public class KerberosDiags implements Closeable { println("(none)"); } - title("Tokens"); - Collection<Token<? extends TokenIdentifier>> tokens - = credentials.getAllTokens(); - if (!tokens.isEmpty()) { - for (Token<? extends TokenIdentifier> token : tokens) { - println("%s", token); - } - } else { - println("(none)"); - } + dumpTokens(ugi); } private void validateUser(String message, UserGroupInformation user) { @@ -299,9 +275,113 @@ public class KerberosDiags implements Closeable { "%s: Null AuthenticationMethod for %s", message, user); } + private void validateKinit() { + String kinit = conf.getTrimmed(KERBEROS_KINIT_COMMAND, ""); + if (!kinit.isEmpty()) { + File kinitPath = new File(kinit); + println("%s = %s", KERBEROS_KINIT_COMMAND, kinitPath); + if (kinitPath.isAbsolute()) { + failif(!kinitPath.exists(), "%s executable does not exist: %s", + KERBEROS_KINIT_COMMAND, kinitPath); + failif(!kinitPath.isFile(), "%s path does not refer to a file: %s", + KERBEROS_KINIT_COMMAND, kinitPath); + } else { + println("Executable %s is relative -must be on the PATH", kinit); + printEnv("PATH"); + } + } + } + + private void validateSasl(String saslPropsResolverKey) { + title("Resolving SASL property %s", saslPropsResolverKey); + String saslPropsResolver = conf.getTrimmed(saslPropsResolverKey); + try { + Class<? extends SaslPropertiesResolver> resolverClass = conf.getClass( + saslPropsResolverKey, + SaslPropertiesResolver.class, SaslPropertiesResolver.class); + println("Resolver is %s", resolverClass.toString()); + } catch (RuntimeException e) { + throw new KerberosDiagsFailure(e, "Failed to load %s class %s", + saslPropsResolverKey, saslPropsResolver); + } + } + + public void dumpTokens(UserGroupInformation user) { + Collection<Token<? extends TokenIdentifier>> tokens + = user.getCredentials().getAllTokens(); + title("Token Count: %d", tokens.size()); + for (Token<? extends TokenIdentifier> token : tokens) { + println("Token %s", token.getKind()); + } + } + + /** + * Set the System property to true; return the old value for caching + * @param sysprop property + * @return the previous value + */ + private boolean getAndSet(String sysprop) { + boolean old = Boolean.getBoolean(sysprop); + System.setProperty(sysprop, "true"); + return old; + } + + /** + * Flush all active output channels, including {@Code System.err}, + * so as to stay in sync with any JRE log messages. + */ + private void flush() { + if (out != null) { + out.flush(); + } else { + System.out.flush(); + } + System.err.flush(); + } + + private void println(String format, Object... args) { + String msg = String.format(format, args); + if (out != null) { + out.println(msg); + } else { + LOG.info(msg); + } + flush(); + } + + private void title(String format, Object... args) { + println(""); + println(""); + String msg = "== " + String.format(format, args) + " =="; + println(msg); + println(""); + } + + private void printSysprop(String key) { + println("%s = \"%s\"", key, System.getProperty(key, "(unset)")); + } + + private void printConfOpt(String key) { + println("%s = \"%s\"", key, conf.get(key, "(unset)")); + } + + + private void printEnv(String key) { + String env = System.getenv(key); + println("%s = \"%s\"", key, env != null ? env : "(unset)"); + } + + private void dump(File file) throws IOException { + try (FileInputStream in = new FileInputStream(file)) { + for (String line : IOUtils.readLines(in)) { + println(line); + } + } + println(""); + } + /** * Format and raise a failure - * @param condition failure condition * @param message string formatting message * @param args any arguments for the formatting * @throws KerberosDiagsFailure containing the formatted text @@ -327,7 +407,7 @@ public class KerberosDiags implements Closeable { } /** - * Diagnostics failures include an exit code 41, "unauth" + * Diagnostics failures return the exit code 41, "unauthorized" */ public static class KerberosDiagsFailure extends ExitUtil.ExitException { public KerberosDiagsFailure( String message) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c5cea48/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index b793bed..a01dde9 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -1025,7 +1025,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService /** * Get the YARN application Attempt report as the logged in user * @param yarnClient client to the RM - * @return the appication report + * @return the application report * @throws YarnException * @throws IOException * @throws InterruptedException http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c5cea48/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy index 04f019f..ca22cbe 100644 --- a/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy @@ -21,16 +21,18 @@ package org.apache.slider.client import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.hadoop.yarn.conf.YarnConfiguration +import static org.apache.slider.common.Constants.SUN_SECURITY_KRB5_DEBUG import org.apache.slider.common.params.ActionDiagnosticArgs import org.apache.slider.common.params.Arguments +import org.apache.slider.common.params.ClientArgs import org.apache.slider.common.params.SliderActions +import org.apache.slider.common.tools.SliderUtils import org.apache.slider.core.main.ServiceLauncher import org.apache.slider.test.SliderTestBase import org.apache.slider.test.YarnMiniClusterTestBase import org.apache.slider.test.YarnZKMiniClusterTestBase import org.junit.Test -@CompileStatic @Slf4j class TestDiagnostics extends YarnZKMiniClusterTestBase { @@ -50,9 +52,24 @@ class TestDiagnostics extends YarnZKMiniClusterTestBase { diagnostics.client = true diagnostics.verbose = true describe("Verbose diagnostics") - - client.actionDiagnostic(diagnostics) + } + + /** + * help should print out help string and then succeed + * @throws Throwable + */ + @Test + public void testKDiag() throws Throwable { + ServiceLauncher launcher = launch(SliderClient, + SliderUtils.createConfiguration(), + [ + ClientArgs.ACTION_KDIAG, + ClientArgs.ARG_FAIL, + ClientArgs.ARG_SYSPROP, + define(SUN_SECURITY_KRB5_DEBUG, "true")]) + assert 0 == launcher.serviceExitCode } + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c5cea48/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy index 8d05df6..61325e2 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy @@ -35,8 +35,7 @@ public class KDiagCommandIT extends CommandTestBase implements Arguments { public void testKdiag() throws Throwable { SliderShell shell = new SliderShell([ SliderActions.ACTION_KDIAG, - ARG_FAIL, - ARG_SYSPROP, define(SUN_SECURITY_KRB5_DEBUG, "true") + ARG_FAIL ], [(ENV_JAAS_DEBUG): "true"] )
