SLIDER-1080 switch to $HADOOP_TOKEN_FILE_LOCATION as origin of oozie tokens; fall back to Conf.get("mapreduce.job.credentials.binary"). This is done in CredentialUtils and not in Launcher code. Also, HADOOP_JAAS_DEBUG is auto-propagated from client to AM, then AM to containers. If you want to debug Hadoop and Kerberos, you want to debug it everywhere.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f56a9311 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f56a9311 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f56a9311 Branch: refs/heads/develop Commit: f56a9311cebaa088762a6d973582b89b425e7936 Parents: f78465e Author: Steve Loughran <ste...@apache.org> Authored: Mon Feb 1 16:21:20 2016 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Mon Feb 1 16:21:20 2016 +0000 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 11 +++-- .../org/apache/slider/common/Constants.java | 2 +- .../apache/slider/common/SliderXmlConfKeys.java | 2 +- .../slider/core/launch/AbstractLauncher.java | 31 +++++++----- .../slider/core/launch/AppMasterLauncher.java | 51 ++------------------ .../slider/core/launch/ContainerLauncher.java | 10 +--- .../slider/core/launch/CredentialUtils.java | 12 ++++- .../server/appmaster/SliderAppMaster.java | 10 ++-- .../funtest/framework/CommandTestBase.groovy | 4 ++ .../funtest/commands/KDiagCommandIT.groovy | 2 +- 10 files changed, 55 insertions(+), 80 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 5f694e2..c141d25 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -208,6 +208,7 @@ import static org.apache.hadoop.registry.client.binding.RegistryUtils.*; import static org.apache.slider.api.InternalKeys.*; import static org.apache.slider.api.OptionKeys.*; import static org.apache.slider.api.ResourceKeys.*; +import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG; import static org.apache.slider.common.params.SliderActions.*; import static org.apache.slider.common.tools.SliderUtils.*; @@ -1915,8 +1916,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe Credentials credentials = null; if (clusterSecure) { // pick up oozie credentials - credentials = CredentialUtils.loadFromEnvironment( - System.getenv(), config); + credentials = CredentialUtils.loadFromEnvironment(System.getenv(), config); if (credentials == null) { // nothing from oozie, so build up directly credentials = new Credentials( @@ -1924,6 +1924,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe CredentialUtils.addRMRenewableFSDelegationTokens(config, sliderFileSystem.getFileSystem(), credentials); + + } else { + log.info("Using externally supplied credentials to launch AM"); } } @@ -2087,8 +2090,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe amLauncher.setEnv("LANG", "en_US.UTF-8"); amLauncher.setEnv("LC_ALL", "en_US.UTF-8"); amLauncher.setEnv("LANGUAGE", "en_US.UTF-8"); + amLauncher.maybeSetEnv(HADOOP_JAAS_DEBUG, + System.getenv(HADOOP_JAAS_DEBUG)); amLauncher.putEnv(getAmLaunchEnv(config)); - + for (Map.Entry<String, String> envs : getSystemEnv().entrySet()) { log.debug("System env {}={}", envs.getKey(), envs.getValue()); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/common/Constants.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/Constants.java b/slider-core/src/main/java/org/apache/slider/common/Constants.java index fdb3452..0e3559a 100644 --- a/slider-core/src/main/java/org/apache/slider/common/Constants.java +++ b/slider-core/src/main/java/org/apache/slider/common/Constants.java @@ -22,7 +22,7 @@ public class Constants { public static final int CONNECT_TIMEOUT = 10000; public static final int RPC_TIMEOUT = 15000; - public static final String ENV_JAAS_DEBUG = "HADOOP_JAAS_DEBUG"; + public static final String HADOOP_JAAS_DEBUG = "HADOOP_JAAS_DEBUG"; public static final String KRB5_CCNAME = "KRB5CCNAME"; public static final String JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java index 26109a7..72dd44f 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java @@ -202,5 +202,5 @@ public interface SliderXmlConfKeys { String DFS_NAMENODE_KEYTAB_FILE_KEY = "dfs.namenode.keytab.file"; String DFS_NAMENODE_DU_RESERVED_KEY = "dfs.namenode.resource.du.reserved"; - + String MAPREDUCE_JOB_CREDENTIALS_BINARY = "mapreduce.job.credentials.binary"; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java index f92ffb1..2ec3cda 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java @@ -58,7 +58,6 @@ public abstract class AbstractLauncher extends Configured { private static final Logger log = LoggerFactory.getLogger(AbstractLauncher.class); public static final String CLASSPATH = "CLASSPATH"; - public static final String MAPREDUCE_JOB_CREDENTIALS_BINARY = "mapreduce.job.credentials.binary"; /** * Filesystem to use for the launch */ @@ -67,20 +66,18 @@ public abstract class AbstractLauncher extends Configured { * Env vars; set up at final launch stage */ protected final Map<String, String> envVars = new HashMap<>(); - protected final MapOperations env = new MapOperations("env", envVars); protected final ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class); protected final List<String> commands = new ArrayList<>(20); protected final Map<String, LocalResource> localResources = new HashMap<>(); private final Map<String, ByteBuffer> serviceData = new HashMap<>(); - // security protected final Credentials credentials; protected LogAggregationContext logAggregationContext; /** - * Create instance + * Create instance. * @param conf configuration * @param coreFileSystem filesystem * @param credentials initial set of credentials -null is permitted @@ -93,15 +90,6 @@ public abstract class AbstractLauncher extends Configured { this.credentials = credentials != null ? credentials: new Credentials(); } - protected AbstractLauncher(Configuration conf, - CoreFileSystem fs) { - this(conf, fs, null); - } - - protected AbstractLauncher(CoreFileSystem fs) { - this(null, fs, null); - } - /** * Get the container. Until "completed", this isn't valid to launch. * @return the container to launch @@ -401,12 +389,29 @@ public abstract class AbstractLauncher extends Configured { public void setClasspath(ClasspathConstructor classpath) { setEnv(CLASSPATH, classpath.buildClasspath()); } + + /** + * Set an environment variable in the launch context + * @param var variable name + * @param value value (must be non null) + */ public void setEnv(String var, String value) { Preconditions.checkArgument(var != null, "null variable name"); Preconditions.checkArgument(value != null, "null value"); env.put(var, value); } + /** + * Set an environment variable if its value is non-null. + * @param var variable name + * @param value value (may be null) + */ + public void maybeSetEnv(String var, String value) { + if (value != null) { + setEnv(var, value); + } + } + public void putEnv(Map<String, String> map) { env.putAll(map); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java index 091b80e..7190c3a 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java @@ -19,11 +19,7 @@ package org.apache.slider.core.launch; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Priority; @@ -38,8 +34,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.text.DateFormat; -import java.util.Date; import java.util.Map; import java.util.Set; @@ -194,7 +188,9 @@ public class AppMasterLauncher extends AbstractLauncher { if (secureCluster) { //tokens - addSecurityTokens(); + log.debug("Credentials: {}", + CredentialUtils.dumpTokens(getCredentials(), "\n")); + } else { propagateUsernameInInsecureCluster(); } @@ -204,47 +200,6 @@ public class AppMasterLauncher extends AbstractLauncher { } /** - * Add the security tokens if this is a secure cluster - * @throws IOException - */ - private void addSecurityTokens() throws IOException { - - CredentialUtils.addRMRenewableFSDelegationTokens(getConf(), - coreFileSystem.getFileSystem(), credentials); - - String tokenRenewer = CredentialUtils.getRMPrincipal(getConf()); - - Token<? extends TokenIdentifier>[] tokens = null; - boolean tokensProvided = getConf().get(MAPREDUCE_JOB_CREDENTIALS_BINARY) != - null; - if (!tokensProvided) { - // For now, only getting tokens for the default file-system. - FileSystem fs = coreFileSystem.getFileSystem(); - tokens = fs.addDelegationTokens(tokenRenewer, credentials); - } - // obtain the token expiry from the first token - should be the same for all - // HDFS tokens - if (tokens != null && tokens.length > 0) { - AbstractDelegationTokenIdentifier id = - (AbstractDelegationTokenIdentifier) tokens[0].decodeIdentifier(); - Date d = new Date(id.getIssueDate() + 24 * 60 * 60 * 1000); - log.info( - "HDFS delegation tokens for AM launch context require renewal by {}", - DateFormat.getDateTimeInstance().format(d)); - } else { - if (!tokensProvided) { - log.warn("No HDFS delegation tokens obtained for AM launch context"); - } else { - log.info("Tokens provided via " + MAPREDUCE_JOB_CREDENTIALS_BINARY + - " property " - + "being used for AM launch"); - } - - } - - } - - /** * Submit the application. * @return a launched application representing the submitted application * @throws IOException http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java index e586743..d220928 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java @@ -43,18 +43,12 @@ public class ContainerLauncher extends AbstractLauncher { public ContainerLauncher(Configuration conf, CoreFileSystem coreFileSystem, - Container container, Credentials credentials) { + Container container, + Credentials credentials) { super(conf, coreFileSystem, credentials); this.container = container; } - public ContainerLauncher(Configuration conf, - CoreFileSystem fs, - Container container) { - super(conf, fs); - this.container = container; - } - /** * This code is in the dist shell examples -it's been moved here * so that if it is needed, it's still here http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java b/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java index 32068e2..3245c13 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java @@ -31,6 +31,7 @@ import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.slider.common.SliderXmlConfKeys; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,6 +56,9 @@ import static org.apache.hadoop.yarn.conf.YarnConfiguration.*; */ public final class CredentialUtils { + public static final String JOB_CREDENTIALS_BINARY + = SliderXmlConfKeys.MAPREDUCE_JOB_CREDENTIALS_BINARY; + private CredentialUtils() { } @@ -94,12 +98,17 @@ public final class CredentialUtils { Configuration conf) throws IOException { String tokenFilename = env.get(HADOOP_TOKEN_FILE_LOCATION); + String source = HADOOP_TOKEN_FILE_LOCATION; + if (tokenFilename == null) { + tokenFilename = conf.get(JOB_CREDENTIALS_BINARY); + source = "Configuration option " + JOB_CREDENTIALS_BINARY; + } if (tokenFilename != null) { // use delegation tokens, i.e. from Oozie File file = new File(tokenFilename.trim()); String details = String.format("Token File %s from environment variable %s", file, - HADOOP_TOKEN_FILE_LOCATION); + source); LOG.debug("Using {}", details); if (!file.exists()) { throw new FileNotFoundException("No " + details); @@ -171,7 +180,6 @@ public final class CredentialUtils { Preconditions.checkArgument(credentials != null); if (UserGroupInformation.isSecurityEnabled()) { String tokenRenewer = CredentialUtils.getRMPrincipal(conf); - Token<? extends TokenIdentifier>[] tokens = null; return fs.addDelegationTokens(tokenRenewer, credentials); } return null; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 82c9fb9..9a284e3 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -64,6 +64,8 @@ import org.apache.hadoop.yarn.client.api.async.NMClientAsync; import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import static org.apache.hadoop.yarn.conf.YarnConfiguration.*; +import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG; + import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.YarnRPC; @@ -775,7 +777,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService uploadServerCertForLocalization(clustername, fs); } - webAppPort = getPortToRequest(); if (webAppPort == 0) { // failure to find a port @@ -914,6 +915,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService if (hadoop_user_name != null) { envVars.put(HADOOP_USER_NAME, hadoop_user_name); } + String debug_kerberos = System.getenv(HADOOP_JAAS_DEBUG); + if (debug_kerberos != null) { + envVars.put(HADOOP_JAAS_DEBUG, debug_kerberos); + } } String rolesTmpSubdir = appMasterContainerID.toString() + "/roles"; @@ -1208,8 +1213,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService log.error("User is not holding on a keytab in a secure deployment:" + " slider will fail as tokens expire"); } - Credentials credentials = - user.getCredentials(); + Credentials credentials = user.getCredentials(); Iterator<Token<? extends TokenIdentifier>> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy index a887271..1a0d894 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.slider.api.StatusKeys import org.apache.slider.api.types.NodeInformationList import org.apache.slider.client.SliderClient +import org.apache.slider.common.Constants import org.apache.slider.common.SliderKeys import org.apache.slider.common.SliderXmlConfKeys import org.apache.slider.api.ClusterDescription @@ -760,6 +761,9 @@ abstract class CommandTestBase extends SliderTestUtils { maybeAddCommandOption(commands, [ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL], SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL)); + + commands << ARG_COMP_OPT << SliderKeys.COMPONENT_AM << + "env." + Constants.HADOOP_JAAS_DEBUG << "true"; commands.addAll(extraArgs) SliderShell shell = new SliderShell(commands) if (0 != shell.execute()) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f56a9311/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy index ebdd00c..5fd46fa 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/KDiagCommandIT.groovy @@ -37,7 +37,7 @@ public class KDiagCommandIT extends CommandTestBase implements Arguments { SliderActions.ACTION_KDIAG, ARG_KEYLEN, "128" ], - [(ENV_JAAS_DEBUG): "true"] + [(HADOOP_JAAS_DEBUG): "true"] ) shell.execute() assertSuccess(shell)