HADOOP-13805. UGI.getCurrentUser() fails if user does not have a keytab associated. Contributed by Xiao Chen, Wei-Chiu Chuang, Yongjun Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c26c241 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c26c241 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c26c241 Branch: refs/heads/HADOOP-13345 Commit: 4c26c241ad2b907dc02cecefa9846cbe2b0465ba Parents: 02c5494 Author: Yongjun Zhang <yzh...@cloudera.com> Authored: Thu Feb 16 22:25:37 2017 -0800 Committer: Yongjun Zhang <yzh...@cloudera.com> Committed: Fri Feb 17 09:18:50 2017 -0800 ---------------------------------------------------------------------- .../hadoop/fs/CommonConfigurationKeys.java | 11 ++ .../hadoop/security/UserGroupInformation.java | 129 +++++++++++++++---- .../hadoop/security/TestUGIWithMiniKdc.java | 1 + .../security/TestUserGroupInformation.java | 16 ++- 4 files changed, 127 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c26c241/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index b8a60d6..e53f71e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -353,6 +353,17 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { public static final String HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS = "hadoop.user.group.metrics.percentiles.intervals"; + /* When creating UGI with UserGroupInformation(Subject), treat the passed + * subject external if set to true, and assume the owner of the subject + * should do the credential renewal. + * + * This is a temporary config to solve the compatibility issue with + * HADOOP-13558 and HADOOP-13805 fix, see the jiras for discussions. + */ + public static final String HADOOP_TREAT_SUBJECT_EXTERNAL_KEY = + "hadoop.treat.subject.external"; + public static final boolean HADOOP_TREAT_SUBJECT_EXTERNAL_DEFAULT = false; + public static final String RPC_METRICS_QUANTILE_ENABLE = "rpc.metrics.quantile.enable"; public static final boolean RPC_METRICS_QUANTILE_ENABLE_DEFAULT = false; http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c26c241/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index 6574e55..a5c6226 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -18,6 +18,8 @@ package org.apache.hadoop.security; import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS; +import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_TREAT_SUBJECT_EXTERNAL_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_TREAT_SUBJECT_EXTERNAL_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_TOKEN_FILES; @@ -79,6 +81,7 @@ import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -273,6 +276,29 @@ public class UserGroupInformation { /** Min time (in seconds) before relogin for Kerberos */ private static long kerberosMinSecondsBeforeRelogin; /** The configuration to use */ + + /* + * This config is a temporary one for backward compatibility. + * It means whether to treat the subject passed to + * UserGroupInformation(Subject) as external. If true, + * - no renewal thread will be created to do the renew credential + * - reloginFromKeytab() and reloginFromTicketCache will not renew + * credential. + * and it assumes that the owner of the subject to renew; if false, it means + * to retain the old behavior prior to fixing HADOOP-13558 and HADOOP-13805. + * The default is false. + */ + private static boolean treatSubjectExternal = false; + + /* + * Some test need the renewal thread to be created even if it does + * UserGroupInformation.loginUserFromSubject(subject); + * The test code may set this variable to true via + * setEnableRenewThreadCreationForTest(boolean) + * method. + */ + private static boolean enableRenewThreadCreationForTest = false; + private static Configuration conf; @@ -338,6 +364,15 @@ public class UserGroupInformation { metrics.getGroupsQuantiles = getGroupsQuantiles; } } + + treatSubjectExternal = conf.getBoolean(HADOOP_TREAT_SUBJECT_EXTERNAL_KEY, + HADOOP_TREAT_SUBJECT_EXTERNAL_DEFAULT); + if (treatSubjectExternal) { + LOG.info("Config " + HADOOP_TREAT_SUBJECT_EXTERNAL_KEY + " is set to " + + "true, the owner of the subject passed to " + + " UserGroupInformation(Subject) is supposed to renew the " + + "credential."); + } } /** @@ -351,7 +386,19 @@ public class UserGroupInformation { public static void setConfiguration(Configuration conf) { initialize(conf, true); } - + + @InterfaceAudience.Private + @VisibleForTesting + static void setEnableRenewThreadCreationForTest(boolean b) { + enableRenewThreadCreationForTest = b; + } + + @InterfaceAudience.Private + @VisibleForTesting + static boolean getEnableRenewThreadCreationForTest() { + return enableRenewThreadCreationForTest; + } + @InterfaceAudience.Private @VisibleForTesting public static void reset() { @@ -361,6 +408,7 @@ public class UserGroupInformation { kerberosMinSecondsBeforeRelogin = 0; setLoginUser(null); HadoopKerberosName.setRules(null); + setEnableRenewThreadCreationForTest(false); } /** @@ -392,6 +440,7 @@ public class UserGroupInformation { private final User user; private final boolean isKeytab; private final boolean isKrbTkt; + private final boolean isLoginExternal; private static String OS_LOGIN_MODULE_NAME; private static Class<? extends Principal> OS_PRINCIPAL_CLASS; @@ -644,28 +693,28 @@ public class UserGroupInformation { /** * Create a UserGroupInformation for the given subject. * This does not change the subject or acquire new credentials. + * + * The creator of subject is responsible for renewing credentials. * @param subject the user's subject */ UserGroupInformation(Subject subject) { - this(subject, false); + this(subject, treatSubjectExternal); } /** * Create a UGI from the given subject. * @param subject the subject - * @param externalKeyTab if the subject's keytab is managed by the user. + * @param isLoginExternal if the subject's keytab is managed by other UGI. * Setting this to true will prevent UGI from attempting * to login the keytab, or to renew it. */ - private UserGroupInformation(Subject subject, final boolean externalKeyTab) { + private UserGroupInformation(Subject subject, final boolean isLoginExternal) { this.subject = subject; this.user = subject.getPrincipals(User.class).iterator().next(); - if (externalKeyTab) { - this.isKeytab = false; - } else { - this.isKeytab = KerberosUtil.hasKerberosKeyTab(subject); - } + + this.isKeytab = KerberosUtil.hasKerberosKeyTab(subject); this.isKrbTkt = KerberosUtil.hasKerberosTicket(subject); + this.isLoginExternal = isLoginExternal; } /** @@ -766,7 +815,7 @@ public class UserGroupInformation { User ugiUser = new User(loginPrincipals.iterator().next().getName(), AuthenticationMethod.KERBEROS, login); loginSubject.getPrincipals().add(ugiUser); - UserGroupInformation ugi = new UserGroupInformation(loginSubject); + UserGroupInformation ugi = new UserGroupInformation(loginSubject, false); ugi.setLogin(login); ugi.setAuthenticationMethod(AuthenticationMethod.KERBEROS); return ugi; @@ -782,7 +831,9 @@ public class UserGroupInformation { /** * Create a UserGroupInformation from a Subject with Kerberos principal. * - * @param subject The KerberosPrincipal to use in UGI + * @param subject The KerberosPrincipal to use in UGI. + * The creator of subject is responsible for + * renewing credentials. * * @throws IOException * @throws KerberosAuthException if the kerberos login fails @@ -843,6 +894,10 @@ public class UserGroupInformation { * Log in a user using the given subject * @param subject the subject to use when logging in a user, or null to * create a new subject. + * + * If subject is not null, the creator of subject is responsible for renewing + * credentials. + * * @throws IOException if login fails */ @InterfaceAudience.Public @@ -850,17 +905,25 @@ public class UserGroupInformation { public synchronized static void loginUserFromSubject(Subject subject) throws IOException { ensureInitialized(); + boolean externalSubject = false; try { if (subject == null) { subject = new Subject(); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Treat subject external: " + treatSubjectExternal + + ". When true, assuming keytab is managed extenally since " + + " logged in from subject"); + } + externalSubject = treatSubjectExternal; } LoginContext login = newLoginContext(authenticationMethod.getLoginAppName(), subject, new HadoopConfiguration()); login.login(); - LOG.debug("Assuming keytab is managed externally since logged in from" - + " subject."); - UserGroupInformation realUser = new UserGroupInformation(subject, true); + + UserGroupInformation realUser = + new UserGroupInformation(subject, externalSubject); realUser.setLogin(login); realUser.setAuthenticationMethod(authenticationMethod); // If the HADOOP_PROXY_USER environment variable or property @@ -959,11 +1022,23 @@ public class UserGroupInformation { return start + (long) ((end - start) * TICKET_RENEW_WINDOW); } + /** + * Should relogin if security is enabled using Kerberos, and + * the Subject is not owned by another UGI. + * @return true if this UGI should relogin + */ + private boolean shouldRelogin() { + return isSecurityEnabled() + && user.getAuthenticationMethod() == AuthenticationMethod.KERBEROS + && !isLoginExternal; + } + /**Spawn a thread to do periodic renewals of kerberos credentials*/ private void spawnAutoRenewalThreadForUserCreds() { - if (!isSecurityEnabled() - || user.getAuthenticationMethod() != AuthenticationMethod.KERBEROS - || isKeytab) { + if (getEnableRenewThreadCreationForTest()) { + LOG.warn("Spawning thread to auto renew user credential since " + + " enableRenewThreadCreationForTest was set to true."); + } else if (!shouldRelogin() || isKeytab) { return; } @@ -1092,7 +1167,7 @@ public class UserGroupInformation { start = Time.now(); login.login(); metrics.loginSuccess.add(Time.now() - start); - loginUser = new UserGroupInformation(subject); + loginUser = new UserGroupInformation(subject, false); loginUser.setLogin(login); loginUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS); } catch (LoginException le) { @@ -1156,8 +1231,9 @@ public class UserGroupInformation { public synchronized void checkTGTAndReloginFromKeytab() throws IOException { if (!isSecurityEnabled() || user.getAuthenticationMethod() != AuthenticationMethod.KERBEROS - || !isKeytab) + || !isKeytab) { return; + } KerberosTicket tgt = getTGT(); if (tgt != null && !shouldRenewImmediatelyForTests && Time.now() < getRefreshTime(tgt)) { @@ -1210,9 +1286,7 @@ public class UserGroupInformation { @InterfaceAudience.Public @InterfaceStability.Evolving public synchronized void reloginFromKeytab() throws IOException { - if (!isSecurityEnabled() - || user.getAuthenticationMethod() != AuthenticationMethod.KERBEROS - || !isKeytab) { + if (!shouldRelogin() || !isKeytab) { return; } @@ -1281,9 +1355,7 @@ public class UserGroupInformation { @InterfaceAudience.Public @InterfaceStability.Evolving public synchronized void reloginFromTicketCache() throws IOException { - if (!isSecurityEnabled() - || user.getAuthenticationMethod() != AuthenticationMethod.KERBEROS - || !isKrbTkt) { + if (!shouldRelogin() || !isKrbTkt) { return; } LoginContext login = getLogin(); @@ -1354,7 +1426,8 @@ public class UserGroupInformation { start = Time.now(); login.login(); metrics.loginSuccess.add(Time.now() - start); - UserGroupInformation newLoginUser = new UserGroupInformation(subject); + UserGroupInformation newLoginUser = + new UserGroupInformation(subject, false); newLoginUser.setLogin(login); newLoginUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS); @@ -1427,7 +1500,7 @@ public class UserGroupInformation { } Subject subject = new Subject(); subject.getPrincipals().add(new User(user)); - UserGroupInformation result = new UserGroupInformation(subject); + UserGroupInformation result = new UserGroupInformation(subject, false); result.setAuthenticationMethod(authMethod); return result; } @@ -1504,7 +1577,7 @@ public class UserGroupInformation { Set<Principal> principals = subject.getPrincipals(); principals.add(new User(user)); principals.add(new RealUser(realUser)); - UserGroupInformation result =new UserGroupInformation(subject); + UserGroupInformation result =new UserGroupInformation(subject, false); result.setAuthenticationMethod(AuthenticationMethod.PROXY); return result; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c26c241/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGIWithMiniKdc.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGIWithMiniKdc.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGIWithMiniKdc.java index 0016a65..2c6c7e4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGIWithMiniKdc.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGIWithMiniKdc.java @@ -75,6 +75,7 @@ public class TestUGIWithMiniKdc { SecurityUtil.setAuthenticationMethod( UserGroupInformation.AuthenticationMethod.KERBEROS, conf); UserGroupInformation.setConfiguration(conf); + UserGroupInformation.setEnableRenewThreadCreationForTest(true); LoginContext loginContext = null; try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c26c241/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java index ed0f58c..510987e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java @@ -61,6 +61,7 @@ import java.util.LinkedHashSet; import java.util.Set; import java.util.concurrent.TimeUnit; +import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_TREAT_SUBJECT_EXTERNAL_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL; @@ -1020,8 +1021,7 @@ public class TestUserGroupInformation { assertTrue(credsugiTokens.contains(token2)); } - @Test - public void testCheckTGTAfterLoginFromSubject() throws Exception { + private void testCheckTGTAfterLoginFromSubjectHelper() throws Exception { // security on, default is remove default realm SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf); UserGroupInformation.setConfiguration(conf); @@ -1031,6 +1031,7 @@ public class TestUserGroupInformation { KeyTab keytab = KeyTab.getInstance(); subject.getPrivateCredentials().add(keytab); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + ugi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws IOException { @@ -1042,6 +1043,17 @@ public class TestUserGroupInformation { }); } + @Test(expected = KerberosAuthException.class) + public void testCheckTGTAfterLoginFromSubject() throws Exception { + testCheckTGTAfterLoginFromSubjectHelper(); + } + + @Test + public void testCheckTGTAfterLoginFromSubjectFix() throws Exception { + conf.setBoolean(HADOOP_TREAT_SUBJECT_EXTERNAL_KEY, true); + testCheckTGTAfterLoginFromSubjectHelper(); + } + @Test public void testGetNextRetryTime() throws Exception { GenericTestUtils.setLogLevel(UserGroupInformation.LOG, Level.DEBUG); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org