HDFS-9525. hadoop utilities need to support provided delegation tokens (HeeSoo Kim via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d22c4239 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d22c4239 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d22c4239 Branch: refs/heads/YARN-3926 Commit: d22c4239a40a1c7ed49c06038138f0e3f387b4a0 Parents: 95363bc Author: Allen Wittenauer <a...@apache.org> Authored: Fri Jan 22 12:15:22 2016 -0800 Committer: Allen Wittenauer <a...@apache.org> Committed: Fri Jan 22 12:15:22 2016 -0800 ---------------------------------------------------------------------- .../fs/CommonConfigurationKeysPublic.java | 3 ++ .../hadoop/security/UserGroupInformation.java | 22 +++++++++ .../src/main/resources/core-default.xml | 6 +++ .../security/TestUserGroupInformation.java | 48 +++++++++++++++++++- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 14 +++--- .../hdfs/web/resources/DelegationParam.java | 5 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../apache/hadoop/hdfs/web/TestWebHdfsUrl.java | 5 +- 8 files changed, 92 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index c9f758b..648ad59 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -311,6 +311,9 @@ public class CommonConfigurationKeysPublic { /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */ public static final String HADOOP_SECURITY_DNS_NAMESERVER_KEY = "hadoop.security.dns.nameserver"; + /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */ + public static final String HADOOP_TOKEN_FILES = + "hadoop.token.files"; /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */ public static final String HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN = http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index 28014bf..d33e1aa 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -20,6 +20,7 @@ package org.apache.hadoop.security; import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_TOKEN_FILES; import static org.apache.hadoop.util.PlatformName.IBM_JAVA; import java.io.File; @@ -70,6 +71,7 @@ import org.apache.hadoop.security.authentication.util.KerberosUtil; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.util.Shell; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import com.google.common.annotations.VisibleForTesting; @@ -821,6 +823,26 @@ public class UserGroupInformation { } loginUser = proxyUser == null ? realUser : createProxyUser(proxyUser, realUser); + String tokenFileLocation = System.getProperty(HADOOP_TOKEN_FILES); + if (tokenFileLocation == null) { + tokenFileLocation = conf.get(HADOOP_TOKEN_FILES); + } + if (tokenFileLocation != null) { + for (String tokenFileName: + StringUtils.getTrimmedStrings(tokenFileLocation)) { + if (tokenFileName.length() > 0) { + File tokenFile = new File(tokenFileName); + if (tokenFile.exists() && tokenFile.isFile()) { + Credentials cred = Credentials.readTokenStorageFile( + tokenFile, conf); + loginUser.addCredentials(cred); + } else { + LOG.info("tokenFile("+tokenFileName+") does not exist"); + } + } + } + } + String fileLocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION); if (fileLocation != null) { // Load the token storage file and put all of the tokens into the http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index bf60a25..c25f49e 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -466,6 +466,12 @@ for ldap providers in the same way as above does. <description>Maps kerberos principals to local user names</description> </property> +<property> + <name>hadoop.token.files</name> + <value></value> + <description>List of token cache files that have delegation tokens for hadoop service</description> +</property> + <!-- i/o properties --> <property> <name>io.file.buffer.size</name> http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java index 54cfc2d..6abe78c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java @@ -18,6 +18,7 @@ package org.apache.hadoop.security; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.security.SaslRpcServer.AuthMethod; @@ -35,6 +36,7 @@ import javax.security.auth.login.AppConfigurationEntry; import javax.security.auth.login.LoginContext; import java.io.BufferedReader; +import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.lang.reflect.Method; @@ -848,7 +850,9 @@ public class TestUserGroupInformation { */ @Test public void testPrivateTokenExclusion() throws Exception { - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + UserGroupInformation ugi = + UserGroupInformation.createUserForTesting( + "privateUser", new String[] { "PRIVATEUSERS" }); TestTokenIdentifier tokenId = new TestTokenIdentifier(); Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>( tokenId.getBytes(), "password".getBytes(), @@ -928,4 +932,46 @@ public class TestUserGroupInformation { } } } + + @Test + public void testExternalTokenFiles() throws Exception { + StringBuilder tokenFullPathnames = new StringBuilder(); + String tokenFilenames = "token1,token2"; + String tokenFiles[] = StringUtils.getTrimmedStrings(tokenFilenames); + final File testDir = new File("target", + TestUserGroupInformation.class.getName() + "-tmpDir").getAbsoluteFile(); + String testDirPath = testDir.getAbsolutePath(); + + // create path for token files + for (String tokenFile: tokenFiles) { + if (tokenFullPathnames.length() > 0) { + tokenFullPathnames.append(","); + } + tokenFullPathnames.append(testDirPath).append("/").append(tokenFile); + } + + // create new token and store it + TestTokenIdentifier tokenId = new TestTokenIdentifier(); + Credentials cred1 = new Credentials(); + Token<TestTokenIdentifier> token1 = new Token<TestTokenIdentifier>( + tokenId.getBytes(), "password".getBytes(), + tokenId.getKind(), new Text("token-service1")); + cred1.addToken(token1.getService(), token1); + cred1.writeTokenStorageFile(new Path(testDirPath, tokenFiles[0]), conf); + + Credentials cred2 = new Credentials(); + Token<TestTokenIdentifier> token2 = new Token<TestTokenIdentifier>( + tokenId.getBytes(), "password".getBytes(), + tokenId.getKind(), new Text("token-service2")); + cred2.addToken(token2.getService(), token2); + cred2.writeTokenStorageFile(new Path(testDirPath, tokenFiles[1]), conf); + + // set property for token external token files + System.setProperty("hadoop.token.files", tokenFullPathnames.toString()); + UserGroupInformation.setLoginUser(null); + UserGroupInformation tokenUgi = UserGroupInformation.getLoginUser(); + Collection<Token<?>> credsugiTokens = tokenUgi.getTokens(); + assertTrue(credsugiTokens.contains(token1)); + assertTrue(credsugiTokens.contains(token2)); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index f9c2c6e..d806d55 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -238,7 +238,7 @@ public class WebHdfsFileSystem extends FileSystem // the first getAuthParams() for a non-token op will either get the // internal token from the ugi or lazy fetch one protected synchronized Token<?> getDelegationToken() throws IOException { - if (canRefreshDelegationToken && delegationToken == null) { + if (delegationToken == null) { Token<?> token = tokenSelector.selectToken( new Text(getCanonicalServiceName()), ugi.getTokens()); // ugi tokens are usually indicative of a task which can't @@ -248,11 +248,13 @@ public class WebHdfsFileSystem extends FileSystem LOG.debug("Using UGI token: {}", token); canRefreshDelegationToken = false; } else { - token = getDelegationToken(null); - if (token != null) { - LOG.debug("Fetched new token: {}", token); - } else { // security is disabled - canRefreshDelegationToken = false; + if (canRefreshDelegationToken) { + token = getDelegationToken(null); + if (token != null) { + LOG.debug("Fetched new token: {}", token); + } else { // security is disabled + canRefreshDelegationToken = false; + } } } setDelegationToken(token); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java index 5329580..fda14438 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.hdfs.web.resources; -import org.apache.hadoop.security.UserGroupInformation; - /** Represents delegation token used for authentication. */ public class DelegationParam extends StringParam { /** Parameter name. */ @@ -33,8 +31,7 @@ public class DelegationParam extends StringParam { * @param str a string representation of the parameter value. */ public DelegationParam(final String str) { - super(DOMAIN, UserGroupInformation.isSecurityEnabled() - && str != null && !str.equals(DEFAULT)? str: null); + super(DOMAIN, str != null && !str.equals(DEFAULT)? str: null); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 940fa90..74b8aac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -55,6 +55,9 @@ Trunk (Unreleased) HDFS-9057. allow/disallow snapshots via webhdfs (Bramma Reddy Battula via vinayakumarb) + HDFS-9525. hadoop utilities need to support provided delegation + tokens (HeeSoo Kim via aw) + IMPROVEMENTS HDFS-4665. Move TestNetworkTopologyWithNodeGroup to common. http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java index 2913a97..24c13af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java @@ -195,7 +195,7 @@ public class TestWebHdfsUrl { checkQueryParams( new String[]{ GetOpParam.Op.GETFILESTATUS.toQueryString(), - new UserParam(ugi.getShortUserName()).toString() + new DelegationParam(tokenString).toString() }, fileStatusUrl); } @@ -280,8 +280,7 @@ public class TestWebHdfsUrl { checkQueryParams( new String[]{ GetOpParam.Op.GETFILESTATUS.toQueryString(), - new UserParam(ugi.getRealUser().getShortUserName()).toString(), - new DoAsParam(ugi.getShortUserName()).toString() + new DelegationParam(tokenString).toString() }, fileStatusUrl); }
