MAPREDUCE-7073. Optimize TokenCache#obtainTokensForNamenodesInternal Signed-off-by: Akira Ajisaka <aajis...@apache.org> (cherry picked from commit 1a95a4524a8c6c7be601ce8b92640a6a76164a2c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/58bcb90e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/58bcb90e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/58bcb90e Branch: refs/heads/YARN-8200 Commit: 58bcb90e7039bd4cec116fe66f1b1428dd493372 Parents: 06632c0 Author: Bibin A Chundatt <bibinchund...@apache.org> Authored: Wed May 2 16:14:28 2018 +0900 Committer: Akira Ajisaka <aajis...@apache.org> Committed: Wed May 2 16:45:24 2018 +0900 ---------------------------------------------------------------------- .../hadoop/mapreduce/security/TokenCache.java | 14 +++++++++----- .../hadoop/mapreduce/security/TestTokenCache.java | 18 +++++++++--------- 2 files changed, 18 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/58bcb90e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java index 78f6c16..8cd407c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java @@ -24,6 +24,7 @@ import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -96,8 +97,9 @@ public class TokenCache { for(Path p: ps) { fsSet.add(p.getFileSystem(conf)); } + String masterPrincipal = Master.getMasterPrincipal(conf); for (FileSystem fs : fsSet) { - obtainTokensForNamenodesInternal(fs, credentials, conf); + obtainTokensForNamenodesInternal(fs, credentials, conf, masterPrincipal); } } @@ -122,15 +124,17 @@ public class TokenCache { * @param conf * @throws IOException */ - static void obtainTokensForNamenodesInternal(FileSystem fs, - Credentials credentials, Configuration conf) throws IOException { + static void obtainTokensForNamenodesInternal(FileSystem fs, + Credentials credentials, Configuration conf, String renewer) + throws IOException { // RM skips renewing token with empty renewer String delegTokenRenewer = ""; if (!isTokenRenewalExcluded(fs, conf)) { - delegTokenRenewer = Master.getMasterPrincipal(conf); - if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) { + if (StringUtils.isEmpty(renewer)) { throw new IOException( "Can't get Master Kerberos principal for use as renewer"); + } else { + delegTokenRenewer = renewer; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/58bcb90e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java index 127f8ae..a44e533 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java @@ -56,8 +56,8 @@ public class TestTokenCache { @Test public void testObtainTokens() throws Exception { Credentials credentials = new Credentials(); - FileSystem fs = mock(FileSystem.class); - TokenCache.obtainTokensForNamenodesInternal(fs, credentials, conf); + FileSystem fs = mock(FileSystem.class); + TokenCache.obtainTokensForNamenodesInternal(fs, credentials, conf, renewer); verify(fs).addDelegationTokens(eq(renewer), eq(credentials)); } @@ -105,23 +105,23 @@ public class TestTokenCache { checkToken(creds, newerToken1); // get token for fs1, see that fs2's token was loaded - TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer); checkToken(creds, newerToken1, token2); // get token for fs2, nothing should change since already present - TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf, renewer); checkToken(creds, newerToken1, token2); // get token for fs3, should only add token for fs3 - TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf, renewer); Token<?> token3 = creds.getToken(new Text(fs3.getCanonicalServiceName())); assertTrue(token3 != null); checkToken(creds, newerToken1, token2, token3); // be paranoid, check one last time that nothing changes - TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf); - TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf); - TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer); + TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf, renewer); + TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf, renewer); checkToken(creds, newerToken1, token2, token3); } @@ -202,7 +202,7 @@ public class TestTokenCache { // wait to set, else the obtain tokens call above will fail with FNF conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, binaryTokenFile); creds.writeTokenStorageFile(new Path(binaryTokenFile), conf); - TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer); String fs_addr = fs1.getCanonicalServiceName(); Token<?> nnt = TokenCache.getDelegationToken(creds, fs_addr); assertNotNull("Token for nn is null", nnt); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org