This is an automated email from the ASF dual-hosted git repository. sunchao pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push: new 84b74b3 HDFS-15447 RBF: Add top real owners metrics for delegation tokens (#2110) 84b74b3 is described below commit 84b74b335c0251afa672643352c6b7ecf003e0fb Author: lfengnan <lfeng...@uber.com> AuthorDate: Sun Jul 12 00:10:12 2020 -0700 HDFS-15447 RBF: Add top real owners metrics for delegation tokens (#2110) --- .../AbstractDelegationTokenSecretManager.java | 101 ++++++++++++++++++++- .../delegation/ZKDelegationTokenSecretManager.java | 3 + .../hdfs/server/federation/metrics/RBFMetrics.java | 18 +++- .../server/federation/metrics/RouterMBean.java | 7 ++ .../server/federation/router/RBFConfigKeys.java | 4 + .../token/ZKDelegationTokenSecretManagerImpl.java | 3 +- .../src/main/resources/hdfs-rbf-default.xml | 10 ++ .../security/TestRouterSecurityManager.java | 68 ++++++++++++++ 8 files changed, 209 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java index 3a22cee..eb65799 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java @@ -22,9 +22,12 @@ import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.IOException; import java.security.MessageDigest; +import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -34,6 +37,8 @@ import javax.crypto.SecretKey; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.Text; +import org.apache.hadoop.metrics2.util.Metrics2Util.NameValuePair; +import org.apache.hadoop.metrics2.util.Metrics2Util.TopN; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.HadoopKerberosName; import org.apache.hadoop.security.token.SecretManager; @@ -64,7 +69,13 @@ extends AbstractDelegationTokenIdentifier> */ protected final Map<TokenIdent, DelegationTokenInformation> currentTokens = new ConcurrentHashMap<>(); - + + /** + * Map of token real owners to its token count. This is used to generate + * metrics of top users by owned tokens. + */ + protected final Map<String, Long> tokenOwnerStats = new ConcurrentHashMap<>(); + /** * Sequence number to create DelegationTokenIdentifier. * Protected by this object lock. @@ -292,6 +303,7 @@ extends AbstractDelegationTokenIdentifier> protected void storeToken(TokenIdent ident, DelegationTokenInformation tokenInfo) throws IOException { currentTokens.put(ident, tokenInfo); + addTokenForOwnerStats(ident); storeNewToken(ident, tokenInfo.getRenewDate()); } @@ -339,6 +351,7 @@ extends AbstractDelegationTokenIdentifier> if (getTokenInfo(identifier) == null) { currentTokens.put(identifier, new DelegationTokenInformation(renewDate, password, getTrackingIdIfEnabled(identifier))); + addTokenForOwnerStats(identifier); } else { throw new IOException("Same delegation token being added twice: " + formatTokenId(identifier)); @@ -578,6 +591,7 @@ extends AbstractDelegationTokenIdentifier> if (info == null) { throw new InvalidToken("Token not found " + formatTokenId(id)); } + removeTokenForOwnerStats(id); removeStoredToken(id); return id; } @@ -634,6 +648,7 @@ extends AbstractDelegationTokenIdentifier> long renewDate = entry.getValue().getRenewDate(); if (renewDate < now) { expiredTokens.add(entry.getKey()); + removeTokenForOwnerStats(entry.getKey()); i.remove(); } } @@ -726,4 +741,88 @@ extends AbstractDelegationTokenIdentifier> return token.decodeIdentifier(); } + /** + * Return top token real owners list as well as the tokens count. + * + * @param n top number of users + * @return map of owners to counts + */ + public List<NameValuePair> getTopTokenRealOwners(int n) { + n = Math.min(n, tokenOwnerStats.size()); + if (n == 0) { + return new ArrayList<>(); + } + + TopN topN = new TopN(n); + for (Map.Entry<String, Long> entry : tokenOwnerStats.entrySet()) { + topN.offer(new NameValuePair( + entry.getKey(), entry.getValue())); + } + + List<NameValuePair> list = new ArrayList<>(); + while (!topN.isEmpty()) { + list.add(topN.poll()); + } + Collections.reverse(list); + return list; + } + + /** + * Return the real owner for a token. If this is a token from a proxy user, + * the real/effective user will be returned. + * + * @param id + * @return real owner + */ + private String getTokenRealOwner(TokenIdent id) { + String realUser; + if (id.getRealUser() != null && !id.getRealUser().toString().isEmpty()) { + realUser = id.getRealUser().toString(); + } else { + // if there is no real user -> this is a non proxy user + // the user itself is the real owner + realUser = id.getUser().getUserName(); + } + return realUser; + } + + /** + * Add token stats to the owner to token count mapping. + * + * @param id + */ + private void addTokenForOwnerStats(TokenIdent id) { + String realOwner = getTokenRealOwner(id); + tokenOwnerStats.put(realOwner, + tokenOwnerStats.getOrDefault(realOwner, 0L)+1); + } + + /** + * Remove token stats to the owner to token count mapping. + * + * @param id + */ + private void removeTokenForOwnerStats(TokenIdent id) { + String realOwner = getTokenRealOwner(id); + if (tokenOwnerStats.containsKey(realOwner)) { + // unlikely to be less than 1 but in case + if (tokenOwnerStats.get(realOwner) <= 1) { + tokenOwnerStats.remove(realOwner); + } else { + tokenOwnerStats.put(realOwner, tokenOwnerStats.get(realOwner)-1); + } + } + } + + /** + * This method syncs token information from currentTokens to tokenOwnerStats. + * It is used when the currentTokens is initialized or refreshed. This is + * called from a single thread thus no synchronization is needed. + */ + protected void syncTokenOwnerStats() { + tokenOwnerStats.clear(); + for (TokenIdent id : currentTokens.keySet()) { + addTokenForOwnerStats(id); + } + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java index f50035d..276573b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java @@ -457,6 +457,9 @@ public abstract class ZKDelegationTokenSecretManager<TokenIdent extends Abstract ++count; } } + if (isTokenCache) { + syncTokenOwnerStats(); + } if (count > 0) { LOG.warn("Ignored {} nodes while loading {} cache.", count, cacheName); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java index 1ff3d7b..e13815b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RBFMetrics.java @@ -124,7 +124,8 @@ public class RBFMetrics implements RouterMBean, FederationMBean { private MountTableStore mountTableStore; /** Router state store. */ private RouterStore routerStore; - + /** The number of top token owners reported in metrics. */ + private int topTokenRealOwners; public RBFMetrics(Router router) throws IOException { this.router = router; @@ -166,7 +167,9 @@ public class RBFMetrics implements RouterMBean, FederationMBean { Configuration conf = router.getConfig(); this.timeOut = conf.getTimeDuration(RBFConfigKeys.DN_REPORT_TIME_OUT, RBFConfigKeys.DN_REPORT_TIME_OUT_MS_DEFAULT, TimeUnit.MILLISECONDS); - + this.topTokenRealOwners = conf.getInt( + RBFConfigKeys.DFS_ROUTER_METRICS_TOP_NUM_TOKEN_OWNERS_KEY, + RBFConfigKeys.DFS_ROUTER_METRICS_TOP_NUM_TOKEN_OWNERS_KEY_DEFAULT); } /** @@ -650,6 +653,17 @@ public class RBFMetrics implements RouterMBean, FederationMBean { } @Override + public String getTopTokenRealOwners() { + RouterSecurityManager mgr = + this.router.getRpcServer().getRouterSecurityManager(); + if (mgr != null && mgr.getSecretManager() != null) { + return JSON.toString(mgr.getSecretManager() + .getTopTokenRealOwners(this.topTokenRealOwners)); + } + return ""; + } + + @Override public boolean isSecurityEnabled() { return UserGroupInformation.isSecurityEnabled(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RouterMBean.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RouterMBean.java index daec154..087c5b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RouterMBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/RouterMBean.java @@ -101,4 +101,11 @@ public interface RouterMBean { * @return true, if security is enabled. */ boolean isSecurityEnabled(); + + /** + * Get the top delegation token owners(realUser). + * + * @return Json string of owners to token counts + */ + String getTopTokenRealOwners(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java index adc9720..7b06ca4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java @@ -79,6 +79,10 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic { public static final Class<? extends RouterRpcMonitor> DFS_ROUTER_METRICS_CLASS_DEFAULT = FederationRPCPerformanceMonitor.class; + public static final String DFS_ROUTER_METRICS_TOP_NUM_TOKEN_OWNERS_KEY = + FEDERATION_ROUTER_PREFIX + "top.num.token.realowners"; + public static final int + DFS_ROUTER_METRICS_TOP_NUM_TOKEN_OWNERS_KEY_DEFAULT = 10; // HDFS Router heartbeat public static final String DFS_ROUTER_HEARTBEAT_ENABLE = diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/token/ZKDelegationTokenSecretManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/token/ZKDelegationTokenSecretManagerImpl.java index 2d55026..a83be71 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/token/ZKDelegationTokenSecretManagerImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/security/token/ZKDelegationTokenSecretManagerImpl.java @@ -30,8 +30,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; import java.io.IOException; import java.util.HashSet; import java.util.List; @@ -197,6 +195,7 @@ public class ZKDelegationTokenSecretManagerImpl extends } } } + syncTokenOwnerStats(); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml index deab549..4bd2ac3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml @@ -657,4 +657,14 @@ </description> </property> + <property> + <name>dfs.federation.router.top.num.token.realowners</name> + <value>10</value> + <description> + The number of top real owners by tokens count to report in the JMX metrics. + Real owners are the effective users whose cretential are used to generate + the tokens. + </description> + </property> + </configuration> diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/security/TestRouterSecurityManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/security/TestRouterSecurityManager.java index b88fd14..d62837c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/security/TestRouterSecurityManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/security/TestRouterSecurityManager.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.server.federation.router.security.RouterSecurityMa import org.apache.hadoop.hdfs.server.federation.router.Router; import org.apache.hadoop.hdfs.server.federation.router.security.token.ZKDelegationTokenSecretManagerImpl; import org.apache.hadoop.io.Text; +import org.apache.hadoop.metrics2.util.Metrics2Util.NameValuePair; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.SecretManager; @@ -50,6 +51,7 @@ import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ import org.hamcrest.core.StringContains; import java.io.IOException; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -125,6 +127,72 @@ public class TestRouterSecurityManager { } @Test + public void testDelgationTokenTopOwners() throws Exception { + UserGroupInformation.reset(); + List<NameValuePair> topOwners; + + UserGroupInformation user = UserGroupInformation + .createUserForTesting("abc", new String[]{"router_group"}); + UserGroupInformation.setLoginUser(user); + Token dt = securityManager.getDelegationToken(new Text("abc")); + topOwners = securityManager.getSecretManager().getTopTokenRealOwners(2); + assertEquals(1, topOwners.size()); + assertEquals("abc", topOwners.get(0).getName()); + assertEquals(1, topOwners.get(0).getValue()); + + securityManager.renewDelegationToken(dt); + topOwners = securityManager.getSecretManager().getTopTokenRealOwners(2); + assertEquals(1, topOwners.size()); + assertEquals("abc", topOwners.get(0).getName()); + assertEquals(1, topOwners.get(0).getValue()); + + securityManager.cancelDelegationToken(dt); + topOwners = securityManager.getSecretManager().getTopTokenRealOwners(2); + assertEquals(0, topOwners.size()); + + + // Use proxy user - the code should use the proxy user as the real owner + UserGroupInformation routerUser = + UserGroupInformation.createRemoteUser("router"); + UserGroupInformation proxyUser = UserGroupInformation + .createProxyUserForTesting("abc", + routerUser, + new String[]{"router_group"}); + UserGroupInformation.setLoginUser(proxyUser); + + Token proxyDT = securityManager.getDelegationToken(new Text("router")); + topOwners = securityManager.getSecretManager().getTopTokenRealOwners(2); + assertEquals(1, topOwners.size()); + assertEquals("router", topOwners.get(0).getName()); + assertEquals(1, topOwners.get(0).getValue()); + + // router to renew tokens + UserGroupInformation.setLoginUser(routerUser); + securityManager.renewDelegationToken(proxyDT); + topOwners = securityManager.getSecretManager().getTopTokenRealOwners(2); + assertEquals(1, topOwners.size()); + assertEquals("router", topOwners.get(0).getName()); + assertEquals(1, topOwners.get(0).getValue()); + + securityManager.cancelDelegationToken(proxyDT); + topOwners = securityManager.getSecretManager().getTopTokenRealOwners(2); + assertEquals(0, topOwners.size()); + + + // check rank by more users + securityManager.getDelegationToken(new Text("router")); + securityManager.getDelegationToken(new Text("router")); + UserGroupInformation.setLoginUser(user); + securityManager.getDelegationToken(new Text("router")); + topOwners = securityManager.getSecretManager().getTopTokenRealOwners(2); + assertEquals(2, topOwners.size()); + assertEquals("router", topOwners.get(0).getName()); + assertEquals(2, topOwners.get(0).getValue()); + assertEquals("abc", topOwners.get(1).getName()); + assertEquals(1, topOwners.get(1).getValue()); + } + + @Test public void testVerifyToken() throws IOException { UserGroupInformation.reset(); UserGroupInformation.setLoginUser(UserGroupInformation --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org