[ https://issues.apache.org/jira/browse/HADOOP-18167?focusedWorklogId=757235&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-757235 ]
ASF GitHub Bot logged work on HADOOP-18167: ------------------------------------------- Author: ASF GitHub Bot Created on: 14/Apr/22 22:12 Start Date: 14/Apr/22 22:12 Worklog Time Spent: 10m Work Description: goiri commented on code in PR #4092: URL: https://github.com/apache/hadoop/pull/4092#discussion_r850843618 ########## hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java: ########## @@ -825,4 +859,68 @@ protected void syncTokenOwnerStats() { addTokenForOwnerStats(id); } } + + /** + * DelegationTokenSecretManagerMetrics tracks token management operations + * and publishes them through the metrics interfaces. + */ + @Metrics(about="Delegation token secret manager metrics", context="token") + static class DelegationTokenSecretManagerMetrics implements IOStatisticsSource { + private static final Logger LOG = LoggerFactory.getLogger(DelegationTokenSecretManagerMetrics.class); + + final static String STORE_TOKEN_STAT = "storeToken"; + final static String UPDATE_TOKEN_STAT = "updateToken"; + final static String REMOVE_TOKEN_STAT = "removeToken"; + final static String TOKEN_FAILURE_STAT = "tokenFailure"; + + final MetricsRegistry registry; + final IOStatisticsStore ioStatistics; + + @Metric("Rate of storage of delegation tokens and latency (milliseconds)") + MutableRate storeToken; + @Metric("Rate of update of delegation tokens and latency (milliseconds)") + MutableRate updateToken; + @Metric("Rate of removal of delegation tokens and latency (milliseconds)") + MutableRate removeToken; + @Metric("Counter of delegation tokens operation failures") + MutableCounterLong tokenFailure; + + static DelegationTokenSecretManagerMetrics create() { + return DefaultMetricsSystem.instance().register(new DelegationTokenSecretManagerMetrics()); + } + + public DelegationTokenSecretManagerMetrics() { + ioStatistics = iostatisticsStore() + .withDurationTracking(STORE_TOKEN_STAT, UPDATE_TOKEN_STAT, REMOVE_TOKEN_STAT) + .withCounters(TOKEN_FAILURE_STAT) + .build(); + registry = new MetricsRegistry("DelegationTokenSecretManagerMetrics"); + LOG.debug("Initialized {}", registry); + } + + public void addStoreToken(long value) { Review Comment: addTimeStoreToken() ########## hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java: ########## @@ -825,4 +859,68 @@ protected void syncTokenOwnerStats() { addTokenForOwnerStats(id); } } + + /** + * DelegationTokenSecretManagerMetrics tracks token management operations + * and publishes them through the metrics interfaces. + */ + @Metrics(about="Delegation token secret manager metrics", context="token") + static class DelegationTokenSecretManagerMetrics implements IOStatisticsSource { + private static final Logger LOG = LoggerFactory.getLogger(DelegationTokenSecretManagerMetrics.class); + + final static String STORE_TOKEN_STAT = "storeToken"; + final static String UPDATE_TOKEN_STAT = "updateToken"; + final static String REMOVE_TOKEN_STAT = "removeToken"; + final static String TOKEN_FAILURE_STAT = "tokenFailure"; + + final MetricsRegistry registry; + final IOStatisticsStore ioStatistics; + + @Metric("Rate of storage of delegation tokens and latency (milliseconds)") + MutableRate storeToken; + @Metric("Rate of update of delegation tokens and latency (milliseconds)") + MutableRate updateToken; + @Metric("Rate of removal of delegation tokens and latency (milliseconds)") + MutableRate removeToken; + @Metric("Counter of delegation tokens operation failures") + MutableCounterLong tokenFailure; + + static DelegationTokenSecretManagerMetrics create() { + return DefaultMetricsSystem.instance().register(new DelegationTokenSecretManagerMetrics()); + } + + public DelegationTokenSecretManagerMetrics() { + ioStatistics = iostatisticsStore() + .withDurationTracking(STORE_TOKEN_STAT, UPDATE_TOKEN_STAT, REMOVE_TOKEN_STAT) + .withCounters(TOKEN_FAILURE_STAT) + .build(); + registry = new MetricsRegistry("DelegationTokenSecretManagerMetrics"); + LOG.debug("Initialized {}", registry); + } + + public void addStoreToken(long value) { + storeToken.add(value); + ioStatistics.addTimedOperation(STORE_TOKEN_STAT, value); + } + + public void addUpdateToken(long value) { Review Comment: addTimeUpdateToken() and same for the rest. ########## hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java: ########## @@ -825,4 +859,68 @@ protected void syncTokenOwnerStats() { addTokenForOwnerStats(id); } } + + /** + * DelegationTokenSecretManagerMetrics tracks token management operations + * and publishes them through the metrics interfaces. + */ + @Metrics(about="Delegation token secret manager metrics", context="token") + static class DelegationTokenSecretManagerMetrics implements IOStatisticsSource { + private static final Logger LOG = LoggerFactory.getLogger(DelegationTokenSecretManagerMetrics.class); + + final static String STORE_TOKEN_STAT = "storeToken"; + final static String UPDATE_TOKEN_STAT = "updateToken"; + final static String REMOVE_TOKEN_STAT = "removeToken"; + final static String TOKEN_FAILURE_STAT = "tokenFailure"; + + final MetricsRegistry registry; + final IOStatisticsStore ioStatistics; + + @Metric("Rate of storage of delegation tokens and latency (milliseconds)") + MutableRate storeToken; + @Metric("Rate of update of delegation tokens and latency (milliseconds)") + MutableRate updateToken; + @Metric("Rate of removal of delegation tokens and latency (milliseconds)") + MutableRate removeToken; + @Metric("Counter of delegation tokens operation failures") + MutableCounterLong tokenFailure; + + static DelegationTokenSecretManagerMetrics create() { + return DefaultMetricsSystem.instance().register(new DelegationTokenSecretManagerMetrics()); + } + + public DelegationTokenSecretManagerMetrics() { + ioStatistics = iostatisticsStore() + .withDurationTracking(STORE_TOKEN_STAT, UPDATE_TOKEN_STAT, REMOVE_TOKEN_STAT) + .withCounters(TOKEN_FAILURE_STAT) + .build(); + registry = new MetricsRegistry("DelegationTokenSecretManagerMetrics"); + LOG.debug("Initialized {}", registry); + } + + public void addStoreToken(long value) { Review Comment: Make the argument final too. ########## hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java: ########## @@ -579,4 +615,65 @@ public void testEmptyToken() throws IOException { assertEquals(token1, token2); assertEquals(token1.encodeToUrlString(), token2.encodeToUrlString()); } + + @Test + public void testDelegationTokenSecretManagerMetrics() throws Exception { + TestDelegationTokenSecretManager dtSecretManager = + new TestDelegationTokenSecretManager(24*60*60*1000, + 10*1000,1*1000,3600000); + try { + dtSecretManager.startThreads(); + + Assert.assertEquals(0, dtSecretManager.metrics.storeToken.lastStat().numSamples()); + final Token<TestDelegationTokenIdentifier> token = + generateDelegationToken(dtSecretManager, "SomeUser", "JobTracker"); + Assert.assertEquals(1, dtSecretManager.metrics.storeToken.lastStat().numSamples()); + + Assert.assertEquals(0, dtSecretManager.metrics.updateToken.lastStat().numSamples()); + dtSecretManager.renewToken(token, "JobTracker"); + Assert.assertEquals(1, dtSecretManager.metrics.updateToken.lastStat().numSamples()); + + Assert.assertEquals(0, dtSecretManager.metrics.removeToken.lastStat().numSamples()); + dtSecretManager.cancelToken(token, "JobTracker"); + Assert.assertEquals(1, dtSecretManager.metrics.removeToken.lastStat().numSamples()); + } finally { + dtSecretManager.stopThreads(); + } + } + + @Test + public void testDelegationTokenSecretManagerMetricsFailures() throws Exception { + TestFailureDelegationTokenSecretManager dtSecretManager = new TestFailureDelegationTokenSecretManager(); + + try { + dtSecretManager.startThreads(); + + final Token<TestDelegationTokenIdentifier> token = + generateDelegationToken(dtSecretManager, "SomeUser", "JobTracker"); + + dtSecretManager.setThrowError(true); + + Assert.assertEquals(0, dtSecretManager.metrics.tokenFailure.value()); + generateDelegationToken(dtSecretManager, "SomeUser", "JobTracker"); + Assert.assertEquals(1, dtSecretManager.metrics.tokenFailure.value()); + + try { + dtSecretManager.renewToken(token, "JobTracker"); + Assert.fail("Expected exception"); + } catch (Exception ex) { + // Expected exception + } + Assert.assertEquals(2, dtSecretManager.metrics.tokenFailure.value()); + + try { + dtSecretManager.cancelToken(token, "JobTracker"); + Assert.fail("Expected exception"); Review Comment: use LamdaTestUtils#intercept ########## hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java: ########## @@ -50,6 +59,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.*; Review Comment: Better to expand this or to use the full name. ########## hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java: ########## @@ -579,4 +615,65 @@ public void testEmptyToken() throws IOException { assertEquals(token1, token2); assertEquals(token1.encodeToUrlString(), token2.encodeToUrlString()); } + + @Test + public void testDelegationTokenSecretManagerMetrics() throws Exception { + TestDelegationTokenSecretManager dtSecretManager = + new TestDelegationTokenSecretManager(24*60*60*1000, + 10*1000,1*1000,3600000); Review Comment: spaces after the commas ########## hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java: ########## @@ -825,4 +859,68 @@ protected void syncTokenOwnerStats() { addTokenForOwnerStats(id); } } + + /** + * DelegationTokenSecretManagerMetrics tracks token management operations + * and publishes them through the metrics interfaces. + */ + @Metrics(about="Delegation token secret manager metrics", context="token") + static class DelegationTokenSecretManagerMetrics implements IOStatisticsSource { + private static final Logger LOG = LoggerFactory.getLogger(DelegationTokenSecretManagerMetrics.class); + + final static String STORE_TOKEN_STAT = "storeToken"; + final static String UPDATE_TOKEN_STAT = "updateToken"; + final static String REMOVE_TOKEN_STAT = "removeToken"; + final static String TOKEN_FAILURE_STAT = "tokenFailure"; + + final MetricsRegistry registry; + final IOStatisticsStore ioStatistics; + + @Metric("Rate of storage of delegation tokens and latency (milliseconds)") + MutableRate storeToken; + @Metric("Rate of update of delegation tokens and latency (milliseconds)") + MutableRate updateToken; + @Metric("Rate of removal of delegation tokens and latency (milliseconds)") + MutableRate removeToken; + @Metric("Counter of delegation tokens operation failures") + MutableCounterLong tokenFailure; + + static DelegationTokenSecretManagerMetrics create() { + return DefaultMetricsSystem.instance().register(new DelegationTokenSecretManagerMetrics()); + } + + public DelegationTokenSecretManagerMetrics() { + ioStatistics = iostatisticsStore() + .withDurationTracking(STORE_TOKEN_STAT, UPDATE_TOKEN_STAT, REMOVE_TOKEN_STAT) + .withCounters(TOKEN_FAILURE_STAT) + .build(); + registry = new MetricsRegistry("DelegationTokenSecretManagerMetrics"); + LOG.debug("Initialized {}", registry); + } + + public void addStoreToken(long value) { Review Comment: And probably time and the unit instead of plain "value" ########## hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java: ########## @@ -579,4 +615,65 @@ public void testEmptyToken() throws IOException { assertEquals(token1, token2); assertEquals(token1.encodeToUrlString(), token2.encodeToUrlString()); } + + @Test + public void testDelegationTokenSecretManagerMetrics() throws Exception { + TestDelegationTokenSecretManager dtSecretManager = + new TestDelegationTokenSecretManager(24*60*60*1000, + 10*1000,1*1000,3600000); Review Comment: Extend the 3600000 too. Issue Time Tracking ------------------- Worklog Id: (was: 757235) Time Spent: 2.5h (was: 2h 20m) > Add metrics to track delegation token secret manager operations > --------------------------------------------------------------- > > Key: HADOOP-18167 > URL: https://issues.apache.org/jira/browse/HADOOP-18167 > Project: Hadoop Common > Issue Type: Improvement > Reporter: Hector Sandoval Chaverri > Priority: Major > Labels: pull-request-available > Attachments: HADOOP-18167-branch-2.10-2.patch, > HADOOP-18167-branch-2.10.patch > > Time Spent: 2.5h > Remaining Estimate: 0h > > New metrics to track operations that store, update and remove delegation > tokens in implementations of AbstractDelegationTokenSecretManager. This will > help evaluate the impact of using different secret managers and add > optimizations. -- This message was sent by Atlassian Jira (v8.20.1#820001) --------------------------------------------------------------------- To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org