Repository: incubator-sentry Updated Branches: refs/heads/master cccebe099 -> 2bd258105
SENTRY-510: Metrics collection for Sentry HDFS plugin (Li Li via Lenni Kuff) Change-Id: Ied49f6872265e2d6319f3bc6b911e8278a1662d0 Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/2bd25810 Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/2bd25810 Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/2bd25810 Branch: refs/heads/master Commit: 2bd258105eb5395a83b14b6830f975342b8c1333 Parents: cccebe0 Author: Lenni Kuff <[email protected]> Authored: Mon Nov 2 22:54:50 2015 -0800 Committer: Lenni Kuff <[email protected]> Committed: Mon Nov 2 22:55:01 2015 -0800 ---------------------------------------------------------------------- .../org/apache/sentry/hdfs/MetastorePlugin.java | 11 ++ .../apache/sentry/hdfs/PluginCacheSyncUtil.java | 14 ++- .../sentry/hdfs/SentryHDFSServiceProcessor.java | 23 ++++- .../sentry/hdfs/SentryHdfsMetricsUtil.java | 101 +++++++++++++++++++ .../org/apache/sentry/hdfs/SentryPlugin.java | 8 ++ .../db/service/thrift/SentryMetrics.java | 23 +++++ 6 files changed, 177 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/2bd25810/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java index 8abdc83..f88295d 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/MetastorePlugin.java @@ -31,6 +31,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import com.codahale.metrics.Timer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; @@ -282,10 +283,15 @@ public class MetastorePlugin extends SentryMetastoreListenerPlugin { } protected void notifySentryNoLock(PathsUpdate update) { + final Timer.Context timerContext = + SentryHdfsMetricsUtil.getNotifyHMSUpdateTimer.time(); try { getClient().notifyHMSUpdate(update); } catch (Exception e) { LOGGER.error("Could not send update to Sentry HDFS Service !!", e); + SentryHdfsMetricsUtil.getFailedNotifyHMSUpdateCounter.inc(); + } finally { + timerContext.stop(); } } @@ -304,7 +310,12 @@ public class MetastorePlugin extends SentryMetastoreListenerPlugin { } protected void applyLocal(PathsUpdate update) { + final Timer.Context timerContext = + SentryHdfsMetricsUtil.getApplyLocalUpdateTimer.time(); authzPaths.updatePartial(Lists.newArrayList(update), new ReentrantReadWriteLock()); + timerContext.stop(); + SentryHdfsMetricsUtil.getApplyLocalUpdateHistogram.update( + update.getPathChanges().size()); } private void notifySentryAndApplyLocal(PathsUpdate update) { http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/2bd25810/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PluginCacheSyncUtil.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PluginCacheSyncUtil.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PluginCacheSyncUtil.java index e297286..5e2f98e 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PluginCacheSyncUtil.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/PluginCacheSyncUtil.java @@ -22,6 +22,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import com.codahale.metrics.Timer; import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong; import org.apache.curator.framework.recipes.cache.PathChildrenCache; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; @@ -118,6 +119,7 @@ public class PluginCacheSyncUtil { } public void handleCacheUpdate(Update update) throws SentryPluginException { + final Timer.Context timerContext = SentryHdfsMetricsUtil.getCacheSyncToZKTimer.time(); // post message to ZK cache try { // Acquire ZK lock for update cache sync. This ensures that the counter @@ -127,10 +129,13 @@ public class PluginCacheSyncUtil { "Failed to get ZK lock for update cache syncup"); } } catch (Exception e1) { + // Stop timer in advance + timerContext.stop(); + SentryHdfsMetricsUtil.getFailedCacheSyncToZK.inc(); throw new SentryPluginException( "Error getting ZK lock for update cache syncup" + e1, e1); } - + boolean failed = false; try { try { // increment the global sequence counter if this is not a full update @@ -142,6 +147,7 @@ public class PluginCacheSyncUtil { } } } catch (Exception e1) { + failed = true; throw new SentryPluginException( "Error setting ZK counter for update cache syncup" + e1, e1); } @@ -154,6 +160,7 @@ public class PluginCacheSyncUtil { haContext.getCuratorFramework().create().creatingParentsIfNeeded() .forPath(newPath, update.serialize()); } catch (Exception e) { + failed = true; throw new SentryPluginException("error posting update to ZK ", e); } } finally { @@ -161,9 +168,14 @@ public class PluginCacheSyncUtil { try { updatorLock.release(); } catch (Exception e) { + // Stop timer in advance + timerContext.stop(); + SentryHdfsMetricsUtil.getFailedCacheSyncToZK.inc(); throw new SentryPluginException( "Error releasing ZK lock for update cache syncup" + e, e); } + timerContext.stop(); + if (failed) SentryHdfsMetricsUtil.getFailedCacheSyncToZK.inc(); } } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/2bd25810/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceProcessor.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceProcessor.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceProcessor.java index 80f3648..e4f3f58 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceProcessor.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHDFSServiceProcessor.java @@ -22,6 +22,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import com.codahale.metrics.Timer; import org.apache.sentry.hdfs.service.thrift.SentryHDFSService; import org.apache.sentry.hdfs.service.thrift.TAuthzUpdateResponse; import org.apache.sentry.hdfs.service.thrift.TPathsUpdate; @@ -45,9 +46,15 @@ public class SentryHDFSServiceProcessor implements SentryHDFSService.Iface { throw new TException( "This Sentry server is not communicating with other nodes and out of sync "); } - List<PermissionsUpdate> permUpdates = SentryPlugin.instance.getAllPermsUpdatesFrom(permSeqNum); - List<PathsUpdate> pathUpdates = SentryPlugin.instance.getAllPathsUpdatesFrom(pathSeqNum); + final Timer.Context timerContext = + SentryHdfsMetricsUtil.getAllAuthzUpdatesTimer.time(); try { + List<PermissionsUpdate> permUpdates = + SentryPlugin.instance.getAllPermsUpdatesFrom(permSeqNum); + SentryHdfsMetricsUtil.getPermUpdateHistogram.update(permUpdates.size()); + List<PathsUpdate> pathUpdates = + SentryPlugin.instance.getAllPathsUpdatesFrom(pathSeqNum); + SentryHdfsMetricsUtil.getPathUpdateHistogram.update(pathUpdates.size()); for (PathsUpdate update : pathUpdates) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("### Sending PATH preUpdate seq [" + update.getSeqNum() + "] ###"); @@ -80,6 +87,8 @@ public class SentryHDFSServiceProcessor implements SentryHDFSService.Iface { } catch (Exception e) { LOGGER.error("Error Sending updates to downstream Cache", e); throw new TException(e); + } finally { + timerContext.stop(); } } else { LOGGER.error("SentryPlugin not initialized yet !!"); @@ -90,6 +99,8 @@ public class SentryHDFSServiceProcessor implements SentryHDFSService.Iface { @Override public void handle_hms_notification(TPathsUpdate update) throws TException { + final Timer.Context timerContext = + SentryHdfsMetricsUtil.getHandleHmsNotificationTimer.time(); try { PathsUpdate hmsUpdate = new PathsUpdate(update); if (SentryPlugin.instance != null) { @@ -100,7 +111,15 @@ public class SentryHDFSServiceProcessor implements SentryHDFSService.Iface { } } catch (Exception e) { LOGGER.error("Error handling notification from HMS", e); + SentryHdfsMetricsUtil.getFailedHandleHmsNotificationCounter.inc(); throw new TException(e); + } finally { + timerContext.stop(); + SentryHdfsMetricsUtil.getHandleHmsPathChangeHistogram.update( + update.getPathChangesSize()); + if (update.isHasFullImage()) { + SentryHdfsMetricsUtil.getHandleHmsHasFullImageCounter.inc(); + } } } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/2bd25810/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java new file mode 100644 index 0000000..b67c94a --- /dev/null +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryHdfsMetricsUtil.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sentry.hdfs; + +import com.codahale.metrics.Counter; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; +import org.apache.sentry.provider.db.service.thrift.SentryMetrics; + +/** + * Util class to support metrics. + */ +public class SentryHdfsMetricsUtil { + // SentryMetrics + private static final SentryMetrics sentryMetrics = SentryMetrics.getInstance(); + + // Metrics for get_all_authz_updates_from in SentryHDFSServiceProcessor + // The time used for each get_all_authz_updates_from + public static final Timer getAllAuthzUpdatesTimer = sentryMetrics.getTimer( + MetricRegistry.name(SentryHDFSServiceProcessor.class, + "get-all-authz-updates-from")); + // The size of perm updates for each get_all_authz_updates_from + public static final Histogram getPermUpdateHistogram = sentryMetrics.getHistogram( + MetricRegistry.name(SentryHDFSServiceProcessor.class, "perm-updates-size")); + // The size of path updates for each get_all_authz_updates_from + public static final Histogram getPathUpdateHistogram = sentryMetrics.getHistogram( + MetricRegistry.name(SentryHDFSServiceProcessor.class, "paths-updates-size")); + + // Metrics for handle_hms_notification in SentryHDFSServiceProcessor + // The time used for each handle_hms_notification + public static final Timer getHandleHmsNotificationTimer = sentryMetrics.getTimer( + MetricRegistry.name(SentryHDFSServiceProcessor.class, "handle-hms-notification")); + // The number of failed handle_hms_notification + public static final Counter getFailedHandleHmsNotificationCounter = + sentryMetrics.getCounter(MetricRegistry.name(SentryHDFSServiceProcessor.class, + "handle-hms-notification", "failed-num")); + // The number of handle_hms_notification with full image update + public static final Counter getHandleHmsHasFullImageCounter = sentryMetrics.getCounter( + MetricRegistry.name(SentryHDFSServiceProcessor.class, "handle-hms-notification", + "has-full-image-num")); + // The size of path changes for each handle_hms_notification + public static final Histogram getHandleHmsPathChangeHistogram = sentryMetrics.getHistogram( + MetricRegistry.name(SentryHDFSServiceProcessor.class, "handle-hms-notification", + "path-changes-size")); + + // Metrics for retrieveFullImage in SentryPlugin.PermImageRetriever + // The time used for each retrieveFullImage + public static final Timer getRetrieveFullImageTimer = sentryMetrics.getTimer( + MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image")); + // The size of privilege changes for each retrieveFullImage + public static final Histogram getPrivilegeChangesHistogram = sentryMetrics.getHistogram( + MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image", + "privilege-changes-size")); + // The size of role changes for each retrieveFullImage call + public static final Histogram getRoleChangesHistogram = sentryMetrics.getHistogram( + MetricRegistry.name(SentryPlugin.PermImageRetriever.class, "retrieve-full-image", + "role-changes-size")); + + // Metrics for notifySentry HMS update in MetaStorePlugin + // The timer used for each notifySentry + public static final Timer getNotifyHMSUpdateTimer = sentryMetrics.getTimer( + MetricRegistry.name(MetastorePlugin.class, "notify-sentry-HMS-update")); + // The number of failed notifySentry + public static final Counter getFailedNotifyHMSUpdateCounter = sentryMetrics.getCounter( + MetricRegistry.name(MetastorePlugin.class, "notify-sentry-HMS-update", + "failed-num")); + + // Metrics for applyLocal update in MetastorePlugin + // The time used for each applyLocal + public static final Timer getApplyLocalUpdateTimer = sentryMetrics.getTimer( + MetricRegistry.name(MetastorePlugin.class, "apply-local-update")); + // The size of path changes for each applyLocal + public static final Histogram getApplyLocalUpdateHistogram = sentryMetrics.getHistogram( + MetricRegistry.name(MetastorePlugin.class, "apply-local-update", + "path-change-size")); + + // Metrics for handleCacheUpdate to ZK in PluginCacheSyncUtil + // The time used for each handleCacheUpdate + public static final Timer getCacheSyncToZKTimer = sentryMetrics.getTimer( + MetricRegistry.name(PluginCacheSyncUtil.class, "cache-sync-to-zk")); + // The number of failed handleCacheUpdate + public static final Counter getFailedCacheSyncToZK = sentryMetrics.getCounter( + MetricRegistry.name(PluginCacheSyncUtil.class, "cache-sync-to-zk", "failed-num")); +} http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/2bd25810/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java ---------------------------------------------------------------------- diff --git a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java index 93514e6..647e8fc 100644 --- a/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java +++ b/sentry-hdfs/sentry-hdfs-service/src/main/java/org/apache/sentry/hdfs/SentryPlugin.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; +import com.codahale.metrics.Timer; import org.apache.hadoop.conf.Configuration; import org.apache.sentry.hdfs.ServiceConstants.ServerConfig; import org.apache.sentry.hdfs.UpdateForwarder.ExternalImageRetriever; @@ -66,6 +67,8 @@ public class SentryPlugin implements SentryPolicyStorePlugin { @Override public PermissionsUpdate retrieveFullImage(long currSeqNum) { + final Timer.Context timerContext = + SentryHdfsMetricsUtil.getRetrieveFullImageTimer.time(); Map<String, HashMap<String, String>> privilegeImage = sentryStore.retrieveFullPrivilegeImage(); Map<String, LinkedList<String>> roleImage = sentryStore.retrieveFullRoleImage(); @@ -85,6 +88,11 @@ public class SentryPlugin implements SentryPolicyStorePlugin { } PermissionsUpdate permissionsUpdate = new PermissionsUpdate(tPermUpdate); permissionsUpdate.setSeqNum(currSeqNum); + timerContext.stop(); + SentryHdfsMetricsUtil.getPrivilegeChangesHistogram.update( + tPermUpdate.getPrivilegeChangesSize()); + SentryHdfsMetricsUtil.getRoleChangesHistogram.update( + tPermUpdate.getRoleChangesSize()); return permissionsUpdate; } http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/2bd25810/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java index 55bec0b..6eb00a1 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java @@ -18,7 +18,9 @@ package org.apache.sentry.provider.db.service.thrift; import com.codahale.metrics.ConsoleReporter; +import com.codahale.metrics.Counter; import com.codahale.metrics.Gauge; +import com.codahale.metrics.Histogram; import com.codahale.metrics.JmxReporter; import com.codahale.metrics.Metric; import com.codahale.metrics.MetricRegistry; @@ -69,6 +71,27 @@ public class SentryMetrics { public final Timer listPrivilegesByAuthorizableTimer = SentryMetricsServletContextListener.METRIC_REGISTRY.timer( MetricRegistry.name(SentryPolicyStoreProcessor.class, "list-privileges-by-authorizable")); + /** + * Return a Timer with name. + */ + public final Timer getTimer(String name) { + return SentryMetricsServletContextListener.METRIC_REGISTRY.timer(name); + } + + /** + * Return a Histogram with name. + */ + public final Histogram getHistogram(String name) { + return SentryMetricsServletContextListener.METRIC_REGISTRY.histogram(name); + } + + /** + * Return a Counter with name. + */ + public final Counter getCounter(String name) { + return SentryMetricsServletContextListener.METRIC_REGISTRY.counter(name); + } + private SentryMetrics() { registerMetricSet("gc", new GarbageCollectorMetricSet(), SentryMetricsServletContextListener.METRIC_REGISTRY); registerMetricSet("buffers", new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer()),
