Repository: ambari Updated Branches: refs/heads/trunk ef0ae8bfd -> b986a3c2d
AMBARI-20103 : Reduce AMS HBase metrics through whitelist. (avijayan) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b986a3c2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b986a3c2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b986a3c2 Branch: refs/heads/trunk Commit: b986a3c2db753a006718592c7689590bc842c6c6 Parents: ef0ae8b Author: Aravindan Vijayan <avija...@hortonworks.com> Authored: Tue Feb 21 16:17:13 2017 -0800 Committer: Aravindan Vijayan <avija...@hortonworks.com> Committed: Tue Feb 21 16:17:13 2017 -0800 ---------------------------------------------------------------------- ambari-metrics/ambari-metrics-assembly/pom.xml | 12 ++ .../src/main/assembly/collector-windows.xml | 1 + .../src/main/assembly/collector.xml | 1 + .../conf/unix/amshbase_metrics_whitelist | 162 +++++++++++++++++++ .../conf/windows/amshbase_metrics_whitelist | 162 +++++++++++++++++++ .../ambari-metrics-timelineservice/pom.xml | 5 + .../timeline/HBaseTimelineMetricStore.java | 2 +- .../timeline/TimelineMetricConfiguration.java | 39 +++++ .../metrics/timeline/TimelineMetricsFilter.java | 45 +++--- .../timeline/TimelineMetricsFilterTest.java | 59 ++++++- .../timeline/discovery/TestMetadataManager.java | 15 +- 11 files changed, 475 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-assembly/pom.xml b/ambari-metrics/ambari-metrics-assembly/pom.xml index cdcc2a7..a4b87de 100644 --- a/ambari-metrics/ambari-metrics-assembly/pom.xml +++ b/ambari-metrics/ambari-metrics-assembly/pom.xml @@ -338,6 +338,9 @@ <location>${collector.dir}/conf/unix/metrics_whitelist</location> </source> <source> + <location>${collector.dir}/conf/unix/amshbase_metrics_whitelist</location> + </source> + <source> <location>${collector.dir}/target/embedded/${hbase.folder}/conf/hbase-site.xml</location> </source> </sources> @@ -882,6 +885,15 @@ </mapper> </data> <data> + <src>${collector.dir}/conf/unix/amshbase_metrics_whitelist</src> + <type>file</type> + <mapper> + <type>perm</type> + <filemode>755</filemode> + <prefix>/etc/ambari-metrics-collector/conf</prefix> + </mapper> + </data> + <data> <src>${collector.dir}/target/embedded/${hbase.folder}/conf/hbase-site.xml</src> <type>file</type> <mapper> http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml index 1ea6c46..8b7a021 100644 --- a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml +++ b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml @@ -46,6 +46,7 @@ <include>ams-site.xml</include> <include>log4j.properties</include> <include>metrics_whitelist</include> + <include>amshbase_metrics_whitelist</include> </includes> </fileSet> <fileSet> http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector.xml b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector.xml index 0997b48..7f338be 100644 --- a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector.xml +++ b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector.xml @@ -45,6 +45,7 @@ <include>ams-site.xml</include> <include>log4j.properties</include> <include>metrics_whitelist</include> + <include>amshbase_metrics_whitelist</include> </includes> </fileSet> <fileSet> http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-timelineservice/conf/unix/amshbase_metrics_whitelist ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/amshbase_metrics_whitelist b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/amshbase_metrics_whitelist new file mode 100644 index 0000000..6cf4319 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/amshbase_metrics_whitelist @@ -0,0 +1,162 @@ +jvm.Master.JvmMetrics.ThreadsBlocked +jvm.Master.JvmMetrics.ThreadsNew +jvm.Master.JvmMetrics.ThreadsRunnable +jvm.Master.JvmMetrics.ThreadsTerminated +jvm.Master.JvmMetrics.ThreadsTimedWaiting +jvm.Master.JvmMetrics.ThreadsWaiting +jvm.RegionServer.JvmMetrics.GcCount +jvm.RegionServer.JvmMetrics.GcCountConcurrentMarkSweep +jvm.RegionServer.JvmMetrics.GcCountParNew +jvm.RegionServer.JvmMetrics.GcTimeMillis +jvm.RegionServer.JvmMetrics.GcTimeMillisConcurrentMarkSweep +jvm.RegionServer.JvmMetrics.GcTimeMillisParNew +jvm.RegionServer.JvmMetrics.MemHeapCommittedM +jvm.RegionServer.JvmMetrics.MemHeapMaxM +jvm.RegionServer.JvmMetrics.MemHeapUsedM +jvm.RegionServer.JvmMetrics.MemNonHeapCommittedM +jvm.RegionServer.JvmMetrics.MemNonHeapMaxM +jvm.RegionServer.JvmMetrics.MemNonHeapUsedM +jvm.RegionServer.JvmMetrics.ThreadsBlocked +jvm.RegionServer.JvmMetrics.ThreadsNew +jvm.RegionServer.JvmMetrics.ThreadsRunnable +jvm.RegionServer.JvmMetrics.ThreadsTerminated +jvm.RegionServer.JvmMetrics.ThreadsTimedWaiting +jvm.RegionServer.JvmMetrics.ThreadsWaiting +master.AssignmentManger.ritCount +master.AssignmentManger.ritCountOverThreshold +master.AssignmentManger.ritOldestAge +master.Master.TotalCallTime_num_ops +master.Master.exceptions +master.Server.averageLoad +master.Server.numDeadRegionServers +master.Server.numRegionServers +regionserver.RegionServer.ProcessCallTime_75th_percentile +regionserver.RegionServer.ProcessCallTime_95th_percentile +regionserver.RegionServer.ProcessCallTime_99th_percentile +regionserver.RegionServer.ProcessCallTime_max +regionserver.RegionServer.ProcessCallTime_mean +regionserver.RegionServer.ProcessCallTime_median +regionserver.RegionServer.QueueCallTime_75th_percentile +regionserver.RegionServer.QueueCallTime_95th_percentile +regionserver.RegionServer.QueueCallTime_99th_percentile +regionserver.RegionServer.QueueCallTime_max +regionserver.RegionServer.QueueCallTime_mean +regionserver.RegionServer.QueueCallTime_median +regionserver.RegionServer.TotalCallTime_num_ops +regionserver.RegionServer.authenticationFailures +regionserver.RegionServer.authenticationSuccesses +regionserver.RegionServer.authorizationFailures +regionserver.RegionServer.authorizationSuccesses +regionserver.RegionServer.exceptions +regionserver.RegionServer.numActiveHandler +regionserver.RegionServer.numCallsInGeneralQueue +regionserver.RegionServer.numCallsInPriorityQueue +regionserver.RegionServer.numCallsInReplicationQueue +regionserver.RegionServer.numOpenConnections +regionserver.RegionServer.queueSize +regionserver.RegionServer.receivedBytes +regionserver.RegionServer.sentBytes +regionserver.Server.Append_75th_percentile +regionserver.Server.Append_95th_percentile +regionserver.Server.Append_99th_percentile +regionserver.Server.Append_max +regionserver.Server.Append_mean +regionserver.Server.Append_median +regionserver.Server.Append_min +regionserver.Server.Append_num_ops +regionserver.Server.Delete_75th_percentile +regionserver.Server.Delete_95th_percentile +regionserver.Server.Delete_99th_percentile +regionserver.Server.Delete_max +regionserver.Server.Delete_mean +regionserver.Server.Delete_median +regionserver.Server.Delete_min +regionserver.Server.Delete_num_ops +regionserver.Server.Get_75th_percentile +regionserver.Server.Get_95th_percentile +regionserver.Server.Get_99th_percentile +regionserver.Server.Get_max +regionserver.Server.Get_mean +regionserver.Server.Get_median +regionserver.Server.Get_min +regionserver.Server.Get_num_ops +regionserver.Server.Increment_75th_percentile +regionserver.Server.Increment_95th_percentile +regionserver.Server.Increment_99th_percentile +regionserver.Server.Increment_max +regionserver.Server.Increment_mean +regionserver.Server.Increment_median +regionserver.Server.Increment_min +regionserver.Server.Increment_num_ops +regionserver.Server.Mutate_75th_percentile +regionserver.Server.Mutate_95th_percentile +regionserver.Server.Mutate_99th_percentile +regionserver.Server.Mutate_max +regionserver.Server.Mutate_mean +regionserver.Server.Mutate_median +regionserver.Server.Mutate_min +regionserver.Server.Mutate_num_ops +regionserver.Server.Replay_75th_percentile +regionserver.Server.Replay_95th_percentile +regionserver.Server.Replay_99th_percentile +regionserver.Server.Replay_max +regionserver.Server.Replay_mean +regionserver.Server.Replay_median +regionserver.Server.Replay_min +regionserver.Server.Replay_num_ops +regionserver.Server.ScanNext_num_ops +regionserver.Server.ScanTime_75th_percentile +regionserver.Server.ScanTime_95th_percentile +regionserver.Server.ScanTime_99th_percentile +regionserver.Server.ScanTime_max +regionserver.Server.ScanTime_mean +regionserver.Server.ScanTime_median +regionserver.Server.blockCacheCount +regionserver.Server.blockCacheCountHitPercent +regionserver.Server.blockCacheEvictionCount +regionserver.Server.blockCacheExpressHitPercent +regionserver.Server.blockCacheFreeSize +regionserver.Server.blockCacheHitCount +regionserver.Server.blockCacheMissCount +regionserver.Server.blockCacheSize +regionserver.Server.compactionQueueLength +regionserver.Server.flushQueueLength +regionserver.Server.hlogFileCount +regionserver.Server.hlogFileSize +regionserver.Server.memStoreSize +regionserver.Server.percentFilesLocal +regionserver.Server.readRequestCount +regionserver.Server.regionCount +regionserver.Server.slowDeleteCount +regionserver.Server.slowGetCount +regionserver.Server.slowPutCount +regionserver.Server.splitQueueLength +regionserver.Server.staticBloomSize +regionserver.Server.staticIndexSize +regionserver.Server.storeCount +regionserver.Server.storeFileCount +regionserver.Server.storeFileIndexSize +regionserver.Server.storeFileSize +regionserver.Server.totalRequestCount +regionserver.Server.writeRequestCount +regionserver.WAL.AppendSize_75th_percentile +regionserver.WAL.AppendSize_95th_percentile +regionserver.WAL.AppendSize_99th_percentile +regionserver.WAL.AppendSize_max +regionserver.WAL.AppendSize_mean +regionserver.WAL.AppendSize_median +regionserver.WAL.AppendTime_75th_percentile +regionserver.WAL.AppendTime_95th_percentile +regionserver.WAL.AppendTime_99th_percentile +regionserver.WAL.AppendTime_max +regionserver.WAL.AppendTime_mean +regionserver.WAL.AppendTime_median +regionserver.WAL.SyncTime_75th_percentile +regionserver.WAL.SyncTime_95th_percentile +regionserver.WAL.SyncTime_99th_percentile +regionserver.WAL.SyncTime_max +regionserver.WAL.SyncTime_mean +regionserver.WAL.SyncTime_median +regionserver.WAL.SyncTime_num_ops +regionserver.WAL.appendCount +regionserver.WAL.slowAppendCount \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-timelineservice/conf/windows/amshbase_metrics_whitelist ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/windows/amshbase_metrics_whitelist b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/amshbase_metrics_whitelist new file mode 100644 index 0000000..6cf4319 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/amshbase_metrics_whitelist @@ -0,0 +1,162 @@ +jvm.Master.JvmMetrics.ThreadsBlocked +jvm.Master.JvmMetrics.ThreadsNew +jvm.Master.JvmMetrics.ThreadsRunnable +jvm.Master.JvmMetrics.ThreadsTerminated +jvm.Master.JvmMetrics.ThreadsTimedWaiting +jvm.Master.JvmMetrics.ThreadsWaiting +jvm.RegionServer.JvmMetrics.GcCount +jvm.RegionServer.JvmMetrics.GcCountConcurrentMarkSweep +jvm.RegionServer.JvmMetrics.GcCountParNew +jvm.RegionServer.JvmMetrics.GcTimeMillis +jvm.RegionServer.JvmMetrics.GcTimeMillisConcurrentMarkSweep +jvm.RegionServer.JvmMetrics.GcTimeMillisParNew +jvm.RegionServer.JvmMetrics.MemHeapCommittedM +jvm.RegionServer.JvmMetrics.MemHeapMaxM +jvm.RegionServer.JvmMetrics.MemHeapUsedM +jvm.RegionServer.JvmMetrics.MemNonHeapCommittedM +jvm.RegionServer.JvmMetrics.MemNonHeapMaxM +jvm.RegionServer.JvmMetrics.MemNonHeapUsedM +jvm.RegionServer.JvmMetrics.ThreadsBlocked +jvm.RegionServer.JvmMetrics.ThreadsNew +jvm.RegionServer.JvmMetrics.ThreadsRunnable +jvm.RegionServer.JvmMetrics.ThreadsTerminated +jvm.RegionServer.JvmMetrics.ThreadsTimedWaiting +jvm.RegionServer.JvmMetrics.ThreadsWaiting +master.AssignmentManger.ritCount +master.AssignmentManger.ritCountOverThreshold +master.AssignmentManger.ritOldestAge +master.Master.TotalCallTime_num_ops +master.Master.exceptions +master.Server.averageLoad +master.Server.numDeadRegionServers +master.Server.numRegionServers +regionserver.RegionServer.ProcessCallTime_75th_percentile +regionserver.RegionServer.ProcessCallTime_95th_percentile +regionserver.RegionServer.ProcessCallTime_99th_percentile +regionserver.RegionServer.ProcessCallTime_max +regionserver.RegionServer.ProcessCallTime_mean +regionserver.RegionServer.ProcessCallTime_median +regionserver.RegionServer.QueueCallTime_75th_percentile +regionserver.RegionServer.QueueCallTime_95th_percentile +regionserver.RegionServer.QueueCallTime_99th_percentile +regionserver.RegionServer.QueueCallTime_max +regionserver.RegionServer.QueueCallTime_mean +regionserver.RegionServer.QueueCallTime_median +regionserver.RegionServer.TotalCallTime_num_ops +regionserver.RegionServer.authenticationFailures +regionserver.RegionServer.authenticationSuccesses +regionserver.RegionServer.authorizationFailures +regionserver.RegionServer.authorizationSuccesses +regionserver.RegionServer.exceptions +regionserver.RegionServer.numActiveHandler +regionserver.RegionServer.numCallsInGeneralQueue +regionserver.RegionServer.numCallsInPriorityQueue +regionserver.RegionServer.numCallsInReplicationQueue +regionserver.RegionServer.numOpenConnections +regionserver.RegionServer.queueSize +regionserver.RegionServer.receivedBytes +regionserver.RegionServer.sentBytes +regionserver.Server.Append_75th_percentile +regionserver.Server.Append_95th_percentile +regionserver.Server.Append_99th_percentile +regionserver.Server.Append_max +regionserver.Server.Append_mean +regionserver.Server.Append_median +regionserver.Server.Append_min +regionserver.Server.Append_num_ops +regionserver.Server.Delete_75th_percentile +regionserver.Server.Delete_95th_percentile +regionserver.Server.Delete_99th_percentile +regionserver.Server.Delete_max +regionserver.Server.Delete_mean +regionserver.Server.Delete_median +regionserver.Server.Delete_min +regionserver.Server.Delete_num_ops +regionserver.Server.Get_75th_percentile +regionserver.Server.Get_95th_percentile +regionserver.Server.Get_99th_percentile +regionserver.Server.Get_max +regionserver.Server.Get_mean +regionserver.Server.Get_median +regionserver.Server.Get_min +regionserver.Server.Get_num_ops +regionserver.Server.Increment_75th_percentile +regionserver.Server.Increment_95th_percentile +regionserver.Server.Increment_99th_percentile +regionserver.Server.Increment_max +regionserver.Server.Increment_mean +regionserver.Server.Increment_median +regionserver.Server.Increment_min +regionserver.Server.Increment_num_ops +regionserver.Server.Mutate_75th_percentile +regionserver.Server.Mutate_95th_percentile +regionserver.Server.Mutate_99th_percentile +regionserver.Server.Mutate_max +regionserver.Server.Mutate_mean +regionserver.Server.Mutate_median +regionserver.Server.Mutate_min +regionserver.Server.Mutate_num_ops +regionserver.Server.Replay_75th_percentile +regionserver.Server.Replay_95th_percentile +regionserver.Server.Replay_99th_percentile +regionserver.Server.Replay_max +regionserver.Server.Replay_mean +regionserver.Server.Replay_median +regionserver.Server.Replay_min +regionserver.Server.Replay_num_ops +regionserver.Server.ScanNext_num_ops +regionserver.Server.ScanTime_75th_percentile +regionserver.Server.ScanTime_95th_percentile +regionserver.Server.ScanTime_99th_percentile +regionserver.Server.ScanTime_max +regionserver.Server.ScanTime_mean +regionserver.Server.ScanTime_median +regionserver.Server.blockCacheCount +regionserver.Server.blockCacheCountHitPercent +regionserver.Server.blockCacheEvictionCount +regionserver.Server.blockCacheExpressHitPercent +regionserver.Server.blockCacheFreeSize +regionserver.Server.blockCacheHitCount +regionserver.Server.blockCacheMissCount +regionserver.Server.blockCacheSize +regionserver.Server.compactionQueueLength +regionserver.Server.flushQueueLength +regionserver.Server.hlogFileCount +regionserver.Server.hlogFileSize +regionserver.Server.memStoreSize +regionserver.Server.percentFilesLocal +regionserver.Server.readRequestCount +regionserver.Server.regionCount +regionserver.Server.slowDeleteCount +regionserver.Server.slowGetCount +regionserver.Server.slowPutCount +regionserver.Server.splitQueueLength +regionserver.Server.staticBloomSize +regionserver.Server.staticIndexSize +regionserver.Server.storeCount +regionserver.Server.storeFileCount +regionserver.Server.storeFileIndexSize +regionserver.Server.storeFileSize +regionserver.Server.totalRequestCount +regionserver.Server.writeRequestCount +regionserver.WAL.AppendSize_75th_percentile +regionserver.WAL.AppendSize_95th_percentile +regionserver.WAL.AppendSize_99th_percentile +regionserver.WAL.AppendSize_max +regionserver.WAL.AppendSize_mean +regionserver.WAL.AppendSize_median +regionserver.WAL.AppendTime_75th_percentile +regionserver.WAL.AppendTime_95th_percentile +regionserver.WAL.AppendTime_99th_percentile +regionserver.WAL.AppendTime_max +regionserver.WAL.AppendTime_mean +regionserver.WAL.AppendTime_median +regionserver.WAL.SyncTime_75th_percentile +regionserver.WAL.SyncTime_95th_percentile +regionserver.WAL.SyncTime_99th_percentile +regionserver.WAL.SyncTime_max +regionserver.WAL.SyncTime_mean +regionserver.WAL.SyncTime_median +regionserver.WAL.SyncTime_num_ops +regionserver.WAL.appendCount +regionserver.WAL.slowAppendCount \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-timelineservice/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/pom.xml b/ambari-metrics/ambari-metrics-timelineservice/pom.xml index 9d269e7..d848eab 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/pom.xml +++ b/ambari-metrics/ambari-metrics-timelineservice/pom.xml @@ -211,6 +211,9 @@ <location>conf/unix/metrics_whitelist</location> </source> <source> + <location>conf/unix/amshbase_metrics_whitelist</location> + </source> + <source> <location>target/embedded/${hbase.folder}/conf/hbase-site.xml</location> </source> </sources> @@ -252,7 +255,9 @@ <configuration> <excludes> <exclude>conf/unix/metrics_whitelist</exclude> + <exclude>conf/unix/amshbase_metrics_whitelist</exclude> <exclude>conf/windows/metrics_whitelist</exclude> + <exclude>conf/windows/amshbase_metrics_whitelist</exclude> </excludes> </configuration> <executions> http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java index ccce3ff..575b8ea 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java @@ -114,7 +114,7 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin } //Initialize whitelisting & blacklisting if needed - TimelineMetricsFilter.initializeMetricFilter(metricsConf); + TimelineMetricsFilter.initializeMetricFilter(configuration); defaultTopNHostsLimit = Integer.parseInt(metricsConf.get(DEFAULT_TOPN_HOSTS_LIMIT, "20")); if (Boolean.parseBoolean(metricsConf.get(USE_GROUPBY_AGGREGATOR_QUERIES, "true"))) { http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java index 92d595e..b7b9968 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java @@ -17,17 +17,25 @@ */ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.net.InetAddress; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; import java.net.UnknownHostException; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; /** * Configuration class that reads properties from ams-site.xml. All values @@ -265,6 +273,8 @@ public class TimelineMetricConfiguration { public static final String DEFAULT_INSTANCE_PORT = "12001"; + public static final String AMSHBASE_METRICS_WHITESLIST_FILE = "amshbase_metrics_whitelist"; + private Configuration hbaseConf; private Configuration metricsConf; private Configuration amsEnvConf; @@ -426,4 +436,33 @@ public class TimelineMetricConfiguration { public boolean isSecurityEnabled() { return hbaseConf.get("hbase.security.authentication", "").equals("kerberos"); } + + public Set<String> getAmshbaseWhitelist() { + + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + if (classLoader == null) { + classLoader = getClass().getClassLoader(); + } + + BufferedReader br = null; + String strLine; + Set<String> whitelist = new HashSet<>(); + + try(InputStream inputStream = classLoader.getResourceAsStream(AMSHBASE_METRICS_WHITESLIST_FILE)) { + br = new BufferedReader(new InputStreamReader(inputStream)); + + while ((strLine = br.readLine()) != null) { + strLine = strLine.trim(); + if (StringUtils.isEmpty(strLine)) { + continue; + } + whitelist.add(strLine); + } + } catch (IOException ioEx) { + LOG.error("Unable to parse ams-hbase metric whitelist file", ioEx); + return Collections.EMPTY_SET; + } + + return whitelist; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java index d05353d..0fe979e 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -27,6 +28,7 @@ import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; import java.io.BufferedReader; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.util.Arrays; import java.util.HashSet; @@ -41,35 +43,44 @@ public class TimelineMetricsFilter { private static Set<String> whitelistedMetrics; private static Set<Pattern> whitelistedMetricPatterns; - private static boolean metricWhitelistingEnabled; - private static boolean appBlacklistingEnabled; private static Set<String> blacklistedApps; private static String patternPrefix = "._p_"; + private static Set<String> amshbaseWhitelist; private static final Log LOG = LogFactory.getLog(TimelineMetricsFilter.class); - public static void initializeMetricFilter(Configuration metricsConf) { + public static void initializeMetricFilter(TimelineMetricConfiguration configuration) { + + Configuration metricsConf = null; + try { + metricsConf = configuration.getMetricsConf(); + } catch (Exception e) { + LOG.error("Error fetching metrics configuration for getting whitelisting information"); + return; + } whitelistedMetrics = new HashSet<String>(); whitelistedMetricPatterns = new HashSet<Pattern>(); blacklistedApps = new HashSet<>(); - metricWhitelistingEnabled = false; - appBlacklistingEnabled = false; + amshbaseWhitelist = new HashSet<>(); String whitelistFile = metricsConf.get(TIMELINE_METRICS_WHITELIST_FILE, ""); if (!StringUtils.isEmpty(whitelistFile)) { - metricWhitelistingEnabled = true; readMetricWhitelistFromFile(whitelistFile); } String appsBlacklist = metricsConf.get(TIMELINE_METRICS_APPS_BLACKLIST, ""); if (!StringUtils.isEmpty(appsBlacklist)) { - appBlacklistingEnabled = true; for (String app : appsBlacklist.split(",")) { blacklistedApps.add(app); } LOG.info("Blacklisted apps : " + blacklistedApps.toString()); } + + amshbaseWhitelist = configuration.getAmshbaseWhitelist(); + if (CollectionUtils.isNotEmpty(amshbaseWhitelist)) { + LOG.info("Whitelisting " + amshbaseWhitelist.size() + " ams-hbase metrics"); + } } private static void readMetricWhitelistFromFile(String whitelistFile) { @@ -108,16 +119,17 @@ public class TimelineMetricsFilter { public static boolean acceptMetric(TimelineMetric metric) { - if (!isEnabled()) { - return true; - } - // App takes precedence. - if (blacklistedApps.contains(metric.getAppId())) { + if (CollectionUtils.isNotEmpty(blacklistedApps) && blacklistedApps.contains(metric.getAppId())) { return false; } - if (!metricWhitelistingEnabled) { + //Special Case appId = ams-hbase whitelisting. + if ("ams-hbase".equals(metric.getAppId()) && CollectionUtils.isNotEmpty(amshbaseWhitelist)) { + return amshbaseWhitelist.contains(metric.getMetricName()); + } + + if (CollectionUtils.isEmpty(whitelistedMetrics) && CollectionUtils.isEmpty(whitelistedMetricPatterns)) { return true; } @@ -139,15 +151,8 @@ public class TimelineMetricsFilter { public static void addToWhitelist(String metricName) { if (StringUtils.isNotEmpty(metricName)) { - if (!metricWhitelistingEnabled) { - metricWhitelistingEnabled = true; - } - whitelistedMetrics.add(metricName); } } - public static boolean isEnabled() { - return appBlacklistingEnabled || metricWhitelistingEnabled; - } } http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java index 049d473..73c66fe 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java @@ -17,20 +17,31 @@ */ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; + import junit.framework.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; +import org.easymock.EasyMock; import org.junit.Test; + import java.net.URL; +import java.util.HashSet; +import java.util.Set; public class TimelineMetricsFilterTest { @Test - public void testAppBlacklisting() { + public void testAppBlacklisting() throws Exception{ Configuration metricsConf = new Configuration(); metricsConf.set("timeline.metrics.apps.blacklist", "hbase,datanode,nimbus"); - TimelineMetricsFilter.initializeMetricFilter(metricsConf); + TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class); + expect(configuration.getMetricsConf()).andReturn(metricsConf).once(); + replay(configuration); + + TimelineMetricsFilter.initializeMetricFilter(configuration); TimelineMetric timelineMetric = new TimelineMetric(); @@ -48,10 +59,14 @@ public class TimelineMetricsFilterTest { public void testMetricWhitelisting() throws Exception { Configuration metricsConf = new Configuration(); + TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class); + expect(configuration.getMetricsConf()).andReturn(metricsConf).once(); + replay(configuration); + URL fileUrl = ClassLoader.getSystemResource("test_data/metric_whitelist.dat"); metricsConf.set("timeline.metrics.whitelist.file", fileUrl.getPath()); - TimelineMetricsFilter.initializeMetricFilter(metricsConf); + TimelineMetricsFilter.initializeMetricFilter(configuration); TimelineMetric timelineMetric = new TimelineMetric(); @@ -73,11 +88,14 @@ public class TimelineMetricsFilterTest { Configuration metricsConf = new Configuration(); metricsConf.set("timeline.metrics.apps.blacklist", "hbase,datanode,nimbus"); + TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class); + expect(configuration.getMetricsConf()).andReturn(metricsConf).once(); + replay(configuration); URL fileUrl = ClassLoader.getSystemResource("test_data/metric_whitelist.dat"); metricsConf.set("timeline.metrics.whitelist.file", fileUrl.getPath()); - TimelineMetricsFilter.initializeMetricFilter(metricsConf); + TimelineMetricsFilter.initializeMetricFilter(configuration); TimelineMetric timelineMetric = new TimelineMetric(); @@ -96,4 +114,37 @@ public class TimelineMetricsFilterTest { Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric)); } + @Test + public void testAmshbaseWhitelisting() throws Exception { + + TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class); + + Configuration metricsConf = new Configuration(); + expect(configuration.getMetricsConf()).andReturn(metricsConf).once(); + + Set<String> whitelist = new HashSet(); + whitelist.add("regionserver.Server.Delete_99th_percentile"); + whitelist.add("regionserver.Server.Delete_max"); + whitelist.add("regionserver.Server.Delete_mean"); + expect(configuration.getAmshbaseWhitelist()).andReturn(whitelist).once(); + + replay(configuration); + + TimelineMetricsFilter.initializeMetricFilter(configuration); + + TimelineMetric timelineMetric = new TimelineMetric(); + + timelineMetric.setMetricName("regionserver.Server.Delete_max"); + timelineMetric.setAppId("ams-hbase"); + Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric)); + + timelineMetric.setMetricName("regionserver.Server.Delete_min3333"); + timelineMetric.setAppId("ams-hbase"); + Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric)); + + timelineMetric.setMetricName("jvm.JvmMetrics.MemHeapUsedM"); + timelineMetric.setAppId("hbase"); + Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric)); + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/b986a3c2/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java index c4cf51d..b243e0b 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java @@ -17,18 +17,23 @@ */ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; + import junit.framework.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.AbstractMiniHBaseClusterTest; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricsFilter; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils; +import org.easymock.EasyMock; import org.junit.Before; import org.junit.Test; import java.io.IOException; +import java.net.URISyntaxException; import java.sql.SQLException; import java.util.Map; import java.util.Set; @@ -38,7 +43,7 @@ public class TestMetadataManager extends AbstractMiniHBaseClusterTest { TimelineMetricMetadataManager metadataManager; @Before - public void insertDummyRecords() throws IOException, SQLException { + public void insertDummyRecords() throws IOException, SQLException, URISyntaxException { // Initialize new manager metadataManager = new TimelineMetricMetadataManager(hdb, new Configuration()); final long now = System.currentTimeMillis(); @@ -87,7 +92,11 @@ public class TestMetadataManager extends AbstractMiniHBaseClusterTest { }}); timelineMetrics.getMetrics().add(metric3); - TimelineMetricsFilter.initializeMetricFilter(new Configuration()); + Configuration metricsConf = new Configuration(); + TimelineMetricConfiguration configuration = EasyMock.createNiceMock(TimelineMetricConfiguration.class); + expect(configuration.getMetricsConf()).andReturn(metricsConf).once(); + replay(configuration); + TimelineMetricsFilter.initializeMetricFilter(configuration); TimelineMetricsFilter.addToWhitelist("dummy_metric1"); TimelineMetricsFilter.addToWhitelist("dummy_metric2");