AMBARI-19141 : Refinements for AMS whitelisting. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9e524d7e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9e524d7e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9e524d7e Branch: refs/heads/trunk Commit: 9e524d7e66fe6031cd20c54946ccfd91255e5961 Parents: 6d196db Author: Aravindan Vijayan <avija...@hortonworks.com> Authored: Thu Dec 22 12:44:16 2016 -0800 Committer: Aravindan Vijayan <avija...@hortonworks.com> Committed: Thu Dec 22 12:44:16 2016 -0800 ---------------------------------------------------------------------- ambari-metrics/ambari-metrics-assembly/pom.xml | 12 + .../src/main/assembly/collector-windows.xml | 1 + .../src/main/assembly/collector.xml | 1 + .../sink/timeline/TimelineMetricMetadata.java | 12 +- .../conf/unix/metrics_whitelist | 527 +++++++++++++++++++ .../conf/windows/metrics_whitelist | 527 +++++++++++++++++++ .../ambari-metrics-timelineservice/pom.xml | 21 + .../timeline/HBaseTimelineMetricStore.java | 17 +- .../metrics/timeline/PhoenixHBaseAccessor.java | 19 +- .../timeline/TimelineMetricConfiguration.java | 3 + .../metrics/timeline/TimelineMetricStore.java | 2 +- .../metrics/timeline/TimelineMetricsFilter.java | 153 ++++++ .../timeline/aggregators/AggregatorUtils.java | 45 -- .../TimelineMetricAppAggregator.java | 5 +- .../TimelineMetricMetadataManager.java | 5 +- .../timeline/query/PhoenixTransactSQL.java | 12 +- .../webapp/TimelineWebServices.java | 7 +- .../timeline/TestTimelineMetricStore.java | 2 +- .../timeline/TimelineMetricsFilterTest.java | 99 ++++ .../timeline/discovery/TestMetadataManager.java | 21 +- .../timeline/discovery/TestMetadataSync.java | 10 +- .../resources/test_data/metric_whitelist.dat | 6 + 22 files changed, 1419 insertions(+), 88 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-assembly/pom.xml b/ambari-metrics/ambari-metrics-assembly/pom.xml index d73a0af..cdcc2a7 100644 --- a/ambari-metrics/ambari-metrics-assembly/pom.xml +++ b/ambari-metrics/ambari-metrics-assembly/pom.xml @@ -335,6 +335,9 @@ <location>${collector.dir}/conf/unix/log4j.properties</location> </source> <source> + <location>${collector.dir}/conf/unix/metrics_whitelist</location> + </source> + <source> <location>${collector.dir}/target/embedded/${hbase.folder}/conf/hbase-site.xml</location> </source> </sources> @@ -870,6 +873,15 @@ </mapper> </data> <data> + <src>${collector.dir}/conf/unix/metrics_whitelist</src> + <type>file</type> + <mapper> + <type>perm</type> + <filemode>755</filemode> + <prefix>/etc/ambari-metrics-collector/conf</prefix> + </mapper> + </data> + <data> <src>${collector.dir}/target/embedded/${hbase.folder}/conf/hbase-site.xml</src> <type>file</type> <mapper> http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml index 4f1ad47..1ea6c46 100644 --- a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml +++ b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector-windows.xml @@ -45,6 +45,7 @@ <include>ams-env.cmd</include> <include>ams-site.xml</include> <include>log4j.properties</include> + <include>metrics_whitelist</include> </includes> </fileSet> <fileSet> http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector.xml b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector.xml index 609ddb3..0997b48 100644 --- a/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector.xml +++ b/ambari-metrics/ambari-metrics-assembly/src/main/assembly/collector.xml @@ -44,6 +44,7 @@ <include>ams-env.sh</include> <include>ams-site.xml</include> <include>log4j.properties</include> + <include>metrics_whitelist</include> </includes> </fileSet> <fileSet> http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricMetadata.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricMetadata.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricMetadata.java index 1f413a0..727becc 100644 --- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricMetadata.java +++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetricMetadata.java @@ -36,6 +36,7 @@ public class TimelineMetricMetadata { private String type = "UNDEFINED"; private Long seriesStartTime; boolean supportsAggregates = true; + boolean isWhitelisted = false; // Serialization ignored helper flag boolean isPersisted = false; @@ -52,13 +53,14 @@ public class TimelineMetricMetadata { public TimelineMetricMetadata(String metricName, String appId, String units, String type, Long seriesStartTime, - boolean supportsAggregates) { + boolean supportsAggregates, boolean isWhitelisted) { this.metricName = metricName; this.appId = appId; this.units = units; this.type = type; this.seriesStartTime = seriesStartTime; this.supportsAggregates = supportsAggregates; + this.isWhitelisted = isWhitelisted; } @XmlElement(name = "metricname") @@ -112,6 +114,11 @@ public class TimelineMetricMetadata { return supportsAggregates; } + @XmlElement(name = "isWhitelisted") + public boolean isWhitelisted() { + return isWhitelisted; + } + public void setSupportsAggregates(boolean supportsAggregates) { this.supportsAggregates = supportsAggregates; } @@ -140,7 +147,8 @@ public class TimelineMetricMetadata { return (this.units != null && !this.units.equals(metadata.getUnits())) || (this.type != null && !this.type.equals(metadata.getType())) || //!this.lastRecordedTime.equals(metadata.getLastRecordedTime()) || // TODO: support - !this.supportsAggregates == metadata.isSupportsAggregates(); + !this.supportsAggregates == metadata.isSupportsAggregates() || + this.isWhitelisted != metadata.isWhitelisted; } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-timelineservice/conf/unix/metrics_whitelist ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/metrics_whitelist b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/metrics_whitelist new file mode 100644 index 0000000..bd36429 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/metrics_whitelist @@ -0,0 +1,527 @@ +AMBARI_METRICS.SmokeTest.FakeMetric +ActiveThreads +BytesQueued +BytesReceivedLast5Minutes +BytesSentLast5Minutes +ChannelSize +EventPutSuccessCount +EventTakeSuccessCount +FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Capacity +FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.DfsUsed +FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.NumFailedVolumes +FlowFilesQueued +FlowFilesReceivedLast5Minutes +FlowFilesSentLast5Minutes +Free Slots +Supervisors +TimelineMetricStoreWatcher.FakeMetric +Topologies +Total Executors +Total Slots +Total Tasks +Used Slots +bytes_in +bytes_out +cache.CacheMetrics.CacheCapacityRemaining +cache.CacheMetrics.CacheCapacityRemainingPercentage +cache.CacheMetrics.CacheCapacityTotal +cache.CacheMetrics.CacheCapacityUsed +cache.CacheMetrics.CacheHitRatio +cache.CacheMetrics.CacheReadRequests +cpu_idle +cpu_intr +cpu_nice +cpu_num +cpu_sintr +cpu_steal +cpu_system +cpu_user +cpu_wio +default.General.active_calls_api_hs2_operation_INITIALIZED +default.General.active_calls_api_hs2_operation_PENDING +default.General.active_calls_api_hs2_operation_RUNNING +default.General.api_create_table_count +default.General.api_get_all_databases_75thpercentile +default.General.api_get_all_databases_999thpercentile +default.General.api_get_partitions_by_names_75thpercentile +default.General.api_get_partitions_by_names_999thpercentile +default.General.api_hs2_operation_INITIALIZED_count +default.General.create_total_count_partitions +default.General.hs2_completed_operation_CLOSED +default.General.hs2_completed_operation_ERROR +default.General.hs2_completed_operation_FINISHED +default.General.init_total_count_dbs +default.General.init_total_count_partitions +default.General.init_total_count_tables +default.General.jvm.pause.extraSleepTime +default.General.memory.heap.committed +default.General.memory.heap.max +default.General.memory.heap.used +default.General.memory.non-heap.committed +default.General.memory.non-heap.max +default.General.memory.non-heap.used +dfs.FSNamesystem.CapacityTotalGB +dfs.FSNamesystem.CapacityUsedGB +dfs.FSNamesystem.ExpiredHeartbeats +dfs.FSNamesystem.LastCheckpointTime +dfs.FSNamesystem.LastWrittenTransactionId +dfs.FSNamesystem.LockQueueLength +dfs.FSNamesystem.MillisSinceLastLoadedEdits +dfs.FSNamesystem.MissingReplOneBlocks +dfs.FSNamesystem.NumFilesUnderConstruction +dfs.FSNamesystem.PendingDataNodeMessageCount +dfs.FSNamesystem.PostponedMisreplicatedBlocks +dfs.FSNamesystem.Snapshots +dfs.FSNamesystem.SnapshottableDirectories +dfs.FSNamesystem.StaleDataNodes +dfs.FSNamesystem.TotalFiles +dfs.FSNamesystem.TotalLoad +dfs.FSNamesystem.TransactionsSinceLastCheckpoint +dfs.FSNamesystem.TransactionsSinceLastLogRoll +dfs.FSNamesystem.UnderReplicatedBlocks +dfs.datanode.BlocksRead +dfs.datanode.BlocksWritten +dfs.datanode.DatanodeNetworkErrors +dfs.datanode.FsyncNanosAvgTime +dfs.datanode.FsyncNanosNumOps +dfs.datanode.PacketAckRoundTripTimeNanosAvgTime +dfs.datanode.PacketAckRoundTripTimeNanosNumOps +dfs.datanode.SendDataPacketBlockedOnNetworkNanosAvgTime +dfs.datanode.SendDataPacketBlockedOnNetworkNanosNumOps +dfs.datanode.SendDataPacketTransferNanosAvgTime +dfs.datanode.SendDataPacketTransferNanosNumOps +dfs.namenode.TotalFileOps +disk_free +disk_total +disk_used +executors.ExecutorMetrics.ExecutorAvailableFreeSlots +executors.ExecutorMetrics.ExecutorAvailableFreeSlotsPercent +executors.ExecutorMetrics.ExecutorCacheMemoryPerInstance +executors.ExecutorMetrics.ExecutorJvmMaxMemory +executors.ExecutorMetrics.ExecutorMaxFreeSlots +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost_300s50thPercentileLatency +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost_300s90thPercentileLatency +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost_300s95thPercentileLatency +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost_300s99thPercentileLatency +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill_300s50thPercentileLatency +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill_300s90thPercentileLatency +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill_300s95thPercentileLatency +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill_300s99thPercentileLatency +executors.ExecutorMetrics.ExecutorMemoryPerInstance +executors.ExecutorMetrics.ExecutorNumExecutorsPerInstance +executors.ExecutorMetrics.ExecutorNumQueuedRequests +executors.ExecutorMetrics.ExecutorTotalEvictedFromWaitQueue +executors.ExecutorMetrics.ExecutorTotalFailed +executors.ExecutorMetrics.ExecutorTotalKilled +executors.ExecutorMetrics.ExecutorTotalRejectedRequests +executors.ExecutorMetrics.ExecutorTotalRequestsHandled +executors.ExecutorMetrics.ExecutorTotalSuccess +gc.ConcurrentMarkSweep.count +gc.ConcurrentMarkSweep.time +gc.ParNew.count +gc.ParNew.time +io.IOMetrics.MaxDecodingTime +io.IOMetrics.PercentileDecodingTime_30s50thPercentileLatency +io.IOMetrics.PercentileDecodingTime_30s90thPercentileLatency +io.IOMetrics.PercentileDecodingTime_30s95thPercentileLatency +io.IOMetrics.PercentileDecodingTime_30s99thPercentileLatency +ipc.client.org.apache.hadoop.ipc.DecayRpcScheduler.Caller(*).Priority +ipc.client.org.apache.hadoop.ipc.DecayRpcScheduler.Caller(*).Volume +jvm.JvmMetrics.GcCount +jvm.JvmMetrics.GcCountConcurrentMarkSweep +jvm.JvmMetrics.GcCountParNew +jvm.JvmMetrics.GcNumWarnThresholdExceeded +jvm.JvmMetrics.GcTimeMillis +jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep +jvm.JvmMetrics.GcTimeMillisParNew +jvm.JvmMetrics.GcTotalExtraSleepTime +jvm.JvmMetrics.LogError +jvm.JvmMetrics.LogFatal +jvm.JvmMetrics.MemHeapCommittedM +jvm.JvmMetrics.MemHeapMaxM +jvm.JvmMetrics.MemHeapUsedM +jvm.JvmMetrics.MemNonHeapCommittedM +jvm.JvmMetrics.MemNonHeapMaxM +jvm.JvmMetrics.MemNonHeapUsedM +jvm.JvmMetrics.ThreadsBlocked +jvm.JvmMetrics.ThreadsNew +jvm.JvmMetrics.ThreadsRunnable +jvm.JvmMetrics.ThreadsTerminated +jvm.JvmMetrics.ThreadsTimedWaiting +jvm.JvmMetrics.ThreadsWaiting +jvm.Master.JvmMetrics.ThreadsBlocked +jvm.Master.JvmMetrics.ThreadsNew +jvm.Master.JvmMetrics.ThreadsRunnable +jvm.Master.JvmMetrics.ThreadsTerminated +jvm.Master.JvmMetrics.ThreadsTimedWaiting +jvm.Master.JvmMetrics.ThreadsWaiting +jvm.RegionServer.JvmMetrics.GcCount +jvm.RegionServer.JvmMetrics.GcCountConcurrentMarkSweep +jvm.RegionServer.JvmMetrics.GcCountParNew +jvm.RegionServer.JvmMetrics.GcTimeMillis +jvm.RegionServer.JvmMetrics.GcTimeMillisConcurrentMarkSweep +jvm.RegionServer.JvmMetrics.GcTimeMillisParNew +jvm.RegionServer.JvmMetrics.MemHeapCommittedM +jvm.RegionServer.JvmMetrics.MemHeapMaxM +jvm.RegionServer.JvmMetrics.MemHeapUsedM +jvm.RegionServer.JvmMetrics.MemNonHeapCommittedM +jvm.RegionServer.JvmMetrics.MemNonHeapMaxM +jvm.RegionServer.JvmMetrics.MemNonHeapUsedM +jvm.RegionServer.JvmMetrics.ThreadsBlocked +jvm.RegionServer.JvmMetrics.ThreadsNew +jvm.RegionServer.JvmMetrics.ThreadsRunnable +jvm.RegionServer.JvmMetrics.ThreadsTerminated +jvm.RegionServer.JvmMetrics.ThreadsTimedWaiting +jvm.RegionServer.JvmMetrics.ThreadsWaiting +jvm.daemon_thread_count +jvm.file_descriptor_usage +jvm.heap_usage +jvm.thread_count +jvm.uptime +kafka.controller.ControllerStats.LeaderElectionRateAndTimeMs.1MinuteRate +kafka.controller.ControllerStats.LeaderElectionRateAndTimeMs.count +kafka.controller.ControllerStats.UncleanLeaderElectionsPerSec.1MinuteRate +kafka.controller.ControllerStats.UncleanLeaderElectionsPerSec.count +kafka.controller.KafkaController.ActiveControllerCount +kafka.controller.KafkaController.OfflinePartitionsCount +kafka.log.LogFlushStats.LogFlushRateAndTimeMs.1MinuteRate +kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.count +kafka.network.RequestMetrics.RequestsPerSec.request.Produce.count +kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.99percentile +kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.max +kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.mean +kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.min +kafka.network.RequestMetrics.TotalTimeMs.request.FetchFollower.99percentile +kafka.network.RequestMetrics.TotalTimeMs.request.FetchFollower.max +kafka.network.RequestMetrics.TotalTimeMs.request.FetchFollower.mean +kafka.network.RequestMetrics.TotalTimeMs.request.FetchFollower.min +kafka.network.RequestMetrics.TotalTimeMs.request.Produce.99percentile +kafka.network.RequestMetrics.TotalTimeMs.request.Produce.max +kafka.network.RequestMetrics.TotalTimeMs.request.Produce.mean +kafka.network.RequestMetrics.TotalTimeMs.request.Produce.min +kafka.server.BrokerTopicMetrics.BytesInPerSec.1MinuteRate +kafka.server.BrokerTopicMetrics.BytesInPerSec.count +kafka.server.BrokerTopicMetrics.BytesInPerSec.topic.*.count +kafka.server.BrokerTopicMetrics.BytesOutPerSec.1MinuteRate +kafka.server.BrokerTopicMetrics.BytesOutPerSec.count +kafka.server.BrokerTopicMetrics.BytesOutPerSec.topic.*.count +kafka.server.BrokerTopicMetrics.MessagesInPerSec.1MinuteRate +kafka.server.BrokerTopicMetrics.MessagesInPerSec.count +kafka.server.BrokerTopicMetrics.MessagesInPerSec.topic.*.count +kafka.server.BrokerTopicMetrics.TotalProduceRequestsPerSec.topic.*.count +kafka.server.FetcherLagMetrics.ConsumerLag.clientId.ReplicaFetcherThread-0-1001.partition.0.topic.* +kafka.server.ReplicaFetcherManager.MaxLag.clientId.Replica +kafka.server.ReplicaManager.IsrExpandsPerSec.count +kafka.server.ReplicaManager.IsrShrinksPerSec.count +kafka.server.ReplicaManager.LeaderCount +kafka.server.ReplicaManager.PartitionCount +kafka.server.ReplicaManager.UnderReplicatedPartitions +load_fifteen +load_five +load_one +master.AssignmentManger.ritCount +master.AssignmentManger.ritCountOverThreshold +master.AssignmentManger.ritOldestAge +master.Master.QueueCallTime_median +master.Master.TotalCallTime_num_ops +master.Master.exceptions +master.Server.averageLoad +master.Server.numDeadRegionServers +master.Server.numRegionServers +mem_buffered +mem_cached +mem_free +mem_shared +mem_total +mem_used +memory.heap.committed +memory.heap.max +memory.heap.used +memory.non-heap.committed +memory.non-heap.max +memory.non-heap.used +pkts_in +pkts_out +proc_run +proc_total +read_bps +read_bytes +read_count +read_time +regionserver.RegionServer.ProcessCallTime_75th_percentile +regionserver.RegionServer.ProcessCallTime_95th_percentile +regionserver.RegionServer.ProcessCallTime_99th_percentile +regionserver.RegionServer.ProcessCallTime_max +regionserver.RegionServer.ProcessCallTime_mean +regionserver.RegionServer.ProcessCallTime_median +regionserver.RegionServer.QueueCallTime_75th_percentile +regionserver.RegionServer.QueueCallTime_95th_percentile +regionserver.RegionServer.QueueCallTime_99th_percentile +regionserver.RegionServer.QueueCallTime_max +regionserver.RegionServer.QueueCallTime_mean +regionserver.RegionServer.QueueCallTime_median +regionserver.RegionServer.TotalCallTime_num_ops +regionserver.RegionServer.authenticationFailures +regionserver.RegionServer.authenticationSuccesses +regionserver.RegionServer.authorizationFailures +regionserver.RegionServer.authorizationSuccesses +regionserver.RegionServer.exceptions +regionserver.RegionServer.numActiveHandler +regionserver.RegionServer.numCallsInGeneralQueue +regionserver.RegionServer.numCallsInPriorityQueue +regionserver.RegionServer.numCallsInReplicationQueue +regionserver.RegionServer.numOpenConnections +regionserver.RegionServer.queueSize +regionserver.RegionServer.receivedBytes +regionserver.RegionServer.sentBytes +regionserver.Server.Append_75th_percentile +regionserver.Server.Append_95th_percentile +regionserver.Server.Append_99th_percentile +regionserver.Server.Append_max +regionserver.Server.Append_mean +regionserver.Server.Append_median +regionserver.Server.Append_min +regionserver.Server.Append_num_ops +regionserver.Server.Delete_75th_percentile +regionserver.Server.Delete_95th_percentile +regionserver.Server.Delete_99th_percentile +regionserver.Server.Delete_max +regionserver.Server.Delete_mean +regionserver.Server.Delete_median +regionserver.Server.Delete_min +regionserver.Server.Delete_num_ops +regionserver.Server.Get_75th_percentile +regionserver.Server.Get_95th_percentile +regionserver.Server.Get_99th_percentile +regionserver.Server.Get_max +regionserver.Server.Get_mean +regionserver.Server.Get_median +regionserver.Server.Get_min +regionserver.Server.Get_num_ops +regionserver.Server.Increment_75th_percentile +regionserver.Server.Increment_95th_percentile +regionserver.Server.Increment_99th_percentile +regionserver.Server.Increment_max +regionserver.Server.Increment_mean +regionserver.Server.Increment_median +regionserver.Server.Increment_min +regionserver.Server.Increment_num_ops +regionserver.Server.Mutate_75th_percentile +regionserver.Server.Mutate_95th_percentile +regionserver.Server.Mutate_99th_percentile +regionserver.Server.Mutate_max +regionserver.Server.Mutate_mean +regionserver.Server.Mutate_median +regionserver.Server.Mutate_min +regionserver.Server.Mutate_num_ops +regionserver.Server.Replay_75th_percentile +regionserver.Server.Replay_95th_percentile +regionserver.Server.Replay_99th_percentile +regionserver.Server.Replay_max +regionserver.Server.Replay_mean +regionserver.Server.Replay_median +regionserver.Server.Replay_min +regionserver.Server.Replay_num_ops +regionserver.Server.ScanNext_95th_percentile +regionserver.Server.ScanNext_num_ops +regionserver.Server.ScanTime_75th_percentile +regionserver.Server.ScanTime_95th_percentile +regionserver.Server.ScanTime_99th_percentile +regionserver.Server.ScanTime_max +regionserver.Server.ScanTime_mean +regionserver.Server.ScanTime_median +regionserver.Server.blockCacheCount +regionserver.Server.blockCacheCountHitPercent +regionserver.Server.blockCacheEvictionCount +regionserver.Server.blockCacheExpressHitPercent +regionserver.Server.blockCacheFreeSize +regionserver.Server.blockCacheHitCount +regionserver.Server.blockCacheMissCount +regionserver.Server.blockCacheSize +regionserver.Server.blockCountHitPercent +regionserver.Server.compactionQueueLength +regionserver.Server.flushQueueLength +regionserver.Server.hlogFileCount +regionserver.Server.hlogFileSize +regionserver.Server.memStoreSize +regionserver.Server.percentFilesLocal +regionserver.Server.readRequestCount +regionserver.Server.regionCount +regionserver.Server.slowDeleteCount +regionserver.Server.slowGetCount +regionserver.Server.slowPutCount +regionserver.Server.splitQueueLength +regionserver.Server.staticBloomSize +regionserver.Server.staticIndexSize +regionserver.Server.storeCount +regionserver.Server.storeFileCount +regionserver.Server.storeFileIndexSize +regionserver.Server.storeFileSize +regionserver.Server.totalRequestCount +regionserver.Server.updatesBlockedTime +regionserver.Server.writeRequestCount +regionserver.Tables.*_metric_averageRegionSize +regionserver.Tables.*_metric_avgStoreFileAge +regionserver.Tables.*_metric_flushMemstoreSize_95th_percentile +regionserver.Tables.*_metric_flushMemstoreSize_max +regionserver.Tables.*_metric_flushMemstoreSize_mean +regionserver.Tables.*_metric_flushMemstoreSize_median +regionserver.Tables.*_metric_flushOutputSize_95th_percentile +regionserver.Tables.*_metric_flushOutputSize_max +regionserver.Tables.*_metric_flushOutputSize_mean +regionserver.Tables.*_metric_flushOutputSize_median +regionserver.Tables.*_metric_flushTime_95th_percentile +regionserver.Tables.*_metric_flushTime_max +regionserver.Tables.*_metric_flushTime_mean +regionserver.Tables.*_metric_flushTime_median +regionserver.Tables.*_metric_flushTime_num_ops +regionserver.Tables.*_metric_flushedMemstoreBytes +regionserver.Tables.*_metric_flushedOutputBytes +regionserver.Tables.*_metric_maxStoreFileAge +regionserver.Tables.*_metric_memStoreSize +regionserver.Tables.*_metric_minStoreFileAge +regionserver.Tables.*_metric_numReferenceFiles +regionserver.Tables.*_metric_readRequestCount +regionserver.Tables.*_metric_regionCount +regionserver.Tables.*_metric_storeCount +regionserver.Tables.*_metric_storeFileCount +regionserver.Tables.*_metric_storeFileSize +regionserver.Tables.*_metric_tableSize +regionserver.Tables.*_metric_totalRequestCount +regionserver.Tables.*_metric_writeRequestCount +regionserver.Users.*_metric_append_num_ops +regionserver.Users.*_metric_delete_num_ops +regionserver.Users.*_metric_get_num_ops +regionserver.Users.*_metric_increment_num_ops +regionserver.Users.*_metric_mutate_num_ops +regionserver.Users.*_metric_scanTime_num_ops +regionserver.WAL.AppendSize_75th_percentile +regionserver.WAL.AppendSize_95th_percentile +regionserver.WAL.AppendSize_99th_percentile +regionserver.WAL.AppendSize_max +regionserver.WAL.AppendSize_mean +regionserver.WAL.AppendSize_median +regionserver.WAL.AppendTime_75th_percentile +regionserver.WAL.AppendTime_95th_percentile +regionserver.WAL.AppendTime_99th_percentile +regionserver.WAL.AppendTime_max +regionserver.WAL.AppendTime_mean +regionserver.WAL.AppendTime_median +regionserver.WAL.SyncTime_75th_percentile +regionserver.WAL.SyncTime_95th_percentile +regionserver.WAL.SyncTime_99th_percentile +regionserver.WAL.SyncTime_max +regionserver.WAL.SyncTime_mean +regionserver.WAL.SyncTime_median +regionserver.WAL.SyncTime_num_ops +regionserver.WAL.appendCount +regionserver.WAL.slowAppendCount +rpc.rpc.CallQueueLength +rpc.rpc.NumOpenConnections +rpc.rpc.RpcAuthenticationFailures +rpc.rpc.RpcAuthorizationFailures +rpc.rpc.RpcProcessingTimeAvgTime +rpc.rpc.RpcQueueTimeAvgTime +rpc.rpc.RpcSlowCalls +rpc.rpc.client.CallQueueLength +rpc.rpc.client.NumOpenConnections +rpc.rpc.client.RpcClientBackoff +rpc.rpc.client.RpcProcessingTimeAvgTime +rpc.rpc.client.RpcProcessingTimeNumOps +rpc.rpc.client.RpcQueueTimeAvgTime +rpc.rpc.client.RpcQueueTimeNumOps +rpc.rpc.client.RpcSlowCalls +rpc.rpc.datanode.CallQueueLength +rpc.rpc.datanode.RpcProcessingTimeAvgTime +rpc.rpc.datanode.RpcProcessingTimeNumOps +rpc.rpc.datanode.RpcQueueTimeAvgTime +rpc.rpc.datanode.RpcQueueTimeNumOps +rpc.rpc.datanode.RpcSlowCalls +rpcdetailed.rpcdetailed.client.AddBlockAvgTime +rpcdetailed.rpcdetailed.client.AddBlockNumOps +swap_free +swap_total +threads.blocked.count +threads.count +threads.daemon.count +threads.deadlock.count +threads.runnable.count +topology.*.%.--ack-count.% +topology.*.%.--complete-latency.% +topology.*.%.--emit-count.% +topology.*.%.--execute-count.% +topology.*.%.--fail-count.% +topology.*.%.--receive.population +topology.*.%.--sendqueue.population +topology.*.*.%.--ack-count.% +topology.*.*.%.--complete-latency.% +topology.*.*.%.--emit-count.% +topology.*.*.%.--execute-count.% +topology.*.*.%.--fail-count.% +topology.*.*.%.--process-latency.% +topology.*.*.%.--receive.population +topology.*.*.%.--sendqueue.population +topology.*.--system.%.%.-1.GC/PSMarkSweep.count +topology.*.--system.%.%.-1.GC/PSMarkSweep.timeMs +topology.*.--system.%.%.-1.GC/PSScavenge.count +topology.*.--system.%.%.-1.GC/PSScavenge.timeMs +topology.*.--system.%.%.-1.memory/heap.usedBytes +topology.*.--system.%.%.-1.memory/nonHeap.usedBytes +topology.*.kafka-topic.*.*.latestCompletedOffset +topology.*.kafka-topic.*.*.latestTimeOffset +write_bps +write_bytes +write_count +write_time +yarn.ClusterMetrics.AMLaunchDelayAvgTime +yarn.ClusterMetrics.AMRegisterDelayAvgTime +yarn.ClusterMetrics.NumActiveNMs +yarn.ClusterMetrics.NumLostNMs +yarn.ClusterMetrics.NumUnhealthyNMs +yarn.NodeManagerMetrics.AllocatedContainers +yarn.NodeManagerMetrics.AllocatedGB +yarn.NodeManagerMetrics.AllocatedVCores +yarn.NodeManagerMetrics.AvailableGB +yarn.NodeManagerMetrics.BadLocalDirs +yarn.NodeManagerMetrics.BadLogDirs +yarn.NodeManagerMetrics.ContainerLaunchDurationAvgTime +yarn.NodeManagerMetrics.ContainersCompleted +yarn.NodeManagerMetrics.ContainersFailed +yarn.NodeManagerMetrics.ContainersIniting +yarn.NodeManagerMetrics.ContainersKilled +yarn.NodeManagerMetrics.ContainersLaunched +yarn.NodeManagerMetrics.ContainersRunning +yarn.NodeManagerMetrics.GoodLocalDirsDiskUtilizationPerc +yarn.NodeManagerMetrics.GoodLogDirsDiskUtilizationPerc +yarn.QueueMetrics.Queue=root.AMResourceLimitMB +yarn.QueueMetrics.Queue=root.AggregateContainersAllocated +yarn.QueueMetrics.Queue=root.AggregateNodeLocalContainersAllocated +yarn.QueueMetrics.Queue=root.AggregateOffSwitchContainersAllocated +yarn.QueueMetrics.Queue=root.AggregateRackLocalContainersAllocated +yarn.QueueMetrics.Queue=root.AllocatedContainers +yarn.QueueMetrics.Queue=root.AllocatedMB +yarn.QueueMetrics.Queue=root.AllocatedVCores +yarn.QueueMetrics.Queue=root.AppAttemptFirstContainerAllocationDelayAvgTime +yarn.QueueMetrics.Queue=root.AppsCompleted +yarn.QueueMetrics.Queue=root.AppsFailed +yarn.QueueMetrics.Queue=root.AppsKilled +yarn.QueueMetrics.Queue=root.AppsPending +yarn.QueueMetrics.Queue=root.AppsRunning +yarn.QueueMetrics.Queue=root.AppsSubmitted +yarn.QueueMetrics.Queue=root.AvailableMB +yarn.QueueMetrics.Queue=root.AvailableVCores +yarn.QueueMetrics.Queue=root.PendingContainers +yarn.QueueMetrics.Queue=root.PendingMB +yarn.QueueMetrics.Queue=root.UsedAMResourceMB +yarn.QueueMetrics.Queue=root.default.ReservedContainers +yarn.QueueMetrics.Queue=root.default.ReservedMB +yarn.QueueMetrics.Queue=root.running_0 +yarn.QueueMetrics.Queue=root.running_1440 +yarn.QueueMetrics.Queue=root.running_300 +yarn.QueueMetrics.Queue=root.running_60 +yarn.TimelineDataManagerMetrics.GetEntitiesTimeAvgTime +yarn.TimelineDataManagerMetrics.GetEntitiesTotal +yarn.TimelineDataManagerMetrics.PostEntitiesTimeAvgTime +yarn.TimelineDataManagerMetrics.PostEntitiesTotal \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-timelineservice/conf/windows/metrics_whitelist ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/windows/metrics_whitelist b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/metrics_whitelist new file mode 100644 index 0000000..bd36429 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/metrics_whitelist @@ -0,0 +1,527 @@ +AMBARI_METRICS.SmokeTest.FakeMetric +ActiveThreads +BytesQueued +BytesReceivedLast5Minutes +BytesSentLast5Minutes +ChannelSize +EventPutSuccessCount +EventTakeSuccessCount +FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.Capacity +FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.DfsUsed +FSDatasetState.org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.NumFailedVolumes +FlowFilesQueued +FlowFilesReceivedLast5Minutes +FlowFilesSentLast5Minutes +Free Slots +Supervisors +TimelineMetricStoreWatcher.FakeMetric +Topologies +Total Executors +Total Slots +Total Tasks +Used Slots +bytes_in +bytes_out +cache.CacheMetrics.CacheCapacityRemaining +cache.CacheMetrics.CacheCapacityRemainingPercentage +cache.CacheMetrics.CacheCapacityTotal +cache.CacheMetrics.CacheCapacityUsed +cache.CacheMetrics.CacheHitRatio +cache.CacheMetrics.CacheReadRequests +cpu_idle +cpu_intr +cpu_nice +cpu_num +cpu_sintr +cpu_steal +cpu_system +cpu_user +cpu_wio +default.General.active_calls_api_hs2_operation_INITIALIZED +default.General.active_calls_api_hs2_operation_PENDING +default.General.active_calls_api_hs2_operation_RUNNING +default.General.api_create_table_count +default.General.api_get_all_databases_75thpercentile +default.General.api_get_all_databases_999thpercentile +default.General.api_get_partitions_by_names_75thpercentile +default.General.api_get_partitions_by_names_999thpercentile +default.General.api_hs2_operation_INITIALIZED_count +default.General.create_total_count_partitions +default.General.hs2_completed_operation_CLOSED +default.General.hs2_completed_operation_ERROR +default.General.hs2_completed_operation_FINISHED +default.General.init_total_count_dbs +default.General.init_total_count_partitions +default.General.init_total_count_tables +default.General.jvm.pause.extraSleepTime +default.General.memory.heap.committed +default.General.memory.heap.max +default.General.memory.heap.used +default.General.memory.non-heap.committed +default.General.memory.non-heap.max +default.General.memory.non-heap.used +dfs.FSNamesystem.CapacityTotalGB +dfs.FSNamesystem.CapacityUsedGB +dfs.FSNamesystem.ExpiredHeartbeats +dfs.FSNamesystem.LastCheckpointTime +dfs.FSNamesystem.LastWrittenTransactionId +dfs.FSNamesystem.LockQueueLength +dfs.FSNamesystem.MillisSinceLastLoadedEdits +dfs.FSNamesystem.MissingReplOneBlocks +dfs.FSNamesystem.NumFilesUnderConstruction +dfs.FSNamesystem.PendingDataNodeMessageCount +dfs.FSNamesystem.PostponedMisreplicatedBlocks +dfs.FSNamesystem.Snapshots +dfs.FSNamesystem.SnapshottableDirectories +dfs.FSNamesystem.StaleDataNodes +dfs.FSNamesystem.TotalFiles +dfs.FSNamesystem.TotalLoad +dfs.FSNamesystem.TransactionsSinceLastCheckpoint +dfs.FSNamesystem.TransactionsSinceLastLogRoll +dfs.FSNamesystem.UnderReplicatedBlocks +dfs.datanode.BlocksRead +dfs.datanode.BlocksWritten +dfs.datanode.DatanodeNetworkErrors +dfs.datanode.FsyncNanosAvgTime +dfs.datanode.FsyncNanosNumOps +dfs.datanode.PacketAckRoundTripTimeNanosAvgTime +dfs.datanode.PacketAckRoundTripTimeNanosNumOps +dfs.datanode.SendDataPacketBlockedOnNetworkNanosAvgTime +dfs.datanode.SendDataPacketBlockedOnNetworkNanosNumOps +dfs.datanode.SendDataPacketTransferNanosAvgTime +dfs.datanode.SendDataPacketTransferNanosNumOps +dfs.namenode.TotalFileOps +disk_free +disk_total +disk_used +executors.ExecutorMetrics.ExecutorAvailableFreeSlots +executors.ExecutorMetrics.ExecutorAvailableFreeSlotsPercent +executors.ExecutorMetrics.ExecutorCacheMemoryPerInstance +executors.ExecutorMetrics.ExecutorJvmMaxMemory +executors.ExecutorMetrics.ExecutorMaxFreeSlots +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost_300s50thPercentileLatency +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost_300s90thPercentileLatency +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost_300s95thPercentileLatency +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeLost_300s99thPercentileLatency +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill_300s50thPercentileLatency +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill_300s90thPercentileLatency +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill_300s95thPercentileLatency +executors.ExecutorMetrics.ExecutorMaxPreemptionTimeToKill_300s99thPercentileLatency +executors.ExecutorMetrics.ExecutorMemoryPerInstance +executors.ExecutorMetrics.ExecutorNumExecutorsPerInstance +executors.ExecutorMetrics.ExecutorNumQueuedRequests +executors.ExecutorMetrics.ExecutorTotalEvictedFromWaitQueue +executors.ExecutorMetrics.ExecutorTotalFailed +executors.ExecutorMetrics.ExecutorTotalKilled +executors.ExecutorMetrics.ExecutorTotalRejectedRequests +executors.ExecutorMetrics.ExecutorTotalRequestsHandled +executors.ExecutorMetrics.ExecutorTotalSuccess +gc.ConcurrentMarkSweep.count +gc.ConcurrentMarkSweep.time +gc.ParNew.count +gc.ParNew.time +io.IOMetrics.MaxDecodingTime +io.IOMetrics.PercentileDecodingTime_30s50thPercentileLatency +io.IOMetrics.PercentileDecodingTime_30s90thPercentileLatency +io.IOMetrics.PercentileDecodingTime_30s95thPercentileLatency +io.IOMetrics.PercentileDecodingTime_30s99thPercentileLatency +ipc.client.org.apache.hadoop.ipc.DecayRpcScheduler.Caller(*).Priority +ipc.client.org.apache.hadoop.ipc.DecayRpcScheduler.Caller(*).Volume +jvm.JvmMetrics.GcCount +jvm.JvmMetrics.GcCountConcurrentMarkSweep +jvm.JvmMetrics.GcCountParNew +jvm.JvmMetrics.GcNumWarnThresholdExceeded +jvm.JvmMetrics.GcTimeMillis +jvm.JvmMetrics.GcTimeMillisConcurrentMarkSweep +jvm.JvmMetrics.GcTimeMillisParNew +jvm.JvmMetrics.GcTotalExtraSleepTime +jvm.JvmMetrics.LogError +jvm.JvmMetrics.LogFatal +jvm.JvmMetrics.MemHeapCommittedM +jvm.JvmMetrics.MemHeapMaxM +jvm.JvmMetrics.MemHeapUsedM +jvm.JvmMetrics.MemNonHeapCommittedM +jvm.JvmMetrics.MemNonHeapMaxM +jvm.JvmMetrics.MemNonHeapUsedM +jvm.JvmMetrics.ThreadsBlocked +jvm.JvmMetrics.ThreadsNew +jvm.JvmMetrics.ThreadsRunnable +jvm.JvmMetrics.ThreadsTerminated +jvm.JvmMetrics.ThreadsTimedWaiting +jvm.JvmMetrics.ThreadsWaiting +jvm.Master.JvmMetrics.ThreadsBlocked +jvm.Master.JvmMetrics.ThreadsNew +jvm.Master.JvmMetrics.ThreadsRunnable +jvm.Master.JvmMetrics.ThreadsTerminated +jvm.Master.JvmMetrics.ThreadsTimedWaiting +jvm.Master.JvmMetrics.ThreadsWaiting +jvm.RegionServer.JvmMetrics.GcCount +jvm.RegionServer.JvmMetrics.GcCountConcurrentMarkSweep +jvm.RegionServer.JvmMetrics.GcCountParNew +jvm.RegionServer.JvmMetrics.GcTimeMillis +jvm.RegionServer.JvmMetrics.GcTimeMillisConcurrentMarkSweep +jvm.RegionServer.JvmMetrics.GcTimeMillisParNew +jvm.RegionServer.JvmMetrics.MemHeapCommittedM +jvm.RegionServer.JvmMetrics.MemHeapMaxM +jvm.RegionServer.JvmMetrics.MemHeapUsedM +jvm.RegionServer.JvmMetrics.MemNonHeapCommittedM +jvm.RegionServer.JvmMetrics.MemNonHeapMaxM +jvm.RegionServer.JvmMetrics.MemNonHeapUsedM +jvm.RegionServer.JvmMetrics.ThreadsBlocked +jvm.RegionServer.JvmMetrics.ThreadsNew +jvm.RegionServer.JvmMetrics.ThreadsRunnable +jvm.RegionServer.JvmMetrics.ThreadsTerminated +jvm.RegionServer.JvmMetrics.ThreadsTimedWaiting +jvm.RegionServer.JvmMetrics.ThreadsWaiting +jvm.daemon_thread_count +jvm.file_descriptor_usage +jvm.heap_usage +jvm.thread_count +jvm.uptime +kafka.controller.ControllerStats.LeaderElectionRateAndTimeMs.1MinuteRate +kafka.controller.ControllerStats.LeaderElectionRateAndTimeMs.count +kafka.controller.ControllerStats.UncleanLeaderElectionsPerSec.1MinuteRate +kafka.controller.ControllerStats.UncleanLeaderElectionsPerSec.count +kafka.controller.KafkaController.ActiveControllerCount +kafka.controller.KafkaController.OfflinePartitionsCount +kafka.log.LogFlushStats.LogFlushRateAndTimeMs.1MinuteRate +kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.count +kafka.network.RequestMetrics.RequestsPerSec.request.Produce.count +kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.99percentile +kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.max +kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.mean +kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.min +kafka.network.RequestMetrics.TotalTimeMs.request.FetchFollower.99percentile +kafka.network.RequestMetrics.TotalTimeMs.request.FetchFollower.max +kafka.network.RequestMetrics.TotalTimeMs.request.FetchFollower.mean +kafka.network.RequestMetrics.TotalTimeMs.request.FetchFollower.min +kafka.network.RequestMetrics.TotalTimeMs.request.Produce.99percentile +kafka.network.RequestMetrics.TotalTimeMs.request.Produce.max +kafka.network.RequestMetrics.TotalTimeMs.request.Produce.mean +kafka.network.RequestMetrics.TotalTimeMs.request.Produce.min +kafka.server.BrokerTopicMetrics.BytesInPerSec.1MinuteRate +kafka.server.BrokerTopicMetrics.BytesInPerSec.count +kafka.server.BrokerTopicMetrics.BytesInPerSec.topic.*.count +kafka.server.BrokerTopicMetrics.BytesOutPerSec.1MinuteRate +kafka.server.BrokerTopicMetrics.BytesOutPerSec.count +kafka.server.BrokerTopicMetrics.BytesOutPerSec.topic.*.count +kafka.server.BrokerTopicMetrics.MessagesInPerSec.1MinuteRate +kafka.server.BrokerTopicMetrics.MessagesInPerSec.count +kafka.server.BrokerTopicMetrics.MessagesInPerSec.topic.*.count +kafka.server.BrokerTopicMetrics.TotalProduceRequestsPerSec.topic.*.count +kafka.server.FetcherLagMetrics.ConsumerLag.clientId.ReplicaFetcherThread-0-1001.partition.0.topic.* +kafka.server.ReplicaFetcherManager.MaxLag.clientId.Replica +kafka.server.ReplicaManager.IsrExpandsPerSec.count +kafka.server.ReplicaManager.IsrShrinksPerSec.count +kafka.server.ReplicaManager.LeaderCount +kafka.server.ReplicaManager.PartitionCount +kafka.server.ReplicaManager.UnderReplicatedPartitions +load_fifteen +load_five +load_one +master.AssignmentManger.ritCount +master.AssignmentManger.ritCountOverThreshold +master.AssignmentManger.ritOldestAge +master.Master.QueueCallTime_median +master.Master.TotalCallTime_num_ops +master.Master.exceptions +master.Server.averageLoad +master.Server.numDeadRegionServers +master.Server.numRegionServers +mem_buffered +mem_cached +mem_free +mem_shared +mem_total +mem_used +memory.heap.committed +memory.heap.max +memory.heap.used +memory.non-heap.committed +memory.non-heap.max +memory.non-heap.used +pkts_in +pkts_out +proc_run +proc_total +read_bps +read_bytes +read_count +read_time +regionserver.RegionServer.ProcessCallTime_75th_percentile +regionserver.RegionServer.ProcessCallTime_95th_percentile +regionserver.RegionServer.ProcessCallTime_99th_percentile +regionserver.RegionServer.ProcessCallTime_max +regionserver.RegionServer.ProcessCallTime_mean +regionserver.RegionServer.ProcessCallTime_median +regionserver.RegionServer.QueueCallTime_75th_percentile +regionserver.RegionServer.QueueCallTime_95th_percentile +regionserver.RegionServer.QueueCallTime_99th_percentile +regionserver.RegionServer.QueueCallTime_max +regionserver.RegionServer.QueueCallTime_mean +regionserver.RegionServer.QueueCallTime_median +regionserver.RegionServer.TotalCallTime_num_ops +regionserver.RegionServer.authenticationFailures +regionserver.RegionServer.authenticationSuccesses +regionserver.RegionServer.authorizationFailures +regionserver.RegionServer.authorizationSuccesses +regionserver.RegionServer.exceptions +regionserver.RegionServer.numActiveHandler +regionserver.RegionServer.numCallsInGeneralQueue +regionserver.RegionServer.numCallsInPriorityQueue +regionserver.RegionServer.numCallsInReplicationQueue +regionserver.RegionServer.numOpenConnections +regionserver.RegionServer.queueSize +regionserver.RegionServer.receivedBytes +regionserver.RegionServer.sentBytes +regionserver.Server.Append_75th_percentile +regionserver.Server.Append_95th_percentile +regionserver.Server.Append_99th_percentile +regionserver.Server.Append_max +regionserver.Server.Append_mean +regionserver.Server.Append_median +regionserver.Server.Append_min +regionserver.Server.Append_num_ops +regionserver.Server.Delete_75th_percentile +regionserver.Server.Delete_95th_percentile +regionserver.Server.Delete_99th_percentile +regionserver.Server.Delete_max +regionserver.Server.Delete_mean +regionserver.Server.Delete_median +regionserver.Server.Delete_min +regionserver.Server.Delete_num_ops +regionserver.Server.Get_75th_percentile +regionserver.Server.Get_95th_percentile +regionserver.Server.Get_99th_percentile +regionserver.Server.Get_max +regionserver.Server.Get_mean +regionserver.Server.Get_median +regionserver.Server.Get_min +regionserver.Server.Get_num_ops +regionserver.Server.Increment_75th_percentile +regionserver.Server.Increment_95th_percentile +regionserver.Server.Increment_99th_percentile +regionserver.Server.Increment_max +regionserver.Server.Increment_mean +regionserver.Server.Increment_median +regionserver.Server.Increment_min +regionserver.Server.Increment_num_ops +regionserver.Server.Mutate_75th_percentile +regionserver.Server.Mutate_95th_percentile +regionserver.Server.Mutate_99th_percentile +regionserver.Server.Mutate_max +regionserver.Server.Mutate_mean +regionserver.Server.Mutate_median +regionserver.Server.Mutate_min +regionserver.Server.Mutate_num_ops +regionserver.Server.Replay_75th_percentile +regionserver.Server.Replay_95th_percentile +regionserver.Server.Replay_99th_percentile +regionserver.Server.Replay_max +regionserver.Server.Replay_mean +regionserver.Server.Replay_median +regionserver.Server.Replay_min +regionserver.Server.Replay_num_ops +regionserver.Server.ScanNext_95th_percentile +regionserver.Server.ScanNext_num_ops +regionserver.Server.ScanTime_75th_percentile +regionserver.Server.ScanTime_95th_percentile +regionserver.Server.ScanTime_99th_percentile +regionserver.Server.ScanTime_max +regionserver.Server.ScanTime_mean +regionserver.Server.ScanTime_median +regionserver.Server.blockCacheCount +regionserver.Server.blockCacheCountHitPercent +regionserver.Server.blockCacheEvictionCount +regionserver.Server.blockCacheExpressHitPercent +regionserver.Server.blockCacheFreeSize +regionserver.Server.blockCacheHitCount +regionserver.Server.blockCacheMissCount +regionserver.Server.blockCacheSize +regionserver.Server.blockCountHitPercent +regionserver.Server.compactionQueueLength +regionserver.Server.flushQueueLength +regionserver.Server.hlogFileCount +regionserver.Server.hlogFileSize +regionserver.Server.memStoreSize +regionserver.Server.percentFilesLocal +regionserver.Server.readRequestCount +regionserver.Server.regionCount +regionserver.Server.slowDeleteCount +regionserver.Server.slowGetCount +regionserver.Server.slowPutCount +regionserver.Server.splitQueueLength +regionserver.Server.staticBloomSize +regionserver.Server.staticIndexSize +regionserver.Server.storeCount +regionserver.Server.storeFileCount +regionserver.Server.storeFileIndexSize +regionserver.Server.storeFileSize +regionserver.Server.totalRequestCount +regionserver.Server.updatesBlockedTime +regionserver.Server.writeRequestCount +regionserver.Tables.*_metric_averageRegionSize +regionserver.Tables.*_metric_avgStoreFileAge +regionserver.Tables.*_metric_flushMemstoreSize_95th_percentile +regionserver.Tables.*_metric_flushMemstoreSize_max +regionserver.Tables.*_metric_flushMemstoreSize_mean +regionserver.Tables.*_metric_flushMemstoreSize_median +regionserver.Tables.*_metric_flushOutputSize_95th_percentile +regionserver.Tables.*_metric_flushOutputSize_max +regionserver.Tables.*_metric_flushOutputSize_mean +regionserver.Tables.*_metric_flushOutputSize_median +regionserver.Tables.*_metric_flushTime_95th_percentile +regionserver.Tables.*_metric_flushTime_max +regionserver.Tables.*_metric_flushTime_mean +regionserver.Tables.*_metric_flushTime_median +regionserver.Tables.*_metric_flushTime_num_ops +regionserver.Tables.*_metric_flushedMemstoreBytes +regionserver.Tables.*_metric_flushedOutputBytes +regionserver.Tables.*_metric_maxStoreFileAge +regionserver.Tables.*_metric_memStoreSize +regionserver.Tables.*_metric_minStoreFileAge +regionserver.Tables.*_metric_numReferenceFiles +regionserver.Tables.*_metric_readRequestCount +regionserver.Tables.*_metric_regionCount +regionserver.Tables.*_metric_storeCount +regionserver.Tables.*_metric_storeFileCount +regionserver.Tables.*_metric_storeFileSize +regionserver.Tables.*_metric_tableSize +regionserver.Tables.*_metric_totalRequestCount +regionserver.Tables.*_metric_writeRequestCount +regionserver.Users.*_metric_append_num_ops +regionserver.Users.*_metric_delete_num_ops +regionserver.Users.*_metric_get_num_ops +regionserver.Users.*_metric_increment_num_ops +regionserver.Users.*_metric_mutate_num_ops +regionserver.Users.*_metric_scanTime_num_ops +regionserver.WAL.AppendSize_75th_percentile +regionserver.WAL.AppendSize_95th_percentile +regionserver.WAL.AppendSize_99th_percentile +regionserver.WAL.AppendSize_max +regionserver.WAL.AppendSize_mean +regionserver.WAL.AppendSize_median +regionserver.WAL.AppendTime_75th_percentile +regionserver.WAL.AppendTime_95th_percentile +regionserver.WAL.AppendTime_99th_percentile +regionserver.WAL.AppendTime_max +regionserver.WAL.AppendTime_mean +regionserver.WAL.AppendTime_median +regionserver.WAL.SyncTime_75th_percentile +regionserver.WAL.SyncTime_95th_percentile +regionserver.WAL.SyncTime_99th_percentile +regionserver.WAL.SyncTime_max +regionserver.WAL.SyncTime_mean +regionserver.WAL.SyncTime_median +regionserver.WAL.SyncTime_num_ops +regionserver.WAL.appendCount +regionserver.WAL.slowAppendCount +rpc.rpc.CallQueueLength +rpc.rpc.NumOpenConnections +rpc.rpc.RpcAuthenticationFailures +rpc.rpc.RpcAuthorizationFailures +rpc.rpc.RpcProcessingTimeAvgTime +rpc.rpc.RpcQueueTimeAvgTime +rpc.rpc.RpcSlowCalls +rpc.rpc.client.CallQueueLength +rpc.rpc.client.NumOpenConnections +rpc.rpc.client.RpcClientBackoff +rpc.rpc.client.RpcProcessingTimeAvgTime +rpc.rpc.client.RpcProcessingTimeNumOps +rpc.rpc.client.RpcQueueTimeAvgTime +rpc.rpc.client.RpcQueueTimeNumOps +rpc.rpc.client.RpcSlowCalls +rpc.rpc.datanode.CallQueueLength +rpc.rpc.datanode.RpcProcessingTimeAvgTime +rpc.rpc.datanode.RpcProcessingTimeNumOps +rpc.rpc.datanode.RpcQueueTimeAvgTime +rpc.rpc.datanode.RpcQueueTimeNumOps +rpc.rpc.datanode.RpcSlowCalls +rpcdetailed.rpcdetailed.client.AddBlockAvgTime +rpcdetailed.rpcdetailed.client.AddBlockNumOps +swap_free +swap_total +threads.blocked.count +threads.count +threads.daemon.count +threads.deadlock.count +threads.runnable.count +topology.*.%.--ack-count.% +topology.*.%.--complete-latency.% +topology.*.%.--emit-count.% +topology.*.%.--execute-count.% +topology.*.%.--fail-count.% +topology.*.%.--receive.population +topology.*.%.--sendqueue.population +topology.*.*.%.--ack-count.% +topology.*.*.%.--complete-latency.% +topology.*.*.%.--emit-count.% +topology.*.*.%.--execute-count.% +topology.*.*.%.--fail-count.% +topology.*.*.%.--process-latency.% +topology.*.*.%.--receive.population +topology.*.*.%.--sendqueue.population +topology.*.--system.%.%.-1.GC/PSMarkSweep.count +topology.*.--system.%.%.-1.GC/PSMarkSweep.timeMs +topology.*.--system.%.%.-1.GC/PSScavenge.count +topology.*.--system.%.%.-1.GC/PSScavenge.timeMs +topology.*.--system.%.%.-1.memory/heap.usedBytes +topology.*.--system.%.%.-1.memory/nonHeap.usedBytes +topology.*.kafka-topic.*.*.latestCompletedOffset +topology.*.kafka-topic.*.*.latestTimeOffset +write_bps +write_bytes +write_count +write_time +yarn.ClusterMetrics.AMLaunchDelayAvgTime +yarn.ClusterMetrics.AMRegisterDelayAvgTime +yarn.ClusterMetrics.NumActiveNMs +yarn.ClusterMetrics.NumLostNMs +yarn.ClusterMetrics.NumUnhealthyNMs +yarn.NodeManagerMetrics.AllocatedContainers +yarn.NodeManagerMetrics.AllocatedGB +yarn.NodeManagerMetrics.AllocatedVCores +yarn.NodeManagerMetrics.AvailableGB +yarn.NodeManagerMetrics.BadLocalDirs +yarn.NodeManagerMetrics.BadLogDirs +yarn.NodeManagerMetrics.ContainerLaunchDurationAvgTime +yarn.NodeManagerMetrics.ContainersCompleted +yarn.NodeManagerMetrics.ContainersFailed +yarn.NodeManagerMetrics.ContainersIniting +yarn.NodeManagerMetrics.ContainersKilled +yarn.NodeManagerMetrics.ContainersLaunched +yarn.NodeManagerMetrics.ContainersRunning +yarn.NodeManagerMetrics.GoodLocalDirsDiskUtilizationPerc +yarn.NodeManagerMetrics.GoodLogDirsDiskUtilizationPerc +yarn.QueueMetrics.Queue=root.AMResourceLimitMB +yarn.QueueMetrics.Queue=root.AggregateContainersAllocated +yarn.QueueMetrics.Queue=root.AggregateNodeLocalContainersAllocated +yarn.QueueMetrics.Queue=root.AggregateOffSwitchContainersAllocated +yarn.QueueMetrics.Queue=root.AggregateRackLocalContainersAllocated +yarn.QueueMetrics.Queue=root.AllocatedContainers +yarn.QueueMetrics.Queue=root.AllocatedMB +yarn.QueueMetrics.Queue=root.AllocatedVCores +yarn.QueueMetrics.Queue=root.AppAttemptFirstContainerAllocationDelayAvgTime +yarn.QueueMetrics.Queue=root.AppsCompleted +yarn.QueueMetrics.Queue=root.AppsFailed +yarn.QueueMetrics.Queue=root.AppsKilled +yarn.QueueMetrics.Queue=root.AppsPending +yarn.QueueMetrics.Queue=root.AppsRunning +yarn.QueueMetrics.Queue=root.AppsSubmitted +yarn.QueueMetrics.Queue=root.AvailableMB +yarn.QueueMetrics.Queue=root.AvailableVCores +yarn.QueueMetrics.Queue=root.PendingContainers +yarn.QueueMetrics.Queue=root.PendingMB +yarn.QueueMetrics.Queue=root.UsedAMResourceMB +yarn.QueueMetrics.Queue=root.default.ReservedContainers +yarn.QueueMetrics.Queue=root.default.ReservedMB +yarn.QueueMetrics.Queue=root.running_0 +yarn.QueueMetrics.Queue=root.running_1440 +yarn.QueueMetrics.Queue=root.running_300 +yarn.QueueMetrics.Queue=root.running_60 +yarn.TimelineDataManagerMetrics.GetEntitiesTimeAvgTime +yarn.TimelineDataManagerMetrics.GetEntitiesTotal +yarn.TimelineDataManagerMetrics.PostEntitiesTimeAvgTime +yarn.TimelineDataManagerMetrics.PostEntitiesTotal \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-timelineservice/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/pom.xml b/ambari-metrics/ambari-metrics-timelineservice/pom.xml index 6896c6b..84290b2 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/pom.xml +++ b/ambari-metrics/ambari-metrics-timelineservice/pom.xml @@ -208,6 +208,9 @@ <location>conf/unix/log4j.properties</location> </source> <source> + <location>conf/unix/metrics_whitelist</location> + </source> + <source> <location>target/embedded/${hbase.folder}/conf/hbase-site.xml</location> </source> </sources> @@ -244,6 +247,24 @@ </configuration> </plugin> <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes> + <exclude>conf/unix/metrics_whitelist</exclude> + <exclude>conf/windows/metrics_whitelist</exclude> + </excludes> + </configuration> + <executions> + <execution> + <phase>test</phase> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> <groupId>org.vafer</groupId> <artifactId>jdeb</artifactId> <version>1.0.1</version> http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java index 5dc3aee..ccce3ff 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java @@ -31,7 +31,6 @@ import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; import org.apache.hadoop.metrics2.sink.timeline.TopNConfig; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineMetricAggregator; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.TimelineMetricAggregatorFactory; @@ -56,13 +55,11 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_WHITELIST_FILE; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.USE_GROUPBY_AGGREGATOR_QUERIES; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.DEFAULT_TOPN_HOSTS_LIMIT; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.availability.AggregationTaskRunner.ACTUAL_AGGREGATOR_NAMES; @@ -116,10 +113,8 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin "initialize HA controller", e); } - String whitelistFile = metricsConf.get(TIMELINE_METRICS_WHITELIST_FILE, ""); - if (!StringUtils.isEmpty(whitelistFile)) { - AggregatorUtils.populateMetricWhitelistFromFile(whitelistFile); - } + //Initialize whitelisting & blacklisting if needed + TimelineMetricsFilter.initializeMetricFilter(metricsConf); defaultTopNHostsLimit = Integer.parseInt(metricsConf.get(DEFAULT_TOPN_HOSTS_LIMIT, "20")); if (Boolean.parseBoolean(metricsConf.get(USE_GROUPBY_AGGREGATOR_QUERIES, "true"))) { @@ -358,13 +353,19 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin } @Override - public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata() throws SQLException, IOException { + public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata(String query) throws SQLException, IOException { Map<TimelineMetricMetadataKey, TimelineMetricMetadata> metadata = metricMetadataManager.getMetadataCache(); + boolean includeBlacklistedMetrics = StringUtils.isNotEmpty(query) && "all".equalsIgnoreCase(query); + // Group Metadata by AppId Map<String, List<TimelineMetricMetadata>> metadataByAppId = new HashMap<>(); for (TimelineMetricMetadata metricMetadata : metadata.values()) { + + if (!includeBlacklistedMetrics && !metricMetadata.isWhitelisted()) { + continue; + } List<TimelineMetricMetadata> metadataList = metadataByAppId.get(metricMetadata.getAppId()); if (metadataList == null) { metadataList = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java index 1174554..c1fa328 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; import com.google.common.collect.Multimap; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -113,6 +112,7 @@ import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.ti import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_METRICS_CLUSTER_AGGREGATE_GROUPED_TABLE_SQL; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_METRICS_CLUSTER_AGGREGATE_TABLE_SQL; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_METRICS_METADATA_TABLE_SQL; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.ALTER_METRICS_METADATA_TABLE; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_METRICS_TABLE_SQL; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.DEFAULT_ENCODING; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.DEFAULT_TABLE_COMPRESSION; @@ -415,6 +415,8 @@ public class PhoenixHBaseAccessor { String metadataSql = String.format(CREATE_METRICS_METADATA_TABLE_SQL, encoding, compression); stmt.executeUpdate(metadataSql); + stmt.executeUpdate(ALTER_METRICS_METADATA_TABLE); + String hostedAppSql = String.format(CREATE_HOSTED_APPS_METADATA_TABLE_SQL, encoding, compression); stmt.executeUpdate(hostedAppSql); @@ -730,20 +732,19 @@ public class PhoenixHBaseAccessor { TimelineMetric tm = iterator.next(); - if (CollectionUtils.isNotEmpty(AggregatorUtils.whitelistedMetrics) && - !AggregatorUtils.whitelistedMetrics.contains(tm.getMetricName())) { - iterator.remove(); - continue; - } + boolean acceptMetric = TimelineMetricsFilter.acceptMetric(tm); // Write to metadata cache on successful write to store if (metadataManager != null) { metadataManager.putIfModifiedTimelineMetricMetadata( - metadataManager.getTimelineMetricMetadata(tm)); + metadataManager.getTimelineMetricMetadata(tm, acceptMetric)); metadataManager.putIfModifiedHostedAppsMetadata( tm.getHostName(), tm.getAppId()); } + if (!acceptMetric) { + iterator.remove(); + } } if (!skipCache && cacheEnabled) { @@ -1537,6 +1538,7 @@ public class PhoenixHBaseAccessor { stmt.setString(4, metadata.getType()); stmt.setLong(5, metadata.getSeriesStartTime()); stmt.setBoolean(6, metadata.isSupportsAggregates()); + stmt.setBoolean(7, metadata.isWhitelisted()); try { stmt.executeUpdate(); @@ -1629,7 +1631,8 @@ public class PhoenixHBaseAccessor { rs.getString("UNITS"), rs.getString("TYPE"), rs.getLong("START_TIME"), - rs.getBoolean("SUPPORTS_AGGREGATION") + rs.getBoolean("SUPPORTS_AGGREGATION"), + rs.getBoolean("IS_WHITELISTED") ); TimelineMetricMetadataKey key = new TimelineMetricMetadataKey(metricName, appId); http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java index 13d361b..92d595e 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java @@ -249,6 +249,9 @@ public class TimelineMetricConfiguration { public static final String TIMELINE_METRIC_METADATA_FILTERS = "timeline.metrics.service.metadata.filters"; + public static final String TIMELINE_METRICS_APPS_BLACKLIST = + "timeline.metrics.apps.blacklist"; + public static final String HBASE_BLOCKING_STORE_FILES = "hbase.hstore.blockingStoreFiles"; http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java index 57186f5..d049e33 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStore.java @@ -78,7 +78,7 @@ public interface TimelineMetricStore { * @throws SQLException * @throws IOException */ - Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata() throws SQLException, IOException; + Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata(String query) throws SQLException, IOException; /** * Returns all hosts that have written metrics with the apps on the host http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java new file mode 100644 index 0000000..d05353d --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilter.java @@ -0,0 +1,153 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_WHITELIST_FILE; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_APPS_BLACKLIST; + +public class TimelineMetricsFilter { + + private static Set<String> whitelistedMetrics; + private static Set<Pattern> whitelistedMetricPatterns; + private static boolean metricWhitelistingEnabled; + private static boolean appBlacklistingEnabled; + private static Set<String> blacklistedApps; + private static String patternPrefix = "._p_"; + + private static final Log LOG = LogFactory.getLog(TimelineMetricsFilter.class); + + public static void initializeMetricFilter(Configuration metricsConf) { + + whitelistedMetrics = new HashSet<String>(); + whitelistedMetricPatterns = new HashSet<Pattern>(); + blacklistedApps = new HashSet<>(); + metricWhitelistingEnabled = false; + appBlacklistingEnabled = false; + + String whitelistFile = metricsConf.get(TIMELINE_METRICS_WHITELIST_FILE, ""); + if (!StringUtils.isEmpty(whitelistFile)) { + metricWhitelistingEnabled = true; + readMetricWhitelistFromFile(whitelistFile); + } + + String appsBlacklist = metricsConf.get(TIMELINE_METRICS_APPS_BLACKLIST, ""); + if (!StringUtils.isEmpty(appsBlacklist)) { + appBlacklistingEnabled = true; + for (String app : appsBlacklist.split(",")) { + blacklistedApps.add(app); + } + LOG.info("Blacklisted apps : " + blacklistedApps.toString()); + } + } + + private static void readMetricWhitelistFromFile(String whitelistFile) { + + BufferedReader br = null; + String strLine; + + try(FileInputStream fstream = new FileInputStream(whitelistFile)) { + br = new BufferedReader(new InputStreamReader(fstream)); + + while ((strLine = br.readLine()) != null) { + strLine = strLine.trim(); + if (StringUtils.isEmpty(strLine)) { + continue; + } + if (strLine.startsWith(patternPrefix)) { + whitelistedMetricPatterns.add(Pattern.compile(strLine.substring(patternPrefix.length()))); + } else { + whitelistedMetrics.add(strLine); + } + } + } catch (IOException ioEx) { + LOG.error("Unable to parse metric whitelist file", ioEx); + } + + LOG.info("Whitelisting " + whitelistedMetrics.size() + " metrics"); + LOG.debug("Whitelisted metrics : " + Arrays.toString(whitelistedMetrics.toArray())); + } + + public static boolean acceptMetric(String metricName, String appId) { + TimelineMetric timelineMetric = new TimelineMetric(); + timelineMetric.setAppId(appId); + timelineMetric.setMetricName(metricName); + return acceptMetric(timelineMetric); + } + + public static boolean acceptMetric(TimelineMetric metric) { + + if (!isEnabled()) { + return true; + } + + // App takes precedence. + if (blacklistedApps.contains(metric.getAppId())) { + return false; + } + + if (!metricWhitelistingEnabled) { + return true; + } + + if (whitelistedMetrics.contains(metric.getMetricName())) { + return true; + } + + for (Pattern p : whitelistedMetricPatterns) { + Matcher m = p.matcher(metric.getMetricName()); + if (m.find()) { + whitelistedMetrics.add(metric.getMetricName()); + return true; + } + } + + return false; + } + + public static void addToWhitelist(String metricName) { + + if (StringUtils.isNotEmpty(metricName)) { + if (!metricWhitelistingEnabled) { + metricWhitelistingEnabled = true; + } + + whitelistedMetrics.add(metricName); + } + } + + public static boolean isEnabled() { + return appBlacklistingEnabled || metricWhitelistingEnabled; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AggregatorUtils.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AggregatorUtils.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AggregatorUtils.java index 55b1c1b..20f72c6 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AggregatorUtils.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AggregatorUtils.java @@ -18,15 +18,7 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators; -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.Arrays; -import java.util.HashSet; import java.util.Map; -import java.util.Set; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -35,7 +27,6 @@ import org.apache.commons.logging.LogFactory; */ public class AggregatorUtils { - public static Set<String> whitelistedMetrics = new HashSet<String>(); private static final Log LOG = LogFactory.getLog(AggregatorUtils.class); public static double[] calculateAggregates(Map<Long, Double> metricValues) { @@ -68,40 +59,4 @@ public class AggregatorUtils { return values; } - - public static void populateMetricWhitelistFromFile(String whitelistFile) { - - FileInputStream fstream = null; - BufferedReader br = null; - String strLine; - - try { - fstream = new FileInputStream(whitelistFile); - br = new BufferedReader(new InputStreamReader(fstream)); - - while ((strLine = br.readLine()) != null) { - strLine = strLine.trim(); - whitelistedMetrics.add(strLine); - } - } catch (IOException ioEx) { - LOG.error("Unable to parse metric whitelist file", ioEx); - } finally { - if (br != null) { - try { - br.close(); - } catch (IOException e) { - } - } - - if (fstream != null) { - try { - fstream.close(); - } catch (IOException e) { - } - } - } - LOG.info("Whitelisting " + whitelistedMetrics.size() + " metrics"); - LOG.debug("Whitelisted metrics : " + Arrays.toString(whitelistedMetrics.toArray())); - } - } http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java index 0d6a09e..d7b0d55 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricAppAggregator.java @@ -21,12 +21,11 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata; +import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricsFilter; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataKey; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.discovery.TimelineMetricMetadataManager; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -141,7 +140,7 @@ public class TimelineMetricAppAggregator { if (hostMetricMetadata != null) { TimelineMetricMetadata timelineMetricMetadata = new TimelineMetricMetadata(clusterMetric.getMetricName(), appId, hostMetricMetadata.getUnits(), hostMetricMetadata.getType(), hostMetricMetadata.getSeriesStartTime(), - hostMetricMetadata.isSupportsAggregates()); + hostMetricMetadata.isSupportsAggregates(), TimelineMetricsFilter.acceptMetric(clusterMetric.getMetricName(), appId)); metadataManagerInstance.putIfModifiedTimelineMetricMetadata(timelineMetricMetadata); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java index d0d1dbf..7eb2457 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TimelineMetricMetadataManager.java @@ -197,14 +197,15 @@ public class TimelineMetricMetadataManager { hBaseAccessor.saveHostAppsMetadata(hostedApps); } - public TimelineMetricMetadata getTimelineMetricMetadata(TimelineMetric timelineMetric) { + public TimelineMetricMetadata getTimelineMetricMetadata(TimelineMetric timelineMetric, boolean isWhitelisted) { return new TimelineMetricMetadata( timelineMetric.getMetricName(), timelineMetric.getAppId(), timelineMetric.getUnits(), timelineMetric.getType(), timelineMetric.getStartTime(), - supportAggregates(timelineMetric) + supportAggregates(timelineMetric), + isWhitelisted ); } http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java index bc218f2..0c8e5a7 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java @@ -137,7 +137,8 @@ public class PhoenixTransactSQL { "UNITS CHAR(20), " + "TYPE CHAR(20), " + "START_TIME UNSIGNED_LONG, " + - "SUPPORTS_AGGREGATION BOOLEAN " + + "SUPPORTS_AGGREGATION BOOLEAN, " + + "IS_WHITELISTED BOOLEAN " + "CONSTRAINT pk PRIMARY KEY (METRIC_NAME, APP_ID)) " + "DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'"; @@ -147,6 +148,9 @@ public class PhoenixTransactSQL { "CONSTRAINT pk PRIMARY KEY (HOSTNAME))" + "DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'"; + public static final String ALTER_METRICS_METADATA_TABLE = + "ALTER TABLE METRICS_METADATA ADD IF NOT EXISTS IS_WHITELISTED BOOLEAN"; + /** * ALTER table to set new options */ @@ -220,8 +224,8 @@ public class PhoenixTransactSQL { public static final String UPSERT_METADATA_SQL = "UPSERT INTO METRICS_METADATA (METRIC_NAME, APP_ID, UNITS, TYPE, " + - "START_TIME, SUPPORTS_AGGREGATION) " + - "VALUES (?, ?, ?, ?, ?, ?)"; + "START_TIME, SUPPORTS_AGGREGATION, IS_WHITELISTED) " + + "VALUES (?, ?, ?, ?, ?, ?, ?)"; public static final String UPSERT_HOSTED_APPS_METADATA_SQL = "UPSERT INTO HOSTED_APPS_METADATA (HOSTNAME, APP_IDS) VALUES (?, ?)"; @@ -300,7 +304,7 @@ public class PhoenixTransactSQL { public static final String GET_METRIC_METADATA_SQL = "SELECT " + "METRIC_NAME, APP_ID, UNITS, TYPE, START_TIME, " + - "SUPPORTS_AGGREGATION FROM METRICS_METADATA"; + "SUPPORTS_AGGREGATION, IS_WHITELISTED FROM METRICS_METADATA"; public static final String GET_HOSTED_APPS_METADATA_SQL = "SELECT " + "HOSTNAME, APP_IDS FROM HOSTED_APPS_METADATA"; http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java index 6b15e29..304a8e0 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java @@ -384,12 +384,13 @@ public class TimelineWebServices { @Produces({ MediaType.APPLICATION_JSON }) public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata( @Context HttpServletRequest req, - @Context HttpServletResponse res - ) { + @Context HttpServletResponse res, + @QueryParam("query") String query + ) { init(res); try { - return timelineMetricStore.getTimelineMetricMetadata(); + return timelineMetricStore.getTimelineMetricMetadata(query); } catch (Exception e) { throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR); } http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java index 2e2d3a8..b2e8cac 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestTimelineMetricStore.java @@ -87,7 +87,7 @@ public class TestTimelineMetricStore implements TimelineMetricStore { } @Override - public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata() throws SQLException, IOException { + public Map<String, List<TimelineMetricMetadata>> getTimelineMetricMetadata(String query) throws SQLException, IOException { return null; } http://git-wip-us.apache.org/repos/asf/ambari/blob/9e524d7e/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java new file mode 100644 index 0000000..049d473 --- /dev/null +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsFilterTest.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; + +import junit.framework.Assert; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; +import org.junit.Test; +import java.net.URL; + +public class TimelineMetricsFilterTest { + + @Test + public void testAppBlacklisting() { + + Configuration metricsConf = new Configuration(); + metricsConf.set("timeline.metrics.apps.blacklist", "hbase,datanode,nimbus"); + TimelineMetricsFilter.initializeMetricFilter(metricsConf); + + TimelineMetric timelineMetric = new TimelineMetric(); + + timelineMetric.setAppId("hbase"); + Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric)); + + timelineMetric.setAppId("namenode"); + Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric)); + + timelineMetric.setAppId("nimbus"); + Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric)); + } + + @Test + public void testMetricWhitelisting() throws Exception { + + Configuration metricsConf = new Configuration(); + URL fileUrl = ClassLoader.getSystemResource("test_data/metric_whitelist.dat"); + + metricsConf.set("timeline.metrics.whitelist.file", fileUrl.getPath()); + TimelineMetricsFilter.initializeMetricFilter(metricsConf); + + TimelineMetric timelineMetric = new TimelineMetric(); + + timelineMetric.setMetricName("cpu_system"); + Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric)); + + timelineMetric.setMetricName("cpu_system1"); + Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric)); + + timelineMetric.setMetricName("jvm.JvmMetrics.MemHeapUsedM"); + Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric)); + + timelineMetric.setMetricName("dfs.FSNamesystem.TotalFiles"); + Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric)); + } + + @Test + public void testTogether() throws Exception { + + Configuration metricsConf = new Configuration(); + metricsConf.set("timeline.metrics.apps.blacklist", "hbase,datanode,nimbus"); + + URL fileUrl = ClassLoader.getSystemResource("test_data/metric_whitelist.dat"); + metricsConf.set("timeline.metrics.whitelist.file", fileUrl.getPath()); + + TimelineMetricsFilter.initializeMetricFilter(metricsConf); + + TimelineMetric timelineMetric = new TimelineMetric(); + + timelineMetric.setMetricName("cpu_system"); + timelineMetric.setAppId("hbase"); + Assert.assertFalse(TimelineMetricsFilter.acceptMetric(timelineMetric)); + + timelineMetric.setMetricName("cpu_system"); + timelineMetric.setAppId("HOST"); + Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric)); + + timelineMetric.setMetricName("jvm.JvmMetrics.MemHeapUsedM"); + Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric)); + + timelineMetric.setMetricName("dfs.FSNamesystem.TotalFiles"); + Assert.assertTrue(TimelineMetricsFilter.acceptMetric(timelineMetric)); + } + +}