Repository: ambari Updated Branches: refs/heads/branch-3.0-ams 8d1c24568 -> 82e622947
AMBARI-21079. Add ability to sink Raw metrics to external system via Http. Compilation error fix. (swagle) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/82e62294 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/82e62294 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/82e62294 Branch: refs/heads/branch-3.0-ams Commit: 82e62294723eeeac04ebc841d617a8e9db2bc459 Parents: 8d1c245 Author: Siddharth Wagle <swa...@hortonworks.com> Authored: Thu Jun 8 16:14:24 2017 -0700 Committer: Siddharth Wagle <swa...@hortonworks.com> Committed: Thu Jun 8 16:14:24 2017 -0700 ---------------------------------------------------------------------- ambari-metrics/ambari-metrics-common/pom.xml | 2 +- .../TimelineMetricsEhCacheSizeOfEngine.java | 22 ++++++ .../cache/InternalMetricsCacheSizeOfEngine.java | 71 ++++++++------------ .../cache/TimelineMetricsCacheSizeOfEngine.java | 17 +---- 4 files changed, 53 insertions(+), 59 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/82e62294/ambari-metrics/ambari-metrics-common/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/pom.xml b/ambari-metrics/ambari-metrics-common/pom.xml index dc2ab5e..ebd0fc9 100644 --- a/ambari-metrics/ambari-metrics-common/pom.xml +++ b/ambari-metrics/ambari-metrics-common/pom.xml @@ -74,7 +74,7 @@ </relocation> <relocation> <pattern>org.apache.commons.io</pattern> - <shadedPattern>org.apache.ambari.metrics.sink.relocated.commons.io</shadedPattern>StormTimelineMetricsReporter + <shadedPattern>org.apache.ambari.metrics.sink.relocated.commons.io</shadedPattern> </relocation> <relocation> <pattern>org.apache.commons.lang</pattern> http://git-wip-us.apache.org/repos/asf/ambari/blob/82e62294/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsEhCacheSizeOfEngine.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsEhCacheSizeOfEngine.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsEhCacheSizeOfEngine.java index ea694b7..0e23e17 100644 --- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsEhCacheSizeOfEngine.java +++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsEhCacheSizeOfEngine.java @@ -24,6 +24,9 @@ import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +import net.sf.ehcache.Element; +import net.sf.ehcache.pool.Size; import net.sf.ehcache.pool.SizeOfEngine; import net.sf.ehcache.pool.impl.DefaultSizeOfEngine; import net.sf.ehcache.pool.sizeof.ReflectionSizeOf; @@ -51,6 +54,7 @@ public abstract class TimelineMetricsEhCacheSizeOfEngine implements SizeOfEngine // Map entry sizing private long sizeOfMapEntry; private long sizeOfMapEntryOverhead; + private long sizeOfElement; protected TimelineMetricsEhCacheSizeOfEngine(SizeOfEngine underlying) { this.underlying = underlying; @@ -62,6 +66,8 @@ public abstract class TimelineMetricsEhCacheSizeOfEngine implements SizeOfEngine this.sizeOfMapEntry = reflectionSizeOf.sizeOf(new Long(1)) + reflectionSizeOf.sizeOf(new Double(2.0)); + this.sizeOfElement = reflectionSizeOf.sizeOf(new Element(new Object(), new Object())); + //SizeOfMapEntryOverhead = SizeOfMapWithOneEntry - (SizeOfEmptyMap + SizeOfOneEntry) TreeMap<Long, Double> map = new TreeMap<>(); long emptyMapSize = reflectionSizeOf.sizeOf(map); @@ -112,4 +118,20 @@ public abstract class TimelineMetricsEhCacheSizeOfEngine implements SizeOfEngine } return size; } + + // Get size of the Cache entry for final size calculation + protected abstract long getSizeOfEntry(Object key, Object value); + + @Override + public Size sizeOf(Object key, Object value, Object container) { + return new Size(sizeOfElement + getSizeOfEntry(key, value), false); + } + + @Override + public SizeOfEngine copyWith(int maxDepth, boolean abortWhenMaxDepthExceeded) { + LOG.debug("Copying tracing sizeof engine, maxdepth: {}, abort: {}", maxDepth, abortWhenMaxDepthExceeded); + + return underlying.copyWith(maxDepth, abortWhenMaxDepthExceeded); + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/82e62294/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCacheSizeOfEngine.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCacheSizeOfEngine.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCacheSizeOfEngine.java index 071dcd4..e36c981 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCacheSizeOfEngine.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCacheSizeOfEngine.java @@ -20,48 +20,33 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline import org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsEhCacheSizeOfEngine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import net.sf.ehcache.pool.Size; -import net.sf.ehcache.pool.SizeOfEngine; -public class InternalMetricsCacheSizeOfEngine { -// extends TimelineMetricsEhCacheSizeOfEngine { -// private final static Logger LOG = LoggerFactory.getLogger(InternalMetricsCacheSizeOfEngine.class); -// -// private InternalMetricsCacheSizeOfEngine(SizeOfEngine underlying) { -// super(underlying); -// } -// -// public InternalMetricsCacheSizeOfEngine() { -// // Invoke default constructor in base class -// } -// -// @Override -// public Size sizeOf(Object key, Object value, Object container) { -// try { -// LOG.debug("BEGIN - Sizeof, key: {}, value: {}", key, value); -// long size = 0; -// if (key instanceof InternalMetricCacheKey) { -// InternalMetricCacheKey metricCacheKey = (InternalMetricCacheKey) key; -// size += reflectionSizeOf.sizeOf(metricCacheKey.getMetricName()); -// size += reflectionSizeOf.sizeOf(metricCacheKey.getAppId()); -// size += reflectionSizeOf.sizeOf(metricCacheKey.getInstanceId()); // null safe -// size += reflectionSizeOf.sizeOf(metricCacheKey.getHostname()); -// } -// if (value instanceof InternalMetricCacheValue) { -// size += getValueMapSize(((InternalMetricCacheValue) value).getMetricValues()); -// } -// // Mark size as not being exact -// return new Size(size, false); -// } finally { -// LOG.debug("END - Sizeof, key: {}", key); -// } -// } -// -// @Override -// public SizeOfEngine copyWith(int maxDepth, boolean abortWhenMaxDepthExceeded) { -// LOG.debug("Copying tracing sizeof engine, maxdepth: {}, abort: {}", -// maxDepth, abortWhenMaxDepthExceeded); -// -// return new InternalMetricsCacheSizeOfEngine(underlying.copyWith(maxDepth, abortWhenMaxDepthExceeded)); -// } +public class InternalMetricsCacheSizeOfEngine extends TimelineMetricsEhCacheSizeOfEngine { + private final static Logger LOG = LoggerFactory.getLogger(InternalMetricsCacheSizeOfEngine.class); + + public InternalMetricsCacheSizeOfEngine() { + // Invoke default constructor in base class + } + + @Override + protected long getSizeOfEntry(Object key, Object value) { + try { + LOG.debug("BEGIN - Sizeof, key: {}, value: {}", key, value); + long size = 0; + if (key instanceof InternalMetricCacheKey) { + InternalMetricCacheKey metricCacheKey = (InternalMetricCacheKey) key; + size += reflectionSizeOf.sizeOf(metricCacheKey.getMetricName()); + size += reflectionSizeOf.sizeOf(metricCacheKey.getAppId()); + size += reflectionSizeOf.sizeOf(metricCacheKey.getInstanceId()); // null safe + size += reflectionSizeOf.sizeOf(metricCacheKey.getHostname()); + } + if (value instanceof InternalMetricCacheValue) { + size += getValueMapSize(((InternalMetricCacheValue) value).getMetricValues()); + } + // Mark size as not being exact + return size; + } finally { + LOG.debug("END - Sizeof, key: {}", key); + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/82e62294/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java index fccf190..45c8967 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricsCacheSizeOfEngine.java @@ -21,9 +21,6 @@ import org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsEhCacheSize import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import net.sf.ehcache.pool.Size; -import net.sf.ehcache.pool.SizeOfEngine; - /** * Cache sizing engine that reduces reflective calls over the Object graph to * find total Heap usage. @@ -32,16 +29,12 @@ public class TimelineMetricsCacheSizeOfEngine extends TimelineMetricsEhCacheSize private final static Logger LOG = LoggerFactory.getLogger(TimelineMetricsCacheSizeOfEngine.class); - private TimelineMetricsCacheSizeOfEngine(SizeOfEngine underlying) { - super(underlying); - } - public TimelineMetricsCacheSizeOfEngine() { // Invoke default constructor in base class } @Override - public Size sizeOf(Object key, Object value, Object container) { + public long getSizeOfEntry(Object key, Object value) { try { LOG.debug("BEGIN - Sizeof, key: {}, value: {}", key, value); @@ -55,7 +48,7 @@ public class TimelineMetricsCacheSizeOfEngine extends TimelineMetricsEhCacheSize size += getTimelineMetricCacheValueSize((TimelineMetricsCacheValue) value); } // Mark size as not being exact - return new Size(size, false); + return size; } finally { LOG.debug("END - Sizeof, key: {}", key); } @@ -86,11 +79,5 @@ public class TimelineMetricsCacheSizeOfEngine extends TimelineMetricsEhCacheSize return size; } - @Override - public SizeOfEngine copyWith(int maxDepth, boolean abortWhenMaxDepthExceeded) { - LOG.debug("Copying tracing sizeof engine, maxdepth: {}, abort: {}", - maxDepth, abortWhenMaxDepthExceeded); - return new TimelineMetricsCacheSizeOfEngine(underlying.copyWith(maxDepth, abortWhenMaxDepthExceeded)); - } }