Repository: ambari Updated Branches: refs/heads/trunk 6f00dd8a1 -> 7f753b444
AMBARI-10809. AMS: navigating graph time ranges are not correct. (swagle) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7f753b44 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7f753b44 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7f753b44 Branch: refs/heads/trunk Commit: 7f753b4448e4b370c100d663e8a4f4fd1f0ec5b0 Parents: 6f00dd8 Author: Siddharth Wagle <swa...@hortonworks.com> Authored: Wed Apr 29 11:34:31 2015 -0700 Committer: Siddharth Wagle <swa...@hortonworks.com> Committed: Wed Apr 29 11:34:31 2015 -0700 ---------------------------------------------------------------------- .../metrics/MetricsPaddingMethod.java | 106 ++++++++++ .../metrics/MetricsPropertyProvider.java | 17 +- .../metrics/timeline/AMSPropertyProvider.java | 4 +- .../timeline/MetricsPaddingMethodTest.java | 208 +++++++++++++++++++ 4 files changed, 332 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7f753b44/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java new file mode 100644 index 0000000..a78beee --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPaddingMethod.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.controller.metrics; + +import org.apache.ambari.server.controller.spi.TemporalInfo; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; +import java.util.Iterator; +import java.util.TreeMap; + +public class MetricsPaddingMethod { + private final PADDING_STRATEGY strategy; + public static final String ZERO_PADDING_PARAM = "params/padding"; + public static enum PADDING_STRATEGY { + ZEROS, + NULLS, + NONE + } + + public MetricsPaddingMethod(PADDING_STRATEGY strategy) { + this.strategy = strategy; + } + + /** + * Adds zero/null values towards the end of metrics sequence as well as the + * beginning to support backward compatibility with Ganglia. + * We use step if only a single datapoint is found. + * It is assumed that @TimelineMetric.metricsValues are sorted in ascending + * order and thee is no padding between the interval. + * + * @param metric AMS @TimelineMetric + * @param temporalInfo @TemporalInfo Requested interval + */ + public void applyPaddingStrategy(TimelineMetric metric, TemporalInfo temporalInfo) { + if (strategy.equals(PADDING_STRATEGY.NONE) || temporalInfo == null) { + return; + } + + // TODO: JSON dser returns LinkedHashMap that is not Navigable + TreeMap<Long, Double> values = new TreeMap<Long, Double>(metric.getMetricValues()); + + long dataInterval = getTimelineMetricInterval(values); + + if (dataInterval == -1) { + dataInterval = temporalInfo.getStep() != null ? temporalInfo.getStep() : -1; + } + // Unable to determine what interval to use for padding + if (dataInterval == -1) { + return; + } + + long intervalStartTime = longToMillis(temporalInfo.getStartTime()); + long intervalEndTime = longToMillis(temporalInfo.getEndTime()); + long dataStartTime = longToMillis(values.firstKey()); + long dataEndTime = longToMillis(values.lastKey()); + + Double paddingValue = 0.0d; + + if (strategy.equals(PADDING_STRATEGY.NULLS)) { + paddingValue = null; + } + // Pad before data interval + for (long counter = intervalStartTime; counter < dataStartTime; counter += dataInterval) { + // Until counter approaches or goes past dataStartTime : pad + values.put(counter, paddingValue); + } + // Pad after data interval + for (long counter = dataEndTime + dataInterval; counter <= intervalEndTime; counter += dataInterval) { + values.put(counter, paddingValue); + } + // Put back new + old values + metric.setMetricValues(values); + } + + private long longToMillis(long time) { + if (time < 9999999999l) { + return time * 1000; + } + return time; + } + + private long getTimelineMetricInterval(TreeMap<Long, Double> values) { + if (values != null && values.size() > 1) { + Iterator<Long> tsValuesIterator = values.descendingKeySet().iterator(); + long lastValue = tsValuesIterator.next(); + long secondToLastValue = tsValuesIterator.next(); + return Math.abs(lastValue - secondToLastValue); + } + // No values found or only one value found + return -1; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/7f753b44/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProvider.java index d58d110..8360f5e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProvider.java @@ -24,14 +24,16 @@ import org.apache.ambari.server.controller.spi.Predicate; import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.spi.SystemException; +import org.apache.ambari.server.controller.utilities.PredicateHelper; import org.apache.ambari.server.controller.utilities.StreamProvider; -import org.apache.http.client.utils.URIBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Map; import java.util.Set; import java.util.regex.Pattern; +import static org.apache.ambari.server.controller.metrics.MetricsPaddingMethod.ZERO_PADDING_PARAM; + public abstract class MetricsPropertyProvider extends AbstractPropertyProvider { protected final static Logger LOG = LoggerFactory.getLogger(MetricsPropertyProvider.class); @@ -50,6 +52,9 @@ public abstract class MetricsPropertyProvider extends AbstractPropertyProvider { protected final ComponentSSLConfiguration configuration; + protected MetricsPaddingMethod metricsPaddingMethod = + new MetricsPaddingMethod(MetricsPaddingMethod.PADDING_STRATEGY.ZEROS); + protected MetricsPropertyProvider(Map<String, Map<String, PropertyInfo>> componentPropertyInfoMap, StreamProvider streamProvider, @@ -124,6 +129,16 @@ public abstract class MetricsPropertyProvider extends AbstractPropertyProvider { return resources; } + Map<String, Object> predicateProperties = PredicateHelper.getProperties(predicate); + if (predicateProperties != null && predicateProperties.keySet().contains(ZERO_PADDING_PARAM)) { + String paddingStr = (String) predicateProperties.get(ZERO_PADDING_PARAM); + for (MetricsPaddingMethod.PADDING_STRATEGY strategy : MetricsPaddingMethod.PADDING_STRATEGY.values()) { + if (paddingStr.equalsIgnoreCase(strategy.name())) { + metricsPaddingMethod = new MetricsPaddingMethod(strategy); + } + } + } + return populateResourcesWithProperties(resources, request, ids); } http://git-wip-us.apache.org/repos/asf/ambari/blob/7f753b44/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java index 7f2b227..c05e112 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/AMSPropertyProvider.java @@ -35,7 +35,6 @@ import org.codehaus.jackson.map.AnnotationIntrospector; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.ObjectReader; import org.codehaus.jackson.xc.JaxbAnnotationIntrospector; - import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; @@ -51,7 +50,6 @@ import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; - import static org.apache.ambari.server.Role.HBASE_MASTER; import static org.apache.ambari.server.Role.HBASE_REGIONSERVER; import static org.apache.ambari.server.Role.METRICS_COLLECTOR; @@ -261,6 +259,8 @@ public abstract class AMSPropertyProvider extends MetricsPropertyProvider { if (metric.getMetricName() != null && metric.getMetricValues() != null && checkMetricName(patterns, metric.getMetricName())) { + // Pad zeros or nulls if needed + metricsPaddingMethod.applyPaddingStrategy(metric, temporalInfo); populateResource(resource, metric); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/7f753b44/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java new file mode 100644 index 0000000..7665e5e --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/MetricsPaddingMethodTest.java @@ -0,0 +1,208 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.controller.metrics.timeline; + +import junit.framework.Assert; +import org.apache.ambari.server.controller.metrics.MetricsPaddingMethod; +import org.apache.ambari.server.controller.spi.TemporalInfo; +import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; +import org.junit.Test; +import java.util.Map; +import java.util.TreeMap; + +public class MetricsPaddingMethodTest { + + @Test + public void testPaddingWithNulls() throws Exception { + MetricsPaddingMethod paddingMethod = + new MetricsPaddingMethod(MetricsPaddingMethod.PADDING_STRATEGY.NULLS); + + long now = System.currentTimeMillis(); + + TimelineMetric timelineMetric = new TimelineMetric(); + timelineMetric.setMetricName("m1"); + timelineMetric.setHostName("h1"); + timelineMetric.setAppId("a1"); + timelineMetric.setTimestamp(now); + TreeMap<Long, Double> inputValues = new TreeMap<Long, Double>(); + inputValues.put(now - 1000, 1.0d); + inputValues.put(now - 2000, 2.0d); + inputValues.put(now - 3000, 3.0d); + timelineMetric.setMetricValues(inputValues); + + TemporalInfo temporalInfo = getTemporalInfo(now - 10000, now, 1l); + paddingMethod.applyPaddingStrategy(timelineMetric, temporalInfo); + TreeMap<Long, Double> values = (TreeMap<Long, Double>) timelineMetric.getMetricValues(); + + Assert.assertEquals(11, values.size()); + Assert.assertEquals(new Long(now - 10000), values.keySet().iterator().next()); + Assert.assertEquals(new Long(now), values.descendingKeySet().iterator().next()); + Assert.assertEquals(null, values.values().iterator().next()); + } + + @Test + public void testPaddingWithZeros() throws Exception { + MetricsPaddingMethod paddingMethod = + new MetricsPaddingMethod(MetricsPaddingMethod.PADDING_STRATEGY.ZEROS); + + long now = System.currentTimeMillis(); + + TimelineMetric timelineMetric = new TimelineMetric(); + timelineMetric.setMetricName("m1"); + timelineMetric.setHostName("h1"); + timelineMetric.setAppId("a1"); + timelineMetric.setTimestamp(now); + TreeMap<Long, Double> inputValues = new TreeMap<Long, Double>(); + inputValues.put(now - 1000, 1.0d); + inputValues.put(now - 2000, 2.0d); + inputValues.put(now - 3000, 3.0d); + timelineMetric.setMetricValues(inputValues); + + TemporalInfo temporalInfo = getTemporalInfo(now - 10000, now, 1l); + paddingMethod.applyPaddingStrategy(timelineMetric, temporalInfo); + TreeMap<Long, Double> values = (TreeMap<Long, Double>) timelineMetric.getMetricValues(); + + Assert.assertEquals(11, values.size()); + Assert.assertEquals(new Long(now - 10000), values.keySet().iterator().next()); + Assert.assertEquals(new Long(now), values.descendingKeySet().iterator().next()); + Assert.assertEquals(0.0, values.values().iterator().next()); + } + + @Test + public void testPaddingWithNoPaddingNeeded() throws Exception { + MetricsPaddingMethod paddingMethod = + new MetricsPaddingMethod(MetricsPaddingMethod.PADDING_STRATEGY.ZEROS); + + long now = System.currentTimeMillis(); + + TimelineMetric timelineMetric = new TimelineMetric(); + timelineMetric.setMetricName("m1"); + timelineMetric.setHostName("h1"); + timelineMetric.setAppId("a1"); + timelineMetric.setTimestamp(now); + TreeMap<Long, Double> inputValues = new TreeMap<Long, Double>(); + inputValues.put(now, 0.0d); + inputValues.put(now - 1000, 1.0d); + inputValues.put(now - 2000, 2.0d); + inputValues.put(now - 3000, 3.0d); + timelineMetric.setMetricValues(inputValues); + + TemporalInfo temporalInfo = getTemporalInfo(now - 3000, now, 1l); + paddingMethod.applyPaddingStrategy(timelineMetric, temporalInfo); + TreeMap<Long, Double> values = (TreeMap<Long, Double>) timelineMetric.getMetricValues(); + + Assert.assertEquals(4, values.size()); + Assert.assertEquals(new Long(now - 3000), values.keySet().iterator().next()); + Assert.assertEquals(new Long(now), values.descendingKeySet().iterator().next()); + } + + @Test + public void testPaddingWithStepProvided() throws Exception { + MetricsPaddingMethod paddingMethod = + new MetricsPaddingMethod(MetricsPaddingMethod.PADDING_STRATEGY.ZEROS); + + long now = System.currentTimeMillis(); + + TimelineMetric timelineMetric = new TimelineMetric(); + timelineMetric.setMetricName("m1"); + timelineMetric.setHostName("h1"); + timelineMetric.setAppId("a1"); + timelineMetric.setTimestamp(now); + TreeMap<Long, Double> inputValues = new TreeMap<Long, Double>(); + inputValues.put(now - 1000, 1.0d); + timelineMetric.setMetricValues(inputValues); + + TemporalInfo temporalInfo = getTemporalInfo(now - 10000, now, 1000l); + paddingMethod.applyPaddingStrategy(timelineMetric, temporalInfo); + TreeMap<Long, Double> values = (TreeMap<Long, Double>) timelineMetric.getMetricValues(); + + Assert.assertEquals(11, values.size()); + Assert.assertEquals(new Long(now - 10000), values.keySet().iterator().next()); + Assert.assertEquals(new Long(now), values.descendingKeySet().iterator().next()); + Assert.assertEquals(0.0, values.values().iterator().next()); + } + + @Test + public void testPaddingWithOneValueReturnedNoStepProvided() throws Exception { + MetricsPaddingMethod paddingMethod = + new MetricsPaddingMethod(MetricsPaddingMethod.PADDING_STRATEGY.ZEROS); + + long now = System.currentTimeMillis(); + + TimelineMetric timelineMetric = new TimelineMetric(); + timelineMetric.setMetricName("m1"); + timelineMetric.setHostName("h1"); + timelineMetric.setAppId("a1"); + timelineMetric.setTimestamp(now); + TreeMap<Long, Double> inputValues = new TreeMap<Long, Double>(); + inputValues.put(now - 1000, 1.0d); + timelineMetric.setMetricValues(inputValues); + + TemporalInfo temporalInfo = getTemporalInfo(now - 10000, now, null); + paddingMethod.applyPaddingStrategy(timelineMetric, temporalInfo); + TreeMap<Long, Double> values = (TreeMap<Long, Double>) timelineMetric.getMetricValues(); + + Assert.assertEquals(1, values.size()); + Assert.assertEquals(new Long(now - 1000), values.keySet().iterator().next()); + Assert.assertEquals(1.0, values.values().iterator().next()); + } + + @Test + public void testNoPaddingRequested() throws Exception { + MetricsPaddingMethod paddingMethod = + new MetricsPaddingMethod(MetricsPaddingMethod.PADDING_STRATEGY.NONE); + + long now = System.currentTimeMillis(); + + TimelineMetric timelineMetric = new TimelineMetric(); + timelineMetric.setMetricName("m1"); + timelineMetric.setHostName("h1"); + timelineMetric.setAppId("a1"); + timelineMetric.setTimestamp(now); + Map<Long, Double> inputValues = new TreeMap<Long, Double>(); + inputValues.put(now - 100, 1.0d); + inputValues.put(now - 200, 2.0d); + inputValues.put(now - 300, 3.0d); + timelineMetric.setMetricValues(inputValues); + + TemporalInfo temporalInfo = getTemporalInfo(now - 1000, now, 10l); + paddingMethod.applyPaddingStrategy(timelineMetric, temporalInfo); + TreeMap<Long, Double> values = (TreeMap<Long, Double>) timelineMetric.getMetricValues(); + + Assert.assertEquals(3, values.size()); + } + + private TemporalInfo getTemporalInfo(final Long startTime, final Long endTime, final Long step) { + return new TemporalInfo() { + @Override + public Long getStartTime() { + return startTime; + } + + @Override + public Long getEndTime() { + return endTime; + } + + @Override + public Long getStep() { + return step; + } + }; + } +}
