Repository: ambari Updated Branches: refs/heads/trunk 7b822e42b -> 34c1e9b2c
AMBARI-10987. Error loading deferred resources when making metric API requests with null padding. (swagle) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/34c1e9b2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/34c1e9b2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/34c1e9b2 Branch: refs/heads/trunk Commit: 34c1e9b2c7726a92d18de68d5b292cb87e564740 Parents: 7b822e4 Author: Siddharth Wagle <swa...@hortonworks.com> Authored: Thu May 7 10:34:33 2015 -0700 Committer: Siddharth Wagle <swa...@hortonworks.com> Committed: Thu May 7 10:37:50 2015 -0700 ---------------------------------------------------------------------- .../metrics/timeline/PhoenixHBaseAccessor.java | 2 + .../query/render/MetricsPaddingRenderer.java | 46 +++++++++++++ .../api/resources/BaseResourceDefinition.java | 5 ++ .../MetricsDownsamplingMethodFactory.java | 14 ++-- .../metrics/MetricsPropertyProvider.java | 20 ++++-- .../metrics/MetricsPropertyProviderProxy.java | 13 ++++ .../render/MetricsPaddingRendererTest.java | 69 ++++++++++++++++++++ 7 files changed, 155 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/34c1e9b2/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java index e27d9a9..7258cad 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java @@ -309,6 +309,8 @@ public class PhoenixHBaseAccessor { clusterDailyTtl)); conn.commit(); + + LOG.info("Metrics schema initialized."); } catch (SQLException sql) { if (sql.getErrorCode() == SQLExceptionCode.SET_UNSUPPORTED_PROP_ON_ALTER_TABLE.getErrorCode()) { http://git-wip-us.apache.org/repos/asf/ambari/blob/34c1e9b2/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/MetricsPaddingRenderer.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/MetricsPaddingRenderer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/MetricsPaddingRenderer.java new file mode 100644 index 0000000..ff776c7 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/MetricsPaddingRenderer.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.api.query.render; + +import org.apache.ambari.server.api.query.QueryInfo; +import org.apache.ambari.server.api.util.TreeNode; +import java.util.Set; + +import static org.apache.ambari.server.controller.metrics.MetricsPaddingMethod.PADDING_STRATEGY; + +public class MetricsPaddingRenderer extends DefaultRenderer { + PADDING_STRATEGY paddingMethod = PADDING_STRATEGY.ZEROS; + + public MetricsPaddingRenderer(String paddingMethod) { + if (paddingMethod.equalsIgnoreCase("null_padding")) { + this.paddingMethod = PADDING_STRATEGY.NULLS; + } else if (paddingMethod.equalsIgnoreCase("no_padding")) { + this.paddingMethod = PADDING_STRATEGY.NONE; + } + } + + @Override + public TreeNode<Set<String>> finalizeProperties(TreeNode<QueryInfo> queryProperties, + boolean isCollection) { + Set<String> properties = queryProperties.getObject().getProperties(); + if (properties != null) { + properties.add("params/padding/" + paddingMethod.name()); + } + return super.finalizeProperties(queryProperties, isCollection); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/34c1e9b2/ambari-server/src/main/java/org/apache/ambari/server/api/resources/BaseResourceDefinition.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/BaseResourceDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/BaseResourceDefinition.java index e63f7eb..c8bf87d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/BaseResourceDefinition.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/BaseResourceDefinition.java @@ -20,6 +20,7 @@ package org.apache.ambari.server.api.resources; import org.apache.ambari.server.api.query.render.DefaultRenderer; +import org.apache.ambari.server.api.query.render.MetricsPaddingRenderer; import org.apache.ambari.server.api.query.render.MinimalRenderer; import org.apache.ambari.server.api.query.render.Renderer; import org.apache.ambari.server.api.services.Request; @@ -125,6 +126,10 @@ public abstract class BaseResourceDefinition implements ResourceDefinition { return new DefaultRenderer(); } else if (name.equals("minimal")) { return new MinimalRenderer(); + } else if (name.contains("null_padding") + || name.contains("no_padding") + || name.contains("zero_padding")) { + return new MetricsPaddingRenderer(name); } else { throw new IllegalArgumentException("Invalid renderer name: " + name + " for resource of type: " + m_type); http://git-wip-us.apache.org/repos/asf/ambari/blob/34c1e9b2/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDownsamplingMethodFactory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDownsamplingMethodFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDownsamplingMethodFactory.java index a53e26d..2d22643 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDownsamplingMethodFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsDownsamplingMethodFactory.java @@ -18,7 +18,6 @@ package org.apache.ambari.server.controller.metrics; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; - import java.util.ArrayList; import java.util.Iterator; import java.util.Map; @@ -64,9 +63,9 @@ class MetricNoDownsampling extends MetricsDownsamplingMethod { class MetricsAveragePerSecondDownsampling extends MetricsDownsamplingMethod { class Accumulo { public long ts; - public double val; + public Double val; - public Accumulo(long t, double v) { + public Accumulo(long t, Double v) { this.ts = t; this.val = v; } @@ -79,7 +78,7 @@ class MetricsAveragePerSecondDownsampling extends MetricsDownsamplingMethod { if (ci.hasNext()) { Map.Entry<Long, Double> e0 = ci.next(); long t0 = e0.getKey() / 1000; - double s0 = e0.getValue(); + Double s0 = e0.getValue(); int nSamples = 1; while(ci.hasNext()) { @@ -87,18 +86,19 @@ class MetricsAveragePerSecondDownsampling extends MetricsDownsamplingMethod { long t = e0.getKey() / 1000; if (t != t0) { - cache.add(new Accumulo(t0, dataTransferMethod.getData(s0 / nSamples))); + cache.add(new Accumulo(t0, s0 != null ? dataTransferMethod.getData(s0 / nSamples) : null)); t0 = t; s0 = e0.getValue(); nSamples = 1; } else { - s0 += e0.getValue(); + // Zero value contributes to nothing for downsampling method + s0 += e0.getValue() != null ? e0.getValue() : 0; nSamples++; } } //Add the last entry into the cache - cache.add(new Accumulo(t0, dataTransferMethod.getData(s0 / nSamples))); + cache.add(new Accumulo(t0, s0 != null ? dataTransferMethod.getData(s0 / nSamples) : null)); } Number[][] datapointsArray = new Number[cache.size()][2]; http://git-wip-us.apache.org/repos/asf/ambari/blob/34c1e9b2/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProvider.java index 8360f5e..569bb17 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProvider.java @@ -52,7 +52,9 @@ public abstract class MetricsPropertyProvider extends AbstractPropertyProvider { protected final ComponentSSLConfiguration configuration; - protected MetricsPaddingMethod metricsPaddingMethod = + protected MetricsPaddingMethod metricsPaddingMethod; + + private static final MetricsPaddingMethod DEFAULT_PADDING_METHOD = new MetricsPaddingMethod(MetricsPaddingMethod.PADDING_STRATEGY.ZEROS); protected MetricsPropertyProvider(Map<String, Map<String, @@ -129,12 +131,16 @@ public abstract class MetricsPropertyProvider extends AbstractPropertyProvider { return resources; } - Map<String, Object> predicateProperties = PredicateHelper.getProperties(predicate); - if (predicateProperties != null && predicateProperties.keySet().contains(ZERO_PADDING_PARAM)) { - String paddingStr = (String) predicateProperties.get(ZERO_PADDING_PARAM); - for (MetricsPaddingMethod.PADDING_STRATEGY strategy : MetricsPaddingMethod.PADDING_STRATEGY.values()) { - if (paddingStr.equalsIgnoreCase(strategy.name())) { - metricsPaddingMethod = new MetricsPaddingMethod(strategy); + // Re-initialize in case of reuse. + metricsPaddingMethod = DEFAULT_PADDING_METHOD; + + Set<String> requestPropertyIds = request.getPropertyIds(); + if (requestPropertyIds != null && !requestPropertyIds.isEmpty()) { + for (String propertyId : requestPropertyIds) { + if (propertyId.startsWith(ZERO_PADDING_PARAM)) { + String paddingStrategyStr = propertyId.substring(ZERO_PADDING_PARAM.length() + 1); + metricsPaddingMethod = new MetricsPaddingMethod( + MetricsPaddingMethod.PADDING_STRATEGY.valueOf(paddingStrategyStr)); } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/34c1e9b2/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProviderProxy.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProviderProxy.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProviderProxy.java index 24f1851..7fa9c5e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProviderProxy.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/MetricsPropertyProviderProxy.java @@ -35,6 +35,8 @@ import org.apache.ambari.server.controller.spi.SystemException; import org.apache.ambari.server.controller.utilities.StreamProvider; import java.util.Map; import java.util.Set; + +import static org.apache.ambari.server.controller.metrics.MetricsPaddingMethod.ZERO_PADDING_PARAM; import static org.apache.ambari.server.controller.metrics.MetricsServiceProvider.MetricsService; import static org.apache.ambari.server.controller.metrics.MetricsServiceProvider.MetricsService.GANGLIA; import static org.apache.ambari.server.controller.metrics.MetricsServiceProvider.MetricsService.TIMELINE_METRICS; @@ -81,6 +83,17 @@ public class MetricsPropertyProviderProxy extends AbstractPropertyProvider { } } + @Override + public Set<String> checkPropertyIds(Set<String> propertyIds) { + Set<String> checkedPropertyIds = super.checkPropertyIds(propertyIds); + for (String propertyId : checkedPropertyIds) { + if (propertyId.startsWith(ZERO_PADDING_PARAM)) { + checkedPropertyIds.remove(propertyId); + } + } + return checkedPropertyIds; + } + private void createHostPropertyProviders(Map<String, Map<String, PropertyInfo>> componentPropertyInfoMap, StreamProvider streamProvider, ComponentSSLConfiguration configuration, http://git-wip-us.apache.org/repos/asf/ambari/blob/34c1e9b2/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/MetricsPaddingRendererTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/MetricsPaddingRendererTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/MetricsPaddingRendererTest.java new file mode 100644 index 0000000..dafc605 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/MetricsPaddingRendererTest.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.api.query.render; + +import org.apache.ambari.server.api.query.QueryInfo; +import org.apache.ambari.server.api.resources.ServiceResourceDefinition; +import org.apache.ambari.server.api.util.TreeNode; +import org.apache.ambari.server.api.util.TreeNodeImpl; +import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.controller.spi.Schema; +import org.apache.ambari.server.controller.spi.SchemaFactory; +import org.junit.Test; +import java.util.HashSet; +import java.util.Set; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class MetricsPaddingRendererTest { + @Test + public void testFinalizeProperties__NullPadding_property() { + SchemaFactory schemaFactory = createNiceMock(SchemaFactory.class); + Schema schema = createNiceMock(Schema.class); + + // schema expectations + expect(schemaFactory.getSchema(Resource.Type.Service)).andReturn(schema).anyTimes(); + expect(schema.getKeyPropertyId(Resource.Type.Service)).andReturn("ServiceInfo/service_name").anyTimes(); + expect(schema.getKeyPropertyId(Resource.Type.Cluster)).andReturn("ServiceInfo/cluster_name").anyTimes(); + + replay(schemaFactory, schema); + + HashSet<String> serviceProperties = new HashSet<String>(); + serviceProperties.add("foo/bar"); + QueryInfo rootQuery = new QueryInfo(new ServiceResourceDefinition(), serviceProperties); + TreeNode<QueryInfo> queryTree = new TreeNodeImpl<QueryInfo>(null, rootQuery, "Service"); + + MetricsPaddingRenderer renderer = new MetricsPaddingRenderer("null_padding"); + renderer.init(schemaFactory); + TreeNode<Set<String>> propertyTree = renderer.finalizeProperties(queryTree, false); + + assertEquals(4, propertyTree.getObject().size()); + assertTrue(propertyTree.getObject().contains("ServiceInfo/service_name")); + assertTrue(propertyTree.getObject().contains("ServiceInfo/cluster_name")); + assertTrue(propertyTree.getObject().contains("foo/bar")); + assertTrue(propertyTree.getObject().contains("params/padding/NULLS")); + assertEquals(0, propertyTree.getChildren().size()); + + + verify(schemaFactory, schema); + } +}