This is an automated email from the ASF dual-hosted git repository. wuzhiguo pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ambari-metrics.git
The following commit(s) were added to refs/heads/master by this push: new 535d6fa AMBARI-25563. Storm dashboards are not showing metrics. (#63) 535d6fa is described below commit 535d6fac7c19566299ed01345acd1b5046def464 Author: lucasbak <lucas.bakal...@gmail.com> AuthorDate: Tue Nov 15 10:28:13 2022 +0100 AMBARI-25563. Storm dashboards are not showing metrics. (#63) --- .../ambari-metrics/datasource.js | 88 +++++++++++++--------- .../core/timeline/PhoenixHBaseAccessor.java | 18 ++++- .../timeline/query/TransientMetricCondition.java | 20 +++-- .../timeline/uuid/HashBasedUuidGenStrategy.java | 6 +- .../core/timeline/uuid/MD5UuidGenStrategy.java | 6 +- .../uuid/MetricUuidGenNullRestrictedStrategy.java | 71 +++++++++++++++++ .../timeline/uuid/Murmur3HashUuidGenStrategy.java | 6 +- .../timeline/uuid/MetricUuidGenStrategyTest.java | 14 ++++ 8 files changed, 174 insertions(+), 55 deletions(-) diff --git a/ambari-metrics-grafana/ambari-metrics/datasource.js b/ambari-metrics-grafana/ambari-metrics/datasource.js index 76b85e3..f46fe00 100644 --- a/ambari-metrics-grafana/ambari-metrics/datasource.js +++ b/ambari-metrics-grafana/ambari-metrics/datasource.js @@ -654,35 +654,47 @@ define([ //Templatized Dashboards for Storm Components if (templateSrv.variables[0].query === "topologies" && templateSrv.variables[1] && templateSrv.variables[1].name === "component") { - if (templateSrv._values) { - var selectedTopology = templateSrv._values.topologies; - var selectedComponent = templateSrv._values.component; + + var allTopologies = templateSrv.variables.filter(function(variable) { + return variable.name === "topologies"; + }); + var allComponents = templateSrv.variables.filter(function(variable) { + return variable.name === "component"; + }); - metricsPromises.push(_.map(options.targets, function(target) { - target.sTopology = selectedTopology; - target.sComponent = selectedComponent; - target.sTopoMetric = target.metric.replace('*', target.sTopology).replace('*', target.sComponent); - return getStormData(target); - })); - } + var selectedTopology = (_.isEmpty(allTopologies)) ? "" : allTopologies[0].current.text; + var selectedComponent = (_.isEmpty(allComponents)) ? "" : allComponents[0].current.text; + + metricsPromises.push(_.map(options.targets, function(target) { + target.sTopology = selectedTopology; + target.sComponent = selectedComponent; + target.sTopoMetric = target.metric.replace('*', target.sTopology).replace('*', target.sComponent); + return getStormData(target); + })); } //Templatized Dashboard for Storm Kafka Offset if (templateSrv.variables[0].query === "topologies" && templateSrv.variables[1] && templateSrv.variables[1].name === "topic") { - if (templateSrv._values) { - var selectedTopology = templateSrv._values.topologies; - var selectedTopic = templateSrv._values.topic; + + var allTopologies = templateSrv.variables.filter(function(variable) { + return variable.name === "topologies"; + }); + var allTopics = templateSrv.variables.filter(function(variable) { + return variable.name === "topic"; + }); - metricsPromises.push(_.map(options.targets, function(target) { - target.sTopology = selectedTopology; - target.sTopic = selectedTopic; - target.sPartition = options.scopedVars.partition.value; - target.sTopoMetric = target.metric.replace('*', target.sTopology).replace('*', target.sTopic) - .replace('*', target.sPartition); - return getStormData(target); - })); - } + var selectedTopology = (_.isEmpty(allTopologies)) ? "" : allTopologies[0].current.text; + var selectedTopic = (_.isEmpty(allTopics)) ? "" : allTopics[0].current.text; + + metricsPromises.push(_.map(options.targets, function(target) { + target.sTopology = selectedTopology; + target.sTopic = selectedTopic; + target.sPartition = options.scopedVars.partition.value; + target.sTopoMetric = target.metric.replace('*', target.sTopology).replace('*', target.sTopic) + .replace('*', target.sPartition); + return getStormData(target); + })); } //Templatized Dashboards for Druid @@ -782,7 +794,7 @@ define([ this.metricFindQuery = function (query) { var interpolated; try { - interpolated = query.split('.')[0]; + interpolated = templateSrv.replace(query); } catch (err) { return $q.reject(err); } @@ -1029,24 +1041,28 @@ define([ if (interpolated.indexOf("stormTopic") >= 0) { var topicName = interpolated.substring(0, interpolated.indexOf('.')); return this.getStormEntities().then(function () { - var topicNames = Object.keys(stormEntities[topicName]); - return _.map(topicNames, function (names) { - return { - text: names - }; - }); + if(!_.isEmpty(stormEntities) && !_.isEmpty(stormEntities[topicName])) { + var topicNames = Object.keys(stormEntities[topicName]); + return _.map(topicNames, function(names){ + return { + text: names + }; + }); + } else return[]; }); } //Templated Variables for Storm Partitions per Topic if (interpolated.indexOf("stormPartition") >= 0) { var topicN, topologyN; return this.getStormEntities().then(function () { - var partitionNames = _.uniq(stormEntities[topologyN][topicN]); - return _.map(partitionNames, function (names) { - return { - text: names - }; - }); + if(!_.isEmpty(stormEntities) && !_.isEmpty(stormEntities[topologyN]) && !_.isEmpty(stormEntities[topologyN][topicN])) { + var partitionNames = _.uniq(stormEntities[topologyN][topicN]); + return _.map(partitionNames, function(names){ + return { + text: names + }; + }); + } else return []; }); } // Templated Variable for YARN Queues. @@ -1131,7 +1147,7 @@ define([ }); } - if (interpolated == 'hosts') { + if (interpolated.indexOf("hosts") >= 0) { return this.suggestHosts(tComponent, templatedCluster); } else if (interpolated == 'cluster') { return this.suggestClusters(tComponent) diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java index 6f1e1c0..cf976dd 100644 --- a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java +++ b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java @@ -1962,7 +1962,7 @@ public class PhoenixHBaseAccessor { TimelineMetricMetadataKey key = new TimelineMetricMetadataKey(metricName, appId, instanceId); metadata.setIsPersisted(true); // Always true on retrieval - metadata.setUuid(rs.getBytes("UUID")); + metadata.setUuid(checkForNull(rs.getBytes("UUID"))); metadataMap.put(key, metadata); } @@ -2018,7 +2018,7 @@ public class PhoenixHBaseAccessor { true ); - metadata.setUuid(rs.getBytes("UUID")); + metadata.setUuid(checkForNull(rs.getBytes("UUID"))); metadataList.add(metadata); } } @@ -2154,4 +2154,18 @@ public class PhoenixHBaseAccessor { this.metadataManagerInstance = metadataManager; TIMELINE_METRIC_READ_HELPER = new TimelineMetricReadHelper(this.metadataManagerInstance); } + + /** + * Null value are being saved to DB as array of zero bytes, so we need to make back converting + */ + private byte[] checkForNull(byte[] uuid) { + if (uuid != null) { + for (byte b : uuid) { + if (b != 0) { + return uuid; + } + } + } + return null; + } } diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/TransientMetricCondition.java b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/TransientMetricCondition.java index b0f526c..9d83300 100644 --- a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/TransientMetricCondition.java +++ b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/TransientMetricCondition.java @@ -60,17 +60,21 @@ public class TransientMetricCondition extends DefaultCondition { appendConjunction = appendHostnameClause(sb, appendConjunction); String appId = getAppId(); - if (appId.contains("%")) { - appendConjunction = append(sb, appendConjunction, getAppId(), " APP_ID LIKE ?"); - } else { - appendConjunction = append(sb, appendConjunction, getAppId(), " APP_ID = ?"); + if (appId != null) { + if (appId.contains("%")) { + appendConjunction = append(sb, appendConjunction, getAppId(), " APP_ID LIKE ?"); + } else { + appendConjunction = append(sb, appendConjunction, getAppId(), " APP_ID = ?"); + } } String instanceId = getInstanceId(); - if (instanceId.contains("%")) { - appendConjunction = append(sb, appendConjunction, getInstanceId(), " INSTANCE_ID LIKE ?"); - } else { - appendConjunction = append(sb, appendConjunction, getInstanceId(), " INSTANCE_ID = ?"); + if (instanceId != null) { + if (instanceId.contains("%")) { + appendConjunction = append(sb, appendConjunction, getInstanceId(), " INSTANCE_ID LIKE ?"); + } else { + appendConjunction = append(sb, appendConjunction, getInstanceId(), " INSTANCE_ID = ?"); + } } appendConjunction = append(sb, appendConjunction, getStartTime(), " SERVER_TIME >= ?"); append(sb, appendConjunction, getEndTime(), " SERVER_TIME < ?"); diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/HashBasedUuidGenStrategy.java b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/HashBasedUuidGenStrategy.java index fdfeb79..278ffd7 100644 --- a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/HashBasedUuidGenStrategy.java +++ b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/HashBasedUuidGenStrategy.java @@ -27,7 +27,7 @@ import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang3.StringUtils; -public class HashBasedUuidGenStrategy implements MetricUuidGenStrategy { +public class HashBasedUuidGenStrategy extends MetricUuidGenNullRestrictedStrategy { /** * Computes the UUID for a timelineClusterMetric. @@ -36,7 +36,7 @@ public class HashBasedUuidGenStrategy implements MetricUuidGenStrategy { * @return byte array of length 'maxlength' */ @Override - public byte[] computeUuid(TimelineClusterMetric timelineClusterMetric, int maxLength) { + protected byte[] computeUuidInternal(TimelineClusterMetric timelineClusterMetric, int maxLength) { int metricNameUuidLength = 12; String instanceId = timelineClusterMetric.getInstanceId(); @@ -173,7 +173,7 @@ public class HashBasedUuidGenStrategy implements MetricUuidGenStrategy { * @return byte array of length 'maxlength' */ @Override - public byte[] computeUuid(String value, int maxLength) { + protected byte[] computeUuidInternal(String value, int maxLength) { if (StringUtils.isEmpty(value)) { return null; diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/MD5UuidGenStrategy.java b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/MD5UuidGenStrategy.java index baa2dce..0e78953 100644 --- a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/MD5UuidGenStrategy.java +++ b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/MD5UuidGenStrategy.java @@ -24,13 +24,13 @@ import java.util.UUID; import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric; import org.apache.commons.lang.StringUtils; -public class MD5UuidGenStrategy implements MetricUuidGenStrategy { +public class MD5UuidGenStrategy extends MetricUuidGenNullRestrictedStrategy { public MD5UuidGenStrategy() { } @Override - public byte[] computeUuid(TimelineClusterMetric timelineClusterMetric, int maxLength) { + protected byte[] computeUuidInternal(TimelineClusterMetric timelineClusterMetric, int maxLength) { String metricString = timelineClusterMetric.getMetricName() + timelineClusterMetric.getAppId(); if (StringUtils.isNotEmpty(timelineClusterMetric.getInstanceId())) { @@ -47,7 +47,7 @@ public class MD5UuidGenStrategy implements MetricUuidGenStrategy { } @Override - public byte[] computeUuid(String value, int maxLength) { + protected byte[] computeUuidInternal(String value, int maxLength) { byte[] valueBytes = value.getBytes(); UUID uuid = UUID.nameUUIDFromBytes(valueBytes); diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenNullRestrictedStrategy.java b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenNullRestrictedStrategy.java new file mode 100644 index 0000000..7a398df --- /dev/null +++ b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenNullRestrictedStrategy.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.metrics.core.timeline.uuid; + +import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * HBase represents null value for BINARY fields as set of zero bytes. This means that we are not able to difference + * byte[]{0,0,0,0} and null values in DB. So we should not generate Uuids which contains only zero bytes. + */ +public abstract class MetricUuidGenNullRestrictedStrategy implements MetricUuidGenStrategy { + private static final Log LOG = LogFactory.getLog(MetricUuidGenNullRestrictedStrategy.class); + + static final int RETRY_NUMBER = 5; + + @Override + public byte[] computeUuid(TimelineClusterMetric timelineClusterMetric, int maxLength) { + int retry_attempt = 0; + byte[] result = null; + while (retry_attempt++ < RETRY_NUMBER) { + result = computeUuidInternal(timelineClusterMetric, maxLength); + for (byte b : result) { + if (b != 0) { + return result; + } + } + LOG.debug("Was generated Uuid which can represent null value in DB."); + } + + LOG.error(String.format("After %n attempts was generated Uuid which can represent null value in DB", RETRY_NUMBER)); + return result; + } + + @Override + public byte[] computeUuid(String value, int maxLength) { + int retry_attempt = 0; + byte[] result = null; + while (retry_attempt++ < RETRY_NUMBER) { + result = computeUuidInternal(value, maxLength); + for (byte b : result) { + if (b != 0) { + return result; + } + } + LOG.debug("Was generated Uuid which can represent null value in DB."); + } + + LOG.error(String.format("After %n attempts was generated Uuid which can represent null value in DB", RETRY_NUMBER)); + return result; + } + + abstract byte[] computeUuidInternal(TimelineClusterMetric timelineClusterMetric, int maxLength); + abstract byte[] computeUuidInternal(String value, int maxLength); +} \ No newline at end of file diff --git a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java index af8cee5..0d5e165 100644 --- a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java +++ b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java @@ -22,7 +22,7 @@ import com.google.common.hash.Hashing; import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric; import org.apache.commons.lang.StringUtils; -public class Murmur3HashUuidGenStrategy implements MetricUuidGenStrategy{ +public class Murmur3HashUuidGenStrategy extends MetricUuidGenNullRestrictedStrategy { /** * Compute Murmur3Hash 16 byte UUID for a Metric-App-Instance. @@ -30,7 +30,7 @@ public class Murmur3HashUuidGenStrategy implements MetricUuidGenStrategy{ * @param maxLength Max length of returned UUID. (Will always be 16 for this technique) * @return 16 byte UUID. */ @Override - public byte[] computeUuid(TimelineClusterMetric timelineClusterMetric, int maxLength) { + byte[] computeUuidInternal(TimelineClusterMetric timelineClusterMetric, int maxLength) { String metricString = timelineClusterMetric.getMetricName() + timelineClusterMetric.getAppId(); if (StringUtils.isNotEmpty(timelineClusterMetric.getInstanceId())) { @@ -47,7 +47,7 @@ public class Murmur3HashUuidGenStrategy implements MetricUuidGenStrategy{ * @return 4 byte UUID. */ @Override - public byte[] computeUuid(String value, int maxLength) { + byte[] computeUuidInternal(String value, int maxLength) { byte[] valueBytes = value.getBytes(); return Hashing.murmur3_32().hashBytes(valueBytes).asBytes(); } diff --git a/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenStrategyTest.java b/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenStrategyTest.java index a25310b..66655b8 100644 --- a/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenStrategyTest.java +++ b/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenStrategyTest.java @@ -19,6 +19,7 @@ package org.apache.ambari.metrics.core.timeline.uuid; import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric; +import org.easymock.EasyMock; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Ignore; @@ -112,6 +113,19 @@ public class MetricUuidGenStrategyTest { public void testMurmur3ConsistentHashing() throws SQLException, InterruptedException { testConsistencyForUuidGenStrategy(new Murmur3HashUuidGenStrategy(), 4); } + + @Test + public void testNotNullCheckForHashing() { + MetricUuidGenNullRestrictedStrategy strategy = EasyMock.createMockBuilder(Murmur3HashUuidGenStrategy.class) + .addMockedMethod("computeUuidInternal", String.class, int.class).createStrictMock(); + + EasyMock.expect(strategy.computeUuidInternal(EasyMock.anyString(), EasyMock.anyInt())) + .andReturn(new byte[]{0,0,0,0}).times(MetricUuidGenNullRestrictedStrategy.RETRY_NUMBER); + + EasyMock.replay(strategy); + strategy.computeUuid("", 0); + EasyMock.verify(strategy); + } private void testMetricCollisionsForUuidGenStrategy(MetricUuidGenStrategy strategy, int uuidLength) { Map<TimelineMetricUuid, TimelineClusterMetric> uuids = new HashMap<>(); --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@ambari.apache.org For additional commands, e-mail: commits-h...@ambari.apache.org