This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ambari-metrics.git
The following commit(s) were added to refs/heads/master by this push:
new 9f800da [AMBARI-25077] Grafana on a workload cluster should show
visualization of only it's own cluster. (#12)
9f800da is described below
commit 9f800dac83fb6761e4627a7030e839642148224c
Author: avijayanhwx <[email protected]>
AuthorDate: Wed Jan 2 10:39:46 2019 -0800
[AMBARI-25077] Grafana on a workload cluster should show visualization of
only it's own cluster. (#12)
---
.../ambari-metrics/datasource.js | 30 +++++-----
.../core/timeline/HBaseTimelineMetricsService.java | 5 +-
.../core/timeline/PhoenixHBaseAccessor.java | 10 ++--
.../core/timeline/TimelineMetricConfiguration.java | 10 ++++
.../core/timeline/query/PhoenixTransactSQL.java | 2 +-
.../ambari/metrics/webapp/TimelineWebServices.java | 68 +++++++++++++++++++++-
6 files changed, 102 insertions(+), 23 deletions(-)
diff --git a/ambari-metrics-grafana/ambari-metrics/datasource.js
b/ambari-metrics-grafana/ambari-metrics/datasource.js
index dd64566..18e2709 100644
--- a/ambari-metrics-grafana/ambari-metrics/datasource.js
+++ b/ambari-metrics-grafana/ambari-metrics/datasource.js
@@ -59,7 +59,7 @@ define([
//We get a list of components and their associated metrics.
this.initMetricAppidMapping = function () {
- return this.doAmbariRequest({url: '/ws/v1/timeline/metrics/metadata'})
+ return this.doAmbariRequest({url: '/metadata'})
.then(function (items) {
items = items.data;
allMetrics = {};
@@ -227,7 +227,7 @@ define([
var metricTransform = !target.transform || target.transform ===
"none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' + target.metric +
metricTransform +
+ url: '?metricNames=' + target.metric + metricTransform +
metricAggregator + "&hostname=" + target.hosts + '&appId=' +
target.app + instanceId + '&startTime=' + from +
'&endTime=' + to + precision + seriesAggregator
}).then(
@@ -268,7 +268,7 @@ define([
var metricTransform = !target.transform || target.transform ===
"none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' + target.metric +
metricTransform
+ url: '?metricNames=' + target.metric + metricTransform
+ metricAggregator + '&hostname=' + tHost + '&appId=' + target.app
+ instanceId + '&startTime=' + from +
'&endTime=' + to + precision + seriesAggregator
}).then(
@@ -302,7 +302,7 @@ define([
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
var templatedComponent = (_.isEmpty(tComponent)) ? target.app :
tComponent;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' + target.metric +
metricTransform
+ url: '?metricNames=' + target.metric + metricTransform
+ metricAggregator + '&hostname=' + target.templatedHost +
'&appId=' + templatedComponent + instanceId
+ '&startTime=' + from + '&endTime=' + to + precision + topN +
seriesAggregator
}).then(
@@ -317,7 +317,7 @@ define([
var metricTransform = !target.transform || target.transform ===
"none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' +
encodeURIComponent(target.queue) + metricTransform
+ url: '?metricNames=' + encodeURIComponent(target.queue) +
metricTransform
+ metricAggregator + '&appId=resourcemanager' + instanceId +
'&startTime=' + from +
'&endTime=' + to + precision + seriesAggregator
}).then(
@@ -330,7 +330,7 @@ define([
+ target.precision;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' + target.hbMetric +
instanceId + '&appId=hbase&startTime='
+ url: '?metricNames=' + target.hbMetric + instanceId +
'&appId=hbase&startTime='
+ from + '&endTime=' + to + precision + seriesAggregator
}).then(
allHostMetricsData(target)
@@ -345,7 +345,7 @@ define([
var metricTransform = !target.transform || target.transform ===
"none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' + target.kbMetric +
metricTransform + instanceId
+ url: '?metricNames=' + target.kbMetric + metricTransform +
instanceId
+ metricAggregator + '&appId=kafka_broker&startTime=' + from +
'&endTime=' + to + precision + seriesAggregator
}).then(
@@ -361,7 +361,7 @@ define([
var metricTransform = !target.transform || target.transform ===
"none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' + target.nnMetric +
metricTransform + instanceId
+ url: '?metricNames=' + target.nnMetric + metricTransform +
instanceId
+ metricAggregator + '&appId=namenode&startTime=' + from +
'&endTime=' + to + precision + seriesAggregator
}).then(
allHostMetricsData(target)
@@ -377,7 +377,7 @@ define([
var metricTransform = !target.transform || target.transform ===
"none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' + target.sTopoMetric +
metricTransform + instanceId
+ url: '?metricNames=' + target.sTopoMetric + metricTransform +
instanceId
+ metricAggregator + '&appId=nimbus&startTime=' + from +
'&endTime=' + to + precision + seriesAggregator
}).then(
allHostMetricsData(target)
@@ -393,7 +393,7 @@ define([
var metricTransform = !target.transform || target.transform ===
"none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' + target.sCoreMetric +
metricTransform + instanceId
+ url: '?metricNames=' + target.sCoreMetric + metricTransform +
instanceId
+ metricAggregator + '&appId=ambari-infra-solr&startTime=' + from
+ '&endTime=' + to + precision + seriesAggregator
}).then(
allHostMetricsData(target)
@@ -408,7 +408,7 @@ define([
var metricTransform = !target.transform || target.transform ===
"none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' +
target.sCollectionMetric + metricTransform + instanceId
+ url: '?metricNames=' + target.sCollectionMetric + metricTransform
+ instanceId
+ metricAggregator + '&appId=ambari-infra-solr&startTime=' + from
+ '&endTime=' + to + precision + seriesAggregator
}).then(
allHostMetricsData(target)
@@ -424,7 +424,7 @@ define([
var metricTransform = !target.transform || target.transform ===
"none" ? '' : '._' + target.transform;
var seriesAggregator = !target.seriesAggregator ||
target.seriesAggregator === "none" ? '' : '&seriesAggregateFunction=' +
target.seriesAggregator;
return self.doAmbariRequest({
- url: '/ws/v1/timeline/metrics?metricNames=' +
target.sDataSourceMetric + metricTransform + instanceId
+ url: '?metricNames=' + target.sDataSourceMetric + metricTransform
+ instanceId
+ metricAggregator + '&appId=druid&startTime=' + from +
'&endTime=' + to + precision + seriesAggregator
}).then(
allHostMetricsData(target)
@@ -1111,7 +1111,7 @@ define([
*/
this.testDatasource = function () {
return this.doAmbariRequest({
- url: '/ws/v1/timeline/metrics/metadata',
+ url: '/metadata',
method: 'GET'
}).then(function (response) {
console.log(response);
@@ -1161,7 +1161,7 @@ define([
}
return this.doAmbariRequest({
method: 'GET',
- url: '/ws/v1/timeline/metrics/instances?' + 'appId=' + app
+ url: '/instance?' + 'appId=' + app
}).then(function (response) {
var clusters = [];
var data = response.data;
@@ -1188,7 +1188,7 @@ define([
}
return this.doAmbariRequest({
method: 'GET',
- url: '/ws/v1/timeline/metrics/instances?' + 'appId=' + app +
'&instanceId=' + cluster
+ url: '/instance?' + 'appId=' + app + '&instanceId=' + cluster
}).then(function (response) {
var hosts = [];
var data = response.data;
diff --git
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
index c90c93b..3619af6 100644
---
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
+++
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
@@ -17,7 +17,6 @@
*/
package org.apache.ambari.metrics.core.timeline;
-import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DEFAULT_INSTANCE_ID;
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.USE_GROUPBY_AGGREGATOR_QUERIES;
import static
org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.ACTUAL_AGGREGATOR_NAMES;
@@ -87,6 +86,7 @@ public class HBaseTimelineMetricsService extends
AbstractService implements Time
private TimelineMetricMetadataManager metricMetadataManager;
private MetricCollectorHAController haController;
private boolean containerMetricsDisabled = false;
+ private String defaultInstanceId = "";
/**
* Construct the service.
@@ -218,6 +218,7 @@ public class HBaseTimelineMetricsService extends
AbstractService implements Time
"delay = " + initDelay + ", delay = " + delay);
}
containerMetricsDisabled = configuration.isContainerMetricsDisabled();
+ defaultInstanceId = configuration.getDefaultInstanceId();
isInitialized = true;
}
@@ -258,7 +259,7 @@ public class HBaseTimelineMetricsService extends
AbstractService implements Time
List<String> transientMetricNames = new ArrayList<>();
if (configuration.getTimelineMetricsMultipleClusterSupport() &&
StringUtils.isEmpty(instanceId)) {
- instanceId = DEFAULT_INSTANCE_ID;
+ instanceId = this.defaultInstanceId;
}
List<byte[]> uuids =
metricMetadataManager.getUuidsForGetMetricQuery(metricFunctions.keySet(),
diff --git
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
index b6ff202..6f1e1c0 100644
---
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
+++
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
@@ -29,7 +29,6 @@ import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguratio
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_SECOND_TABLE_TTL;
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CONTAINER_METRICS_TTL;
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DATE_TIERED_COMPACTION_POLICY;
-import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DEFAULT_INSTANCE_ID;
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.FIFO_COMPACTION_POLICY_CLASS;
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.GLOBAL_MAX_RETRIES;
import static
org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.GLOBAL_RESULT_LIMIT;
@@ -203,6 +202,7 @@ public class PhoenixHBaseAccessor {
private TimelineMetricMetadataManager metadataManagerInstance;
private Set<String> eventMetricPatterns = new HashSet<>();
private boolean supportMultipleClusterMetrics = false;
+ private String defaultInstanceId = "";
private Map<String, Integer> tableTTL = new HashMap<>();
@@ -263,6 +263,7 @@ public class PhoenixHBaseAccessor {
tableTTL.put(METRIC_TRANSIENT_TABLE_NAME,
metricsConf.getInt(METRICS_TRANSIENT_TABLE_TTL, 7 * 86400)); //7 days
this.supportMultipleClusterMetrics =
Boolean.valueOf(metricsConf.get(TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS,
"false"));
+ this.defaultInstanceId = configuration.getDefaultInstanceId();
if (cacheEnabled) {
LOG.debug("Initialising and starting metrics cache committer thread...");
@@ -903,6 +904,10 @@ public class PhoenixHBaseAccessor {
boolean acceptMetric = TimelineMetricsFilter.acceptMetric(tm);
+ if (supportMultipleClusterMetrics &&
StringUtils.isEmpty(tm.getInstanceId())) {
+ tm.setInstanceId(defaultInstanceId);
+ }
+
// Write to metadata cache on successful write to store
if (metadataManager != null) {
metadataManager.putIfModifiedTimelineMetricMetadata(
@@ -912,9 +917,6 @@ public class PhoenixHBaseAccessor {
tm.getHostName(), tm.getAppId());
if (!tm.getAppId().equals("FLUME_HANDLER")) {
- if (supportMultipleClusterMetrics &&
StringUtils.isEmpty(tm.getInstanceId())) {
- tm.setInstanceId(DEFAULT_INSTANCE_ID);
- }
metadataManager.putIfModifiedHostedInstanceMetadata(tm.getInstanceId(),
tm.getHostName());
}
}
diff --git
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
index 9766f0c..f8f4bb0 100644
---
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
+++
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
@@ -292,6 +292,9 @@ public class TimelineMetricConfiguration {
public static final String TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS =
"timeline.metrics.support.multiple.clusters";
+ public static final String TIMELINE_METRICS_DEFAULT_INSTANCE_ID =
+ "timeline.metrics.default.instanceid";
+
public static final String TIMELINE_METRICS_EVENT_METRIC_PATTERNS =
"timeline.metrics.downsampler.event.metric.patterns";
@@ -512,6 +515,13 @@ public class TimelineMetricConfiguration {
return false;
}
+ public String getDefaultInstanceId() {
+ if (metricsConf != null) {
+ return metricsConf.get(TIMELINE_METRICS_DEFAULT_INSTANCE_ID,
DEFAULT_INSTANCE_ID);
+ }
+ return DEFAULT_INSTANCE_ID;
+ }
+
public String getTimelineServiceRpcAddress() {
String defaultRpcAddress = "0.0.0.0:60200";
if (metricsConf != null) {
diff --git
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
index ee50051..092da51 100644
---
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
+++
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
@@ -154,7 +154,7 @@ public class PhoenixTransactSQL {
"START_TIME UNSIGNED_LONG, " +
"SUPPORTS_AGGREGATION BOOLEAN, " +
"IS_WHITELISTED BOOLEAN " +
- "CONSTRAINT pk PRIMARY KEY (METRIC_NAME, APP_ID)) " +
+ "CONSTRAINT pk PRIMARY KEY (METRIC_NAME, APP_ID, INSTANCE_ID)) " +
"DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'";
public static final String CREATE_HOSTED_APPS_METADATA_TABLE_SQL =
diff --git
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java
index b3378be..71477ec 100644
---
a/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java
+++
b/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java
@@ -36,6 +36,7 @@ import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
@@ -288,6 +289,31 @@ public class TimelineWebServices {
}
@GET
+ @Path("metrics/{instanceId}")
+ @Produces({ MediaType.APPLICATION_JSON })
+ public TimelineMetrics getTimelineMetricsForInstance(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("instanceId") String instanceId,
+ @QueryParam("metricNames") String metricNames,
+ @QueryParam("appId") String appId,
+ @QueryParam("hostname") String hostname,
+ @QueryParam("startTime") String startTime,
+ @QueryParam("endTime") String endTime,
+ @QueryParam("precision") String precision,
+ @QueryParam("limit") String limit,
+ @QueryParam("grouped") String grouped,
+ @QueryParam("topN") String topN,
+ @QueryParam("topNFunction") String topNFunction,
+ @QueryParam("isBottomN") String isBottomN,
+ @QueryParam("seriesAggregateFunction") String seriesAggregateFunction
+ ) {
+
+ return getTimelineMetrics(req, res, metricNames, appId, instanceId,
hostname, startTime, endTime, precision, limit,
+ grouped, topN, topNFunction, isBottomN, seriesAggregateFunction);
+ }
+
+ @GET
@Path("/metrics/summary")
@Produces({ MediaType.APPLICATION_JSON })
public TimelineMetricServiceSummary getTimelineMetricServiceSummary(
@@ -325,6 +351,28 @@ public class TimelineWebServices {
}
@GET
+ @Path("/metrics/{instanceId}/metadata")
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Map<String, List<TimelineMetricMetadata>>
getTimelineMetricMetadataForInstance(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @QueryParam("appId") String appId,
+ @QueryParam("metricName") String metricPattern,
+ @QueryParam("includeAll") String includeBlacklistedMetrics
+ ) {
+ init(res);
+
+ try {
+ return timelineMetricStore.getTimelineMetricMetadata(
+ parseStr(appId),
+ parseStr(metricPattern),
+ parseBoolean(includeBlacklistedMetrics));
+ } catch (Exception e) {
+ throw new WebApplicationException(e,
Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ @GET
@Path("/metrics/hosts")
@Produces({ MediaType.APPLICATION_JSON })
public Map<String, Set<String>> getHostedAppsMetadata(
@@ -341,7 +389,7 @@ public class TimelineWebServices {
}
@GET
- @Path("/metrics/instances")
+ @Path("/metrics/instance")
@Produces({ MediaType.APPLICATION_JSON })
public Map<String, Map<String, Set<String>>> getClusterHostsMetadata(
@Context HttpServletRequest req,
@@ -359,6 +407,24 @@ public class TimelineWebServices {
}
@GET
+ @Path("/metrics/{instanceId}/instance")
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Map<String, Map<String, Set<String>>>
getClusterHostsMetadataForInstance(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @QueryParam("appId") String appId,
+ @PathParam("instanceId") String instanceId
+ ) {
+ init(res);
+
+ try {
+ return timelineMetricStore.getInstanceHostsMetadata(instanceId, appId);
+ } catch (Exception e) {
+ throw new WebApplicationException(e,
Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ @GET
@Path("/metrics/uuid")
@Produces({ MediaType.APPLICATION_JSON })
public byte[] getUuid(
