This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new d4403c15aa Upgrade prometheus version, add more labels to
PrometheusEmitter (#12769)
d4403c15aa is described below
commit d4403c15aa8a43c957d6a18ad27812c8fe7dca90
Author: Jianhuan Liu <[email protected]>
AuthorDate: Fri Jul 15 17:13:12 2022 +0800
Upgrade prometheus version, add more labels to PrometheusEmitter (#12769)
Changes:
- Upgrade prometheus to version 0.16.0
- Add optional labels `druid_service` and `host_name` to `PrometheusEmitter`
---
docs/development/extensions-contrib/prometheus.md | 2 +
extensions-contrib/prometheus-emitter/pom.xml | 6 +--
.../apache/druid/emitter/prometheus/Metrics.java | 14 +++++-
.../emitter/prometheus/PrometheusEmitter.java | 19 +++++++-
.../prometheus/PrometheusEmitterConfig.java | 22 ++++++++-
.../druid/emitter/prometheus/MetricsTest.java | 10 ++--
.../emitter/prometheus/PrometheusEmitterTest.java | 54 ++++++++++++++++------
7 files changed, 102 insertions(+), 25 deletions(-)
diff --git a/docs/development/extensions-contrib/prometheus.md
b/docs/development/extensions-contrib/prometheus.md
index 02506bfae1..375a21c215 100644
--- a/docs/development/extensions-contrib/prometheus.md
+++ b/docs/development/extensions-contrib/prometheus.md
@@ -40,6 +40,8 @@ All the configuration parameters for the Prometheus emitter
are under `druid.emi
|`druid.emitter.prometheus.port`|The port on which to expose the prometheus
HTTPServer. Required if using exporter strategy.|no|none|
|`druid.emitter.prometheus.namespace`|Optional metric namespace. Must match
the regex `[a-zA-Z_:][a-zA-Z0-9_:]*`|no|"druid"|
|`druid.emitter.prometheus.dimensionMapPath`|JSON file defining the Prometheus
metric type, desired dimensions, help text, and conversionFactor for every
Druid metric.|no|Default mapping provided. See below.|
+|`druid.emitter.prometheus.addHostAsLabel`|Flag to include the hostname as a
prometheus label.|no|false|
+|`druid.emitter.prometheus.addServiceAsLabel`|Flag to include the druid
service name (e.g. `druid/broker`, `druid/coordinator`, etc.) as a prometheus
label.|no|false|
|`druid.emitter.prometheus.pushGatewayAddress`|Pushgateway address. Required
if using Pushgateway strategy|no|none|
diff --git a/extensions-contrib/prometheus-emitter/pom.xml
b/extensions-contrib/prometheus-emitter/pom.xml
index 4047815f8a..3d28550921 100644
--- a/extensions-contrib/prometheus-emitter/pom.xml
+++ b/extensions-contrib/prometheus-emitter/pom.xml
@@ -43,17 +43,17 @@
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient</artifactId>
- <version>0.7.0</version>
+ <version>0.16.0</version>
</dependency>
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_httpserver</artifactId>
- <version>0.7.0</version>
+ <version>0.16.0</version>
</dependency>
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_pushgateway</artifactId>
- <version>0.7.0</version>
+ <version>0.16.0</version>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
diff --git
a/extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/Metrics.java
b/extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/Metrics.java
index 7006da3ca8..7e71c88503 100644
---
a/extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/Metrics.java
+++
b/extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/Metrics.java
@@ -49,6 +49,9 @@ public class Metrics
private final ObjectMapper mapper = new ObjectMapper();
public static final Pattern PATTERN =
Pattern.compile("[^a-zA-Z_:][^a-zA-Z0-9_:]*");
+ private static final String TAG_HOSTNAME = "host_name";
+ private static final String TAG_SERVICE = "druid_service";
+
public DimensionsAndCollector getByName(String name, String service)
{
if (registeredMetrics.containsKey(name)) {
@@ -58,13 +61,22 @@ public class Metrics
}
}
- public Metrics(String namespace, String path)
+ public Metrics(String namespace, String path, boolean isAddHostAsLabel,
boolean isAddServiceAsLabel)
{
Map<String, Metric> metrics = readConfig(path);
for (Map.Entry<String, Metric> entry : metrics.entrySet()) {
String name = entry.getKey();
Metric metric = entry.getValue();
Metric.Type type = metric.type;
+
+ if (isAddHostAsLabel) {
+ metric.dimensions.add(TAG_HOSTNAME);
+ }
+
+ if (isAddServiceAsLabel) {
+ metric.dimensions.add(TAG_SERVICE);
+ }
+
String[] dimensions = metric.dimensions.toArray(new String[0]);
String formattedName =
PATTERN.matcher(StringUtils.toLowerCase(name)).replaceAll("_");
SimpleCollector collector = null;
diff --git
a/extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/PrometheusEmitter.java
b/extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/PrometheusEmitter.java
index 080800dde0..37a668fc64 100644
---
a/extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/PrometheusEmitter.java
+++
b/extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/PrometheusEmitter.java
@@ -49,6 +49,9 @@ public class PrometheusEmitter implements Emitter
private final PrometheusEmitterConfig.Strategy strategy;
private static final Pattern PATTERN =
Pattern.compile("[^a-zA-Z0-9_][^a-zA-Z0-9_]*");
+ private static final String TAG_HOSTNAME = "host_name";
+ private static final String TAG_SERVICE = "druid_service";
+
private HTTPServer server;
private PushGateway pushGateway;
private String identifier;
@@ -62,7 +65,7 @@ public class PrometheusEmitter implements Emitter
{
this.config = config;
this.strategy = config.getStrategy();
- metrics = new Metrics(config.getNamespace(), config.getDimensionMapPath());
+ metrics = new Metrics(config.getNamespace(), config.getDimensionMapPath(),
config.isAddHostAsLabel(), config.isAddServiceAsLabel());
}
@@ -113,6 +116,7 @@ public class PrometheusEmitter implements Emitter
{
String name = metricEvent.getMetric();
String service = metricEvent.getService();
+ String host = metricEvent.getHost();
Map<String, Object> userDims = metricEvent.getUserDims();
identifier = (userDims.get("task") == null ? metricEvent.getHost() :
(String) userDims.get("task"));
Number value = metricEvent.getValue();
@@ -125,7 +129,18 @@ public class PrometheusEmitter implements Emitter
String labelName = labelNames[i];
//labelName is controlled by the user. Instead of potential NPE on
invalid labelName we use "unknown" as the dimension value
Object userDim = userDims.get(labelName);
- labelValues[i] = userDim != null ?
PATTERN.matcher(userDim.toString()).replaceAll("_") : "unknown";
+
+ if (userDim != null) {
+ labelValues[i] = PATTERN.matcher(userDim.toString()).replaceAll("_");
+ } else {
+ if (config.isAddHostAsLabel() && TAG_HOSTNAME.equals(labelName)) {
+ labelValues[i] = host;
+ } else if (config.isAddServiceAsLabel() &&
TAG_SERVICE.equals(labelName)) {
+ labelValues[i] = service;
+ } else {
+ labelValues[i] = "unknown";
+ }
+ }
}
if (metric.getCollector() instanceof Counter) {
diff --git
a/extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/PrometheusEmitterConfig.java
b/extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/PrometheusEmitterConfig.java
index 475f370841..0356c7e969 100644
---
a/extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/PrometheusEmitterConfig.java
+++
b/extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/PrometheusEmitterConfig.java
@@ -53,13 +53,21 @@ public class PrometheusEmitterConfig
@Nullable
private final String pushGatewayAddress;
+ @JsonProperty
+ private final boolean addHostAsLabel;
+
+ @JsonProperty
+ private final boolean addServiceAsLabel;
+
@JsonCreator
public PrometheusEmitterConfig(
@JsonProperty("strategy") @Nullable Strategy strategy,
@JsonProperty("namespace") @Nullable String namespace,
@JsonProperty("dimensionMapPath") @Nullable String dimensionMapPath,
@JsonProperty("port") @Nullable Integer port,
- @JsonProperty("pushGatewayAddress") @Nullable String pushGatewayAddress
+ @JsonProperty("pushGatewayAddress") @Nullable String pushGatewayAddress,
+ @JsonProperty("addHostAsLabel") boolean addHostAsLabel,
+ @JsonProperty("addServiceAsLabel") boolean addServiceAsLabel
)
{
@@ -72,6 +80,8 @@ public class PrometheusEmitterConfig
Preconditions.checkNotNull(pushGatewayAddress, "Invalid pushGateway
address");
}
this.pushGatewayAddress = pushGatewayAddress;
+ this.addHostAsLabel = addHostAsLabel;
+ this.addServiceAsLabel = addServiceAsLabel;
}
public String getNamespace()
@@ -99,6 +109,16 @@ public class PrometheusEmitterConfig
return strategy;
}
+ public boolean isAddHostAsLabel()
+ {
+ return addHostAsLabel;
+ }
+
+ public boolean isAddServiceAsLabel()
+ {
+ return addServiceAsLabel;
+ }
+
public enum Strategy
{
exporter, pushgateway
diff --git
a/extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/MetricsTest.java
b/extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/MetricsTest.java
index 968b29e951..d4f8a6a826 100644
---
a/extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/MetricsTest.java
+++
b/extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/MetricsTest.java
@@ -28,18 +28,22 @@ public class MetricsTest
@Test
public void testMetricsConfiguration()
{
- Metrics metrics = new Metrics("test", null);
+ Metrics metrics = new Metrics("test", null, true, true);
DimensionsAndCollector dimensionsAndCollector =
metrics.getByName("query/time", "historical");
Assert.assertNotNull(dimensionsAndCollector);
String[] dimensions = dimensionsAndCollector.getDimensions();
Assert.assertEquals("dataSource", dimensions[0]);
- Assert.assertEquals("type", dimensions[1]);
+ Assert.assertEquals("druid_service", dimensions[1]);
+ Assert.assertEquals("host_name", dimensions[2]);
+ Assert.assertEquals("type", dimensions[3]);
Assert.assertEquals(1000.0, dimensionsAndCollector.getConversionFactor(),
0.0);
Assert.assertTrue(dimensionsAndCollector.getCollector() instanceof
Histogram);
DimensionsAndCollector d = metrics.getByName("segment/loadQueue/count",
"historical");
Assert.assertNotNull(d);
String[] dims = d.getDimensions();
- Assert.assertEquals("server", dims[0]);
+ Assert.assertEquals("druid_service", dims[0]);
+ Assert.assertEquals("host_name", dims[1]);
+ Assert.assertEquals("server", dims[2]);
}
}
diff --git
a/extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/PrometheusEmitterTest.java
b/extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/PrometheusEmitterTest.java
index b4979df094..10f0cb6f33 100644
---
a/extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/PrometheusEmitterTest.java
+++
b/extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/PrometheusEmitterTest.java
@@ -38,20 +38,43 @@ import static org.easymock.EasyMock.mock;
public class PrometheusEmitterTest
{
@Test
- public void testEmitter()
+ public void testEmitterWithServiceLabel()
{
- PrometheusEmitterConfig config = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null,
0, null);
+ CollectorRegistry.defaultRegistry.clear();
+ PrometheusEmitterConfig config = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null,
0, null, false, true);
PrometheusEmitterModule prometheusEmitterModule = new
PrometheusEmitterModule();
Emitter emitter = prometheusEmitterModule.getEmitter(config);
ServiceMetricEvent build = ServiceMetricEvent.builder()
.setDimension("server",
"druid-data01.vpc.region")
.build("segment/loadQueue/count", 10)
-
.build(ImmutableMap.of("service", "historical"));
+
.build(ImmutableMap.of("service", "historical", "host", "druid.test.cn"));
Assert.assertEquals("historical", build.getService());
+ Assert.assertEquals("druid.test.cn", build.getHost());
Assert.assertFalse(build.getUserDims().isEmpty());
emitter.emit(build);
Double count = CollectorRegistry.defaultRegistry.getSampleValue(
- "druid_segment_loadqueue_count", new String[]{"server"}, new
String[]{"druid_data01_vpc_region"}
+ "druid_segment_loadqueue_count", new String[]{"druid_service",
"server"}, new String[]{"historical", "druid_data01_vpc_region"}
+ );
+ Assert.assertEquals(10, count.intValue());
+ }
+
+ @Test
+ public void testEmitterWithServiceAndHostLabel()
+ {
+ CollectorRegistry.defaultRegistry.clear();
+ PrometheusEmitterConfig config = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null,
0, null, true, true);
+ PrometheusEmitterModule prometheusEmitterModule = new
PrometheusEmitterModule();
+ Emitter emitter = prometheusEmitterModule.getEmitter(config);
+ ServiceMetricEvent build = ServiceMetricEvent.builder()
+ .setDimension("server", "druid-data01.vpc.region")
+ .build("segment/loadQueue/count", 10)
+ .build(ImmutableMap.of("service", "historical", "host",
"druid.test.cn"));
+ Assert.assertEquals("historical", build.getService());
+ Assert.assertEquals("druid.test.cn", build.getHost());
+ Assert.assertFalse(build.getUserDims().isEmpty());
+ emitter.emit(build);
+ Double count = CollectorRegistry.defaultRegistry.getSampleValue(
+ "druid_segment_loadqueue_count", new String[]{"druid_service",
"host_name", "server"}, new String[]{"historical", "druid.test.cn",
"druid_data01_vpc_region"}
);
Assert.assertEquals(10, count.intValue());
}
@@ -59,33 +82,34 @@ public class PrometheusEmitterTest
@Test
public void testEmitterMetric()
{
- PrometheusEmitterConfig config = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway,
"namespace", null, 0, "pushgateway");
+ CollectorRegistry.defaultRegistry.clear();
+ PrometheusEmitterConfig config = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway,
"namespace", null, 0, "pushgateway", true, true);
PrometheusEmitterModule prometheusEmitterModule = new
PrometheusEmitterModule();
Emitter emitter = prometheusEmitterModule.getEmitter(config);
ServiceMetricEvent build = ServiceMetricEvent.builder()
.setDimension("dataSource", "test")
.setDimension("taskType", "index_parallel")
.build("task/run/time", 500)
- .build(ImmutableMap.of("service", "overlord"));
+ .build(ImmutableMap.of("service", "overlord", "host",
"druid.test.cn"));
emitter.emit(build);
double assertEpsilon = 0.0001;
Assert.assertEquals(0.0, CollectorRegistry.defaultRegistry.getSampleValue(
- "namespace_task_run_time_bucket", new String[]{"dataSource",
"taskType", "le"}, new String[]{"test", "index_parallel", "0.1"}
+ "namespace_task_run_time_bucket", new String[]{"dataSource",
"druid_service", "host_name", "taskType", "le"}, new String[]{"test",
"overlord", "druid.test.cn", "index_parallel", "0.1"}
), assertEpsilon);
Assert.assertEquals(1.0, CollectorRegistry.defaultRegistry.getSampleValue(
- "namespace_task_run_time_bucket", new String[]{"dataSource",
"taskType", "le"}, new String[]{"test", "index_parallel", "0.5"}
+ "namespace_task_run_time_bucket", new String[]{"dataSource",
"druid_service", "host_name", "taskType", "le"}, new String[]{"test",
"overlord", "druid.test.cn", "index_parallel", "0.5"}
), assertEpsilon);
}
@Test
public void testEmitterStart()
{
- PrometheusEmitterConfig exportEmitterConfig = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter,
"namespace1", null, 0, null);
+ PrometheusEmitterConfig exportEmitterConfig = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter,
"namespace1", null, 0, null, true, true);
PrometheusEmitter exportEmitter = new
PrometheusEmitter(exportEmitterConfig);
exportEmitter.start();
Assert.assertNotNull(exportEmitter.getServer());
- PrometheusEmitterConfig pushEmitterConfig = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway,
"namespace2", null, 0, "pushgateway");
+ PrometheusEmitterConfig pushEmitterConfig = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway,
"namespace2", null, 0, "pushgateway", true, true);
PrometheusEmitter pushEmitter = new PrometheusEmitter(pushEmitterConfig);
pushEmitter.start();
Assert.assertNotNull(pushEmitter.getPushGateway());
@@ -94,7 +118,7 @@ public class PrometheusEmitterTest
@Test
public void testEmitterPush() throws IOException
{
- PrometheusEmitterConfig emitterConfig = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway,
"namespace3", null, 0, "pushgateway");
+ PrometheusEmitterConfig emitterConfig = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway,
"namespace3", null, 0, "pushgateway", true, true);
PushGateway mockPushGateway = mock(PushGateway.class);
mockPushGateway.push(anyObject(Collector.class), anyString(),
anyObject(ImmutableMap.class));
@@ -105,7 +129,7 @@ public class PrometheusEmitterTest
ServiceMetricEvent build = ServiceMetricEvent.builder()
.setDimension("task", "index_parallel")
.build("task/run/time", 500)
- .build(ImmutableMap.of("service", "peon"));
+ .build(ImmutableMap.of("service", "peon", "host",
"druid.test.cn"));
emitter.emit(build);
emitter.flush();
}
@@ -113,13 +137,13 @@ public class PrometheusEmitterTest
@Test
public void testEmitterConfigCreationWithNullAsAddress()
{
- Assert.assertThrows(NullPointerException.class, () -> new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway,
"namespace4", null, 0, null));
+ Assert.assertThrows(NullPointerException.class, () -> new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway,
"namespace4", null, 0, null, true, true));
}
@Test
public void testEmitterStartWithHttpUrl()
{
- PrometheusEmitterConfig pushEmitterConfig = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway,
"namespace4", null, 0, "http://pushgateway";);
+ PrometheusEmitterConfig pushEmitterConfig = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway,
"namespace4", null, 0, "http://pushgateway";, true, true);
PrometheusEmitter pushEmitter = new PrometheusEmitter(pushEmitterConfig);
pushEmitter.start();
Assert.assertNotNull(pushEmitter.getPushGateway());
@@ -128,7 +152,7 @@ public class PrometheusEmitterTest
@Test
public void testEmitterStartWithHttpsUrl()
{
- PrometheusEmitterConfig pushEmitterConfig = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway,
"namespace5", null, 0, "https://pushgateway";);
+ PrometheusEmitterConfig pushEmitterConfig = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway,
"namespace5", null, 0, "https://pushgateway";, true, true);
PrometheusEmitter pushEmitter = new PrometheusEmitter(pushEmitterConfig);
pushEmitter.start();
Assert.assertNotNull(pushEmitter.getPushGateway());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
