This is an automated email from the ASF dual-hosted git repository.
abhishekrb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new e0baddd1edb Gracefully handle negative counter values for
prometheus-emitter & update metric types for `sys/disk/*` metrics (#18719)
e0baddd1edb is described below
commit e0baddd1edb76fa2c16b7a180b2a9a9aee2a9d24
Author: aho135 <[email protected]>
AuthorDate: Mon Nov 17 15:45:19 2025 -0800
Gracefully handle negative counter values for prometheus-emitter & update
metric types for `sys/disk/*` metrics (#18719)
Graceful handling of prometheus counter values when a negative value is
encountered.
Update the OshiSysMonitor DiskStats sys/disk/* delta metrics from counter
to gauge type for prometheus-emitter, dropwizard-emitter and statsd-emitter.
---
.../main/resources/defaultMetricDimensions.json | 12 +++++-----
.../emitter/prometheus/PrometheusEmitter.java | 9 +++++--
.../src/main/resources/defaultMetrics.json | 16 ++++++-------
.../emitter/prometheus/PrometheusEmitterTest.java | 28 +++++++++++++++-------
.../main/resources/defaultMetricDimensions.json | 12 +++++-----
5 files changed, 47 insertions(+), 30 deletions(-)
diff --git
a/extensions-contrib/dropwizard-emitter/src/main/resources/defaultMetricDimensions.json
b/extensions-contrib/dropwizard-emitter/src/main/resources/defaultMetricDimensions.json
index 623cce0e3e8..8e24ff49683 100644
---
a/extensions-contrib/dropwizard-emitter/src/main/resources/defaultMetricDimensions.json
+++
b/extensions-contrib/dropwizard-emitter/src/main/resources/defaultMetricDimensions.json
@@ -449,33 +449,33 @@
"dimensions": [
"fsDevName"
],
- "type": "counter"
+ "type": "gauge"
},
"sys/disk/read/count": {
"dimensions": [
"fsDevName"
],
- "type": "counter"
+ "type": "gauge"
},
"sys/disk/write/size": {
"dimensions": [
"fsDevName"
],
- "type": "counter"
+ "type": "gauge"
},
"sys/disk/read/size": {
"dimensions": [
"fsDevName"
],
- "type": "counter"
+ "type": "gauge"
},
"sys/net/write/size": {
"dimensions": [],
- "type": "counter"
+ "type": "gauge"
},
"sys/net/read/size": {
"dimensions": [],
- "type": "counter"
+ "type": "gauge"
},
"sys/fs/used": {
"dimensions": [
diff --git
a/extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/PrometheusEmitter.java
b/extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/PrometheusEmitter.java
index 4cad38a511a..60af27fcd4d 100644
---
a/extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/PrometheusEmitter.java
+++
b/extensions-contrib/prometheus-emitter/src/main/java/org/apache/druid/emitter/prometheus/PrometheusEmitter.java
@@ -179,8 +179,13 @@ public class PrometheusEmitter implements Emitter
}
if (metric.getCollector() instanceof Counter) {
- ((Counter)
metric.getCollector()).labels(labelValues).inc(value.doubleValue());
- metric.resetLastUpdateTime(Arrays.asList(labelValues));
+ double metricValue = value.doubleValue();
+ if (metricValue >= 0) {
+ ((Counter)
metric.getCollector()).labels(labelValues).inc(metricValue);
+ metric.resetLastUpdateTime(Arrays.asList(labelValues));
+ } else {
+ log.warn("Counter increment amount must be non-negative, but got
value[%f] for metric[%s]. If this is valid, metric should be defined as a gauge
instead of counter.", metricValue, name);
+ }
} else if (metric.getCollector() instanceof Gauge) {
((Gauge)
metric.getCollector()).labels(labelValues).set(value.doubleValue());
metric.resetLastUpdateTime(Arrays.asList(labelValues));
diff --git
a/extensions-contrib/prometheus-emitter/src/main/resources/defaultMetrics.json
b/extensions-contrib/prometheus-emitter/src/main/resources/defaultMetrics.json
index a4fa1e83f92..82ced1e9abb 100644
---
a/extensions-contrib/prometheus-emitter/src/main/resources/defaultMetrics.json
+++
b/extensions-contrib/prometheus-emitter/src/main/resources/defaultMetrics.json
@@ -223,16 +223,16 @@
"sys/swap/max" : { "dimensions" : [], "type" : "gauge", "help": "Max swap"},
"sys/swap/pageIn" : { "dimensions" : [], "type" : "gauge", "help": "Paged in
swap"},
"sys/swap/pageOut" : { "dimensions" : [], "type" : "gauge", "help": "Paged
out swap"},
- "sys/disk/write/count" : { "dimensions" : ["diskName"], "type" : "count",
"help": "Writes to disk."},
- "sys/disk/read/count" : { "dimensions" : ["diskName"], "type" : "count",
"help": "Reads from disk."},
- "sys/disk/write/size" : { "dimensions" : ["diskName"], "type" : "count",
"help": "Bytes written to disk. Can we used to determine how much paging is
occurring with regards to segments."},
- "sys/disk/read/size" : { "dimensions" : ["diskName"], "type" : "count",
"help": "Bytes read from disk. Can we used to determine how much paging is
occurring with regards to segments."},
+ "sys/disk/write/count" : { "dimensions" : ["diskName"], "type" : "gauge",
"help": "Writes to disk."},
+ "sys/disk/read/count" : { "dimensions" : ["diskName"], "type" : "gauge",
"help": "Reads from disk."},
+ "sys/disk/write/size" : { "dimensions" : ["diskName"], "type" : "gauge",
"help": "Bytes written to disk. Can we used to determine how much paging is
occurring with regards to segments."},
+ "sys/disk/read/size" : { "dimensions" : ["diskName"], "type" : "gauge",
"help": "Bytes read from disk. Can we used to determine how much paging is
occurring with regards to segments."},
"sys/disk/queue" : { "dimensions" : ["diskName"], "type" : "gauge", "help":
"Disk queue length."},
"sys/disk/transferTime" : { "dimensions" : ["diskName"], "type" : "timer",
"help": "Transfer time to read from or write to disk"},
- "sys/net/write/size" : { "dimensions" : [], "type" : "count", "help": "Bytes
written to the network."},
- "sys/net/read/size" : { "dimensions" : [], "type" : "count", "help": "Bytes
read from the network."},
- "sys/net/read/packets" : { "dimensions" : [], "type" : "count", "help":
"Total packets read from the network"},
- "sys/net/write/packets" : { "dimensions" : [], "type" : "count", "help":
"Total packets written to the network"},
+ "sys/net/write/size" : { "dimensions" : [], "type" : "gauge", "help": "Bytes
written to the network."},
+ "sys/net/read/size" : { "dimensions" : [], "type" : "gauge", "help": "Bytes
read from the network."},
+ "sys/net/read/packets" : { "dimensions" : [], "type" : "gauge", "help":
"Total packets read from the network"},
+ "sys/net/write/packets" : { "dimensions" : [], "type" : "gauge", "help":
"Total packets written to the network"},
"sys/net/read/errors" : { "dimensions" : [], "type" : "gauge", "help":
"Total network read errors"},
"sys/net/write/errors" : { "dimensions" : [], "type" : "gauge", "help":
"Total network write errors"},
"sys/net/read/dropped" : { "dimensions" : [], "type" : "gauge", "help":
"Total packets dropped coming from network"},
diff --git
a/extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/PrometheusEmitterTest.java
b/extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/PrometheusEmitterTest.java
index 88ce81629c4..2158260afb3 100644
---
a/extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/PrometheusEmitterTest.java
+++
b/extensions-contrib/prometheus-emitter/src/test/java/org/apache/druid/emitter/prometheus/PrometheusEmitterTest.java
@@ -28,6 +28,7 @@ import
org.apache.druid.java.util.common.concurrent.ScheduledExecutors;
import org.apache.druid.java.util.emitter.core.Emitter;
import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
import org.easymock.EasyMock;
+import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
@@ -49,7 +50,6 @@ public class PrometheusEmitterTest
@Test
public void testEmitterWithServiceLabel()
{
- CollectorRegistry.defaultRegistry.clear();
PrometheusEmitterConfig config = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null,
0, null, false, true, 60, null, false, null);
PrometheusEmitterModule prometheusEmitterModule = new
PrometheusEmitterModule();
Emitter emitter = prometheusEmitterModule.getEmitter(config);
@@ -70,7 +70,6 @@ public class PrometheusEmitterTest
@Test
public void testEmitterWithServiceAndHostLabel()
{
- CollectorRegistry.defaultRegistry.clear();
PrometheusEmitterConfig config = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null,
0, null, true, true, 60, null, false, null);
PrometheusEmitterModule prometheusEmitterModule = new
PrometheusEmitterModule();
Emitter emitter = prometheusEmitterModule.getEmitter(config);
@@ -91,7 +90,6 @@ public class PrometheusEmitterTest
@Test
public void testEmitterWithExtraLabels()
{
- CollectorRegistry.defaultRegistry.clear();
Map<String, String> extraLabels = new HashMap<>();
extraLabels.put("labelName", "labelValue");
PrometheusEmitterConfig config = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null,
0, null, false, false, 60, extraLabels, false, null);
@@ -113,7 +111,6 @@ public class PrometheusEmitterTest
@Test
public void testEmitterWithServiceLabelAndExtraLabel()
{
- CollectorRegistry.defaultRegistry.clear();
Map<String, String> extraLabels = new HashMap<>();
extraLabels.put("labelName", "labelValue");
PrometheusEmitterConfig config = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null,
0, null, false, true, 60, extraLabels, false, null);
@@ -135,7 +132,6 @@ public class PrometheusEmitterTest
@Test
public void testEmitterWithEmptyExtraLabels()
{
- CollectorRegistry.defaultRegistry.clear();
Map<String, String> extraLabels = new HashMap<>();
PrometheusEmitterConfig config = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, null, null,
0, null, false, true, 60, extraLabels, false, null);
PrometheusEmitterModule prometheusEmitterModule = new
PrometheusEmitterModule();
@@ -156,7 +152,6 @@ public class PrometheusEmitterTest
@Test
public void testEmitterWithMultipleExtraLabels()
{
- CollectorRegistry.defaultRegistry.clear();
Map<String, String> extraLabels = new HashMap<>();
extraLabels.put("labelName1", "labelValue1");
extraLabels.put("labelName2", "labelValue2");
@@ -179,7 +174,6 @@ public class PrometheusEmitterTest
@Test
public void testEmitterWithLabelCollision()
{
- CollectorRegistry.defaultRegistry.clear();
// ExtraLabels contains a label that collides with a service label
Map<String, String> extraLabels = new HashMap<>();
extraLabels.put("server", "collisionLabelValue");
@@ -209,7 +203,6 @@ public class PrometheusEmitterTest
@Test
public void testEmitterMetric()
{
- CollectorRegistry.defaultRegistry.clear();
PrometheusEmitterConfig config = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.pushgateway,
"namespace", null, 0, "pushgateway", true, true, 60, null, false, null);
PrometheusEmitterModule prometheusEmitterModule = new
PrometheusEmitterModule();
Emitter emitter = prometheusEmitterModule.getEmitter(config);
@@ -598,4 +591,23 @@ public class PrometheusEmitterTest
Assert.assertEquals(1, metric.getLabelValuesToStopwatch().size());
emitter.close();
}
+
+ @Test
+ public void testCounterWithNegativeValue()
+ {
+ PrometheusEmitterConfig config = new
PrometheusEmitterConfig(PrometheusEmitterConfig.Strategy.exporter, "test",
null, 0, null, true, true, null, null, false, null);
+ PrometheusEmitter emitter = new PrometheusEmitter(config);
+ ServiceMetricEvent event = ServiceMetricEvent.builder()
+ .setMetric("segment/moveSkipped/count", -1)
+ .setDimension("server", "historical1")
+ .build(ImmutableMap.of("service", "historical", "host",
"druid.test.cn"));
+ emitter.emit(event);
+ emitter.close();
+ }
+
+ @After
+ public void tearDown()
+ {
+ CollectorRegistry.defaultRegistry.clear();
+ }
}
diff --git
a/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json
b/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json
index bc28ace0b04..4da2d27a2b0 100644
---
a/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json
+++
b/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json
@@ -137,12 +137,12 @@
"sys/swap/max" : { "dimensions" : [], "type" : "gauge"},
"sys/swap/pageIn" : { "dimensions" : [], "type" : "gauge"},
"sys/swap/pageOut" : { "dimensions" : [], "type" : "gauge"},
- "sys/disk/write/count" : { "dimensions" : ["fsDevName"], "type" : "count"},
- "sys/disk/read/count" : { "dimensions" : ["fsDevName"], "type" : "count"},
- "sys/disk/write/size" : { "dimensions" : ["fsDevName"], "type" : "count"},
- "sys/disk/read/size" : { "dimensions" : ["fsDevName"], "type" : "count"},
- "sys/net/write/size" : { "dimensions" : [], "type" : "count"},
- "sys/net/read/size" : { "dimensions" : [], "type" : "count"},
+ "sys/disk/write/count" : { "dimensions" : ["fsDevName"], "type" : "gauge"},
+ "sys/disk/read/count" : { "dimensions" : ["fsDevName"], "type" : "gauge"},
+ "sys/disk/write/size" : { "dimensions" : ["fsDevName"], "type" : "gauge"},
+ "sys/disk/read/size" : { "dimensions" : ["fsDevName"], "type" : "gauge"},
+ "sys/net/write/size" : { "dimensions" : [], "type" : "gauge"},
+ "sys/net/read/size" : { "dimensions" : [], "type" : "gauge"},
"sys/fs/used" : { "dimensions" : ["fsDevName", "fsDirName", "fsTypeName",
"fsSysTypeName", "fsOptions"], "type" : "gauge"},
"sys/fs/max" : { "dimensions" : ["fsDevName", "fsDirName", "fsTypeName",
"fsSysTypeName", "fsOptions"], "type" : "gauge"},
"sys/mem/used" : { "dimensions" : [], "type" : "gauge"},
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
