This is an automated email from the ASF dual-hosted git repository.
zaynt4606 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 1256618a2b [CELEBORN-2357] Add Metrics Prefix
1256618a2b is described below
commit 1256618a2b4d0119deb365990cb01dcf30ffabaf
Author: afterincomparableyum
<[email protected]>
AuthorDate: Mon Oct 12 14:14:48 2026 +0800
[CELEBORN-2357] Add Metrics Prefix
### What changes were proposed in this pull request?
Enable users the option to prefix their metrics with something unique that
will allow them to easily find any celeborn related metircs in observability
platforms.
### Why are the changes needed?
Users will be able to easily identify celeborn metrics on observability
tools with this prefix.
### Does this PR resolve a correctness bug?
- [ ] Yes
### Does this PR introduce _any_ user-facing change?
- [ ] Yes
### How was this patch tested?
CI/CD
Closes #3851 from afterincomparableyum/celeborn-2357.
Authored-by: afterincomparableyum
<[email protected]>
Signed-off-by: zhengtao <[email protected]>
AI-Contributed/Feature: 0/18
AI-Contributed/UT: 0/31
---
.../org/apache/celeborn/common/CelebornConf.scala | 13 +++++++++
.../common/metrics/source/AbstractSource.scala | 4 ++-
.../metrics/source/CelebornSourceSuite.scala | 31 ++++++++++++++++++++++
docs/configuration/metrics.md | 1 +
4 files changed, 48 insertions(+), 1 deletion(-)
diff --git
a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
index e3309ceb11..3eef83205e 100644
--- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
@@ -918,6 +918,7 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable
with Logging with Se
def metricsConf: Option[String] = get(METRICS_CONF)
def metricsSystemEnable: Boolean = get(METRICS_ENABLED)
def metricsSampleRate: Double = get(METRICS_SAMPLE_RATE)
+ def metricsPrefix: String = get(METRICS_PREFIX)
def metricsSlidingWindowSize: Int = get(METRICS_SLIDING_WINDOW_SIZE)
def metricsCollectCriticalEnabled: Boolean =
get(METRICS_COLLECT_CRITICAL_ENABLED)
def metricsCapacity: Int = get(METRICS_CAPACITY)
@@ -6022,6 +6023,18 @@ object CelebornConf extends Logging {
.stringConf
.createOptional
+ val METRICS_PREFIX: ConfigEntry[String] =
+ buildConf("celeborn.metrics.prefix")
+ .categories("metrics")
+ .doc("Prefix metrics with this value. It must be a valid Prometheus
metric name, " +
+ "matching the regex `[a-zA-Z_][a-zA-Z0-9_]*`.")
+ .version("1.0.0")
+ .stringConf
+ .checkValue(
+ p => p.matches("[a-zA-Z_][a-zA-Z0-9_]*"),
+ "metrics prefix must match the regex [a-zA-Z_][a-zA-Z0-9_]*")
+ .createWithDefault("metrics")
+
val METRICS_ENABLED: ConfigEntry[Boolean] =
buildConf("celeborn.metrics.enabled")
.categories("metrics")
diff --git
a/common/src/main/scala/org/apache/celeborn/common/metrics/source/AbstractSource.scala
b/common/src/main/scala/org/apache/celeborn/common/metrics/source/AbstractSource.scala
index e7b71e5ec2..2047a7531b 100644
---
a/common/src/main/scala/org/apache/celeborn/common/metrics/source/AbstractSource.scala
+++
b/common/src/main/scala/org/apache/celeborn/common/metrics/source/AbstractSource.scala
@@ -60,6 +60,8 @@ abstract class AbstractSource(conf: CelebornConf, role:
String)
val metricsCapacity: Int = conf.metricsCapacity
+ val metricsPrefix: String = conf.metricsPrefix
+
val timerSupplier = new TimerSupplier(metricsSlidingWindowSize)
val histogramSupplier = new HistogramSupplier(metricsSlidingWindowSize)
@@ -701,7 +703,7 @@ abstract class AbstractSource(conf: CelebornConf, role:
String)
}
protected def normalizeKey(key: String): String = {
- s"metrics_${key.replaceAll("[^a-zA-Z0-9]", "_")}_"
+ s"${metricsPrefix}_${key.replaceAll("[^a-zA-Z0-9]", "_")}_"
}
def reportNanosAsMills(value: Double): Double = {
diff --git
a/common/src/test/scala/org/apache/celeborn/common/metrics/source/CelebornSourceSuite.scala
b/common/src/test/scala/org/apache/celeborn/common/metrics/source/CelebornSourceSuite.scala
index 110d0b1ebe..c6eeda1956 100644
---
a/common/src/test/scala/org/apache/celeborn/common/metrics/source/CelebornSourceSuite.scala
+++
b/common/src/test/scala/org/apache/celeborn/common/metrics/source/CelebornSourceSuite.scala
@@ -38,6 +38,37 @@ class CelebornSourceSuite extends CelebornFunSuite {
assert(res.contains("metrics_abc_Count"))
}
+ test("test customized metrics prefix") {
+ val conf = new CelebornConf()
+ conf.set(CelebornConf.METRICS_PREFIX.key, "celeborn")
+
+ val mockSource = new AbstractSource(conf, Role.WORKER) {
+ override def sourceName: String = "mockSource"
+ }
+ val histogram = "abc"
+ mockSource.addHistogram(histogram)
+ for (i <- 1 to 100) {
+ mockSource.updateHistogram(histogram, 10)
+ }
+ val res = mockSource.getMetrics
+
+ assert(res.contains("celeborn_abc_Count"))
+ assert(!res.contains("metrics_abc_Count"))
+ }
+
+ test("test invalid metrics prefix") {
+ Seq("team-prod", "1celeborn", "celeborn.metrics", "").foreach { prefix =>
+ val conf = new CelebornConf()
+ conf.set(CelebornConf.METRICS_PREFIX.key, prefix)
+ val error = intercept[IllegalArgumentException] {
+ new AbstractSource(conf, Role.WORKER) {
+ override def sourceName: String = "mockSource"
+ }
+ }
+ assert(error.getMessage.contains(CelebornConf.METRICS_PREFIX.key))
+ }
+ }
+
test("test getMetrics with customized label") {
val conf = new CelebornConf()
createAbstractSourceAndCheck(conf, "", Role.MASTER)
diff --git a/docs/configuration/metrics.md b/docs/configuration/metrics.md
index 148c49f074..31cd4007d5 100644
--- a/docs/configuration/metrics.md
+++ b/docs/configuration/metrics.md
@@ -28,6 +28,7 @@ license: |
| celeborn.metrics.json.pretty.enabled | true | false | When true, view
metrics in json pretty format | 0.4.0 | |
| celeborn.metrics.loggerSink.output.enabled | false | false | Whether to
output scraped metrics to the logger. This config will have effect if you
enabled logger sink.If you will not scrape metrics periodically, do add
`org.apache.celeborn.common.metrics.sink.LoggerSink` to metrics.properties. |
0.6.0 | |
| celeborn.metrics.loggerSink.scrape.interval | 30min | false | The interval
of logger sink to scrape its own metrics. This config will have effect if you
enabled logger sink. If you will not scrape metrics periodically, do add
`org.apache.celeborn.common.metrics.sink.LoggerSink` to metrics.properties. |
0.6.0 | |
+| celeborn.metrics.prefix | metrics | false | Prefix metrics with this value.
It must be a valid Prometheus metric name, matching the regex
`[a-zA-Z_][a-zA-Z0-9_]*`. | 1.0.0 | |
| celeborn.metrics.prometheus.path | /metrics/prometheus | false | URI context
path of prometheus metrics HTTP server. | 0.4.0 | |
| celeborn.metrics.sample.rate | 1.0 | false | It controls if Celeborn collect
timer metrics for some operations. Its value should be in [0.0, 1.0]. | 0.2.0 |
|
| celeborn.metrics.timer.slidingWindow.size | 4096 | false | The sliding
window size of timer metric. | 0.2.0 | |
