This is an automated email from the ASF dual-hosted git repository.
zaynt4606 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 04ec937ff0 [CELEBORN-2387] Add ReserveSlotsFailCount worker metric
04ec937ff0 is described below
commit 04ec937ff0b86b952d17b90a14566a4f07caf6b9
Author: hzheng182 <[email protected]>
AuthorDate: Wed Oct 21 12:03:37 2026 +0800
[CELEBORN-2387] Add ReserveSlotsFailCount worker metric
### What changes were proposed in this pull request?
Adds a new worker metric, `ReserveSlotsFailCount`, and increments it on
every non-success path of `Controller.handleReserveSlots`.
Documents the counter in `docs/monitoring.md`, specifying that it counts
failed reservation requests once per request rather than per slot.
JIRA: https://issues.apache.org/jira/browse/CELEBORN-2387
### Why are the changes needed?
`handleReserveSlots` already replies with distinct non-success status codes:
- `WORKER_SHUTDOWN` — worker is shutting down
- `NO_AVAILABLE_WORKING_DIR` — no healthy local dirs and no remote storage
- `RESERVE_SLOTS_FAILED` — not all primary partitions could be satisfied
- `RESERVE_SLOTS_FAILED` — not all replica partitions could be satisfied
...but none of these paths incremented a counter. Slot reservation was the
only major worker operation without a `*FailCount` metric (push, replicate,
fetch, commit, register all have one), so failed reservations were invisible to
metrics/dashboards and only visible via logs.
`WorkerSource.scala` declares `RESERVE_SLOTS_FAIL_COUNT =
"ReserveSlotsFailCount"` and registers it with `addCounter(...)`, alongside
`SlotsAllocated`. `Controller.scala` calls
`workerSource.incCounter(WorkerSource.RESERVE_SLOTS_FAIL_COUNT)` before each of
the four non-success `context.reply(...)` sites in `handleReserveSlots`.
One fail counter per failed reservation regardless of cause, matching the
existing `*FailCount` convention (e.g. `CommitFilesFailCount`,
`RegisterWithMasterFailCount`). This gives a single "reserve slots did not
succeed" rate; the specific cause remains distinguishable via the status code
in logs.
### Does this PR introduce _any_ user-facing change?
Yes — adds a new worker metric `ReserveSlotsFailCount`. No config or API
changes.
### How was this patch tested?
Follows the established counter pattern exactly (constant + `addCounter`
registration + `incCounter` at the failure path), with no new imports or
signatures.
The documentation follow-up was checked against the counter registration
and all four explicit failure paths; `git diff --check` passes.
Closes #3765 from shellfish007/upstream-pr/reserve-slots-fail-count-metric.
Lead-authored-by: hzheng182 <[email protected]>
Co-authored-by: zhengtao <[email protected]>
Signed-off-by: zhengtao <[email protected]>
Co-Authored-By: Codex <[email protected]>
AI-Model: gpt-6
AI-Contributed/Feature: 1/7
AI-Contributed/UT: 0/0
---
docs/monitoring.md | 1 +
.../scala/org/apache/celeborn/service/deploy/worker/Controller.scala | 4 ++++
.../org/apache/celeborn/service/deploy/worker/WorkerSource.scala | 2 ++
3 files changed, 7 insertions(+)
diff --git a/docs/monitoring.md b/docs/monitoring.md
index 22e8f1062d..eb0da59170 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -247,6 +247,7 @@ These metrics are exposed by Celeborn worker.
| CommitFilesTime | The time for a worker to flush
buffers and close files related to specified shuffle.
|
| CommitFilesFailCount | The count of commit files
request failed in current worker.
|
| SlotsAllocated | Slots allocated in last hour.
|
+ | ReserveSlotsFailCount | The count of failed slot
reservation requests, counted once per request rather than per slot.
|
| ActiveSlotsCount | The number of slots currently
being used in a worker.
|
| ReserveSlotsTime | ReserveSlots means acquire a
disk buffer and record partition location.
|
| ActiveConnectionCount | The count of active network
connection.
|
diff --git
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Controller.scala
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Controller.scala
index 9a723d6c60..fcb542f58b 100644
---
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Controller.scala
+++
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Controller.scala
@@ -186,6 +186,7 @@ private[deploy] class Controller(
if (shutdown.get()) {
val msg = "Current worker is shutting down!"
logError(s"[handleReserveSlots] $msg")
+ workerSource.incCounter(WorkerSource.RESERVE_SLOTS_FAIL_COUNT)
context.reply(ReserveSlotsResponse(StatusCode.WORKER_SHUTDOWN, msg))
return
}
@@ -193,6 +194,7 @@ private[deploy] class Controller(
if (storageManager.healthyLocalWorkingDirs().size <= 0 &&
remoteStorageDirs.isEmpty) {
val msg = "Local storage has no available dirs!"
logError(s"[handleReserveSlots] $msg")
+ workerSource.incCounter(WorkerSource.RESERVE_SLOTS_FAIL_COUNT)
context.reply(ReserveSlotsResponse(StatusCode.NO_AVAILABLE_WORKING_DIR,
msg))
return
}
@@ -217,6 +219,7 @@ private[deploy] class Controller(
if (primaryLocs.size() < requestPrimaryLocs.size()) {
val msg = s"Not all primary partition satisfied for $shuffleKey"
logWarning(s"[handleReserveSlots] $msg, will destroy writers.")
+ workerSource.incCounter(WorkerSource.RESERVE_SLOTS_FAIL_COUNT)
destroyWriters(primaryLocs, shuffleKey)
context.reply(ReserveSlotsResponse(StatusCode.RESERVE_SLOTS_FAILED, msg))
return
@@ -238,6 +241,7 @@ private[deploy] class Controller(
if (replicaLocs.size() < requestReplicaLocs.size()) {
val msg = s"Not all replica partition satisfied for $shuffleKey"
logWarning(s"[handleReserveSlots] $msg, destroy writers.")
+ workerSource.incCounter(WorkerSource.RESERVE_SLOTS_FAIL_COUNT)
destroyWriters(primaryLocs, shuffleKey)
destroyWriters(replicaLocs, shuffleKey)
context.reply(ReserveSlotsResponse(StatusCode.RESERVE_SLOTS_FAILED, msg))
diff --git
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala
index 20c7cf6426..6ddde7c6e9 100644
---
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala
+++
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala
@@ -62,6 +62,7 @@ class WorkerSource(conf: CelebornConf) extends
AbstractSource(conf, Role.WORKER)
addCounter(SEGMENT_START_FAIL_COUNT)
addCounter(SLOTS_ALLOCATED)
+ addCounter(RESERVE_SLOTS_FAIL_COUNT)
addCounter(REGISTER_WITH_MASTER_FAIL_COUNT)
addCounter(COMMIT_FILES_FAIL_COUNT)
@@ -237,6 +238,7 @@ object WorkerSource {
// slots
val SLOTS_ALLOCATED = "SlotsAllocated"
+ val RESERVE_SLOTS_FAIL_COUNT = "ReserveSlotsFailCount"
val ACTIVE_SLOTS_COUNT = "ActiveSlotsCount"
val RESERVE_SLOTS_TIME = "ReserveSlotsTime"
