Gargi-jais11 commented on code in PR #10609:
URL: https://github.com/apache/ozone/pull/10609#discussion_r3748503200
##########
hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - Disk
Balancer.json:
##########
@@ -0,0 +1,1093 @@
+{
+ "__inputs": [
+ {
+ "name": "DS_PROMETHEUS",
+ "label": "prometheus",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "prometheus",
+ "pluginName": "Prometheus"
+ }
+ ],
+ "__elements": {},
+ "__requires": [
+ {
+ "type": "panel",
+ "id": "bargauge",
+ "name": "Bar gauge",
+ "version": ""
+ },
+ {
+ "type": "panel",
+ "id": "gauge",
+ "name": "Gauge",
+ "version": ""
+ },
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "13.0.2"
+ },
+ {
+ "type": "panel",
+ "id": "piechart",
+ "name": "Pie chart",
+ "version": ""
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "1.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "stat",
+ "name": "Stat",
+ "version": ""
+ },
+ {
+ "type": "panel",
+ "id": "timeseries",
+ "name": "Time series",
+ "version": ""
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "grafana",
+ "uid": "-- Grafana --"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ },
+ {
+ "builtIn": 1,
+ "enable": true,
+ "hide": true,
+ "iconColor": "",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "Monitors Ozone Disk Balancer progress, throughput, and idle
state per datanode.",
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 1,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "description": "Shows how full each disk is over time. When the balancer
is running, you should see these lines slowly move closer together until they
meet in the middle.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "smooth",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "showValues": false,
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 12,
+ "x": 0,
+ "y": 0
+ },
+ "id": 2,
+ "options": {
+ "annotations": {
+ "clustering": -1,
+ "multiLane": false
+ },
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "13.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum({__name__=~\"volume_info_metrics_.*_ozone_used\",
instance=~\"$datanode\"}) by (instance, storagedirectory)",
+ "legendFormat": "{{instance}} {{storagedirectory}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Disk Space Convergence Profile (Historical Before vs After)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "description": "The exact size of the data gap between your most full
disk and your emptiest disk right now. The balancer's goal is to shrink this
number down to zero.",
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 12,
+ "x": 12,
+ "y": 0
+ },
+ "id": 3,
+ "options": {
+ "displayMode": "lcd",
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "maxVizHeight": 300,
+ "minVizHeight": 16,
+ "minVizWidth": 8,
+ "namePlacement": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showUnfilled": true,
+ "sizing": "auto",
+ "valueMode": "color"
+ },
+ "pluginVersion": "13.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "max(sum({__name__=~\"volume_info_metrics_.*_ozone_used\",
instance=~\"$datanode\"}) by (instance, storagedirectory)) by (instance) -
min(sum({__name__=~\"volume_info_metrics_.*_ozone_used\",
instance=~\"$datanode\"}) by (instance, storagedirectory)) by (instance)",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Current Disk Space Allocation Skew (Max - Min Volume Gap)",
+ "type": "bargauge"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "description": "How much time it takes the system to successfully copy,
verify, and finalize a single container.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "showValues": false,
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 24,
+ "x": 0,
+ "y": 4
+ },
+ "id": 6,
+ "options": {
+ "annotations": {
+ "clustering": -1,
+ "multiLane": false
+ },
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "13.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr":
"disk_balancer_service_metrics_move_success_time_avg_time{instance=~\"$datanode\"}",
+ "legendFormat": "{{instance}} success avg latency",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr":
"disk_balancer_service_metrics_move_failure_time_avg_time{instance=~\"$datanode\"}",
+ "legendFormat": "{{instance}} failure avg latency",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr":
"disk_balancer_service_metrics_move_success_time_num_ops{instance=~\"$datanode\"}",
+ "legendFormat": "{{instance}} moves/interval",
+ "refId": "C"
+ }
+ ],
+ "title": "Container Migration Latency Profile (Processing Duration)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "description": "The live speedometer. This shows exactly how fast data
is physically copying from one disk to another across the node.",
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 0,
+ "y": 9
+ },
+ "id": 5,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "value",
+ "wideLayout": true
+ },
+ "pluginVersion": "13.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr":
"rate(disk_balancer_service_metrics_success_bytes{instance=~\"$datanode\"}[$__rate_interval])",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Active Balancing Throughput Speed",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "description": "The remaining amount of data the balancer still needs to
queue up to reach perfect balance.",
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 8,
+ "x": 6,
+ "y": 9
+ },
+ "id": 7,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "value",
+ "wideLayout": true
+ },
+ "pluginVersion": "13.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "max(sum({__name__=~\"volume_info_metrics_.*_ozone_used\",
instance=~\"$datanode\"}) by (instance, storagedirectory)) by (instance) -
avg(sum({__name__=~\"volume_info_metrics_.*_ozone_used\",
instance=~\"$datanode\"}) by (instance, storagedirectory)) by (instance)",
Review Comment:
This panel uses `max(...) - avg(...)`, but the description says “remaining
amount of data … to reach perfect balance.”
Those are not the same thing:
- **max − min** (used in the Skew panel) = spread between fullest and
emptiest disk.
- **max − avg** = how far only the *fullest* disk is above the node average.
When **multiple disks** are above average, max − avg **undercounts** total
bytes still to move.
Example: disks at 100 / 90 / 50 GB (avg = 80):
- max − avg = 20 GB
- actual backlog = (100−80) + (90−80) = **30 GB**
**Suggested fix** — sum per-volume excess above the node average:
```
sum by (instance) (
clamp_min(
sum by (instance, storagedirectory) (
{__name__=~"volume_info_metrics_.*_ozone_used", instance=~"$datanode"}
)
-
avg by (instance) (
sum by (instance, storagedirectory) (
{__name__=~"volume_info_metrics_.*_ozone_used",
instance=~"$datanode"}
)
),
0
)
)
```
The “Est. Time for Unscheduled Backlog” panel reuses the same max − avg
numerator, so its ETA will be wrong until that is updated too.
```
(
sum by (instance) (
clamp_min(
sum by (instance, storagedirectory) (
{__name__=~"volume_info_metrics_.*_ozone_used",
instance=~"$datanode"}
)
-
avg by (instance) (
sum by (instance, storagedirectory) (
{__name__=~"volume_info_metrics_.*_ozone_used",
instance=~"$datanode"}
)
),
0
)
)
/
clamp_min(
sum(rate(disk_balancer_service_metrics_success_bytes{instance=~"$datanode"}[5m])),
1
)
)
/ 60
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
