Gargi-jais11 commented on code in PR #10609:
URL: https://github.com/apache/ozone/pull/10609#discussion_r3748576358
##########
hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - Disk
Balancer.json:
##########
@@ -0,0 +1,967 @@
+{
+ "__inputs": [
+ {
+ "name": "DS_PROMETHEUS",
+ "label": "prometheus",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "prometheus",
+ "pluginName": "Prometheus"
+ }
+ ],
+ "__elements": {},
+ "__requires": [
+ {
+ "type": "panel",
+ "id": "bargauge",
+ "name": "Bar gauge",
+ "version": ""
+ },
+ {
+ "type": "panel",
+ "id": "gauge",
+ "name": "Gauge",
+ "version": ""
+ },
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "13.0.2"
+ },
+ {
+ "type": "panel",
+ "id": "piechart",
+ "name": "Pie chart",
+ "version": ""
+ },
+ {
+ "type": "datasource",
+ "id": "prometheus",
+ "name": "Prometheus",
+ "version": "1.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "stat",
+ "name": "Stat",
+ "version": ""
+ },
+ {
+ "type": "panel",
+ "id": "timeseries",
+ "name": "Time series",
+ "version": ""
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "grafana",
+ "uid": "-- Grafana --"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ },
+ {
+ "builtIn": 1,
+ "enable": true,
+ "hide": true,
+ "iconColor": "",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "",
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 1,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "description": "Shows how full each disk is over time. When the balancer
is running, you should see these lines slowly move closer together until they
meet in the middle.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "smooth",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "showValues": false,
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 12,
+ "x": 0,
+ "y": 0
+ },
+ "id": 2,
+ "options": {
+ "annotations": {
+ "clustering": -1,
+ "multiLane": false
+ },
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "13.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "sum({__name__=~\"volume_info_metrics_.*_ozone_used\",
instance=~\"$datanode\"}) by (instance, storagedirectory)",
+ "legendFormat": "{{storagedirectory}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Disk Space Convergence Profile (Historical Before vs After)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "description": "The exact size of the data gap between your most full
disk and your emptiest disk right now. The balancer's goal is to shrink this
number down to zero.",
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 12,
+ "x": 12,
+ "y": 0
+ },
+ "id": 3,
+ "options": {
+ "displayMode": "lcd",
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "maxVizHeight": 300,
+ "minVizHeight": 16,
+ "minVizWidth": 8,
+ "namePlacement": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showUnfilled": true,
+ "sizing": "auto",
+ "valueMode": "color"
+ },
+ "pluginVersion": "13.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "max(sum({__name__=~\"volume_info_metrics_.*_ozone_used\",
instance=~\"$datanode\"}) by (instance, storagedirectory)) by (instance) -
min(sum({__name__=~\"volume_info_metrics_.*_ozone_used\",
instance=~\"$datanode\"}) by (instance, storagedirectory)) by (instance)",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Current Disk Space Allocation Skew (Max - Min Volume Gap)",
+ "type": "bargauge"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "description": "How much time it takes the system to successfully copy,
verify, and finalize a single container.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "showValues": false,
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 24,
+ "x": 0,
+ "y": 4
+ },
+ "id": 6,
+ "options": {
+ "annotations": {
+ "clustering": -1,
+ "multiLane": false
+ },
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "13.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr":
"{__name__=~\"disk_balancer_service_metrics_move_success_time_avg_time|disk_balancer_service_metrics_move_failure_time_avg_time\",
instance=~\"$datanode\"}",
+ "legendFormat": "{{__name__}}",
+ "refId": "A"
+ }
+ ],
+ "title": "Container Migration Latency Profile (Processing Duration)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "description": "The live speedometer. This shows exactly how fast data
is physically copying from one disk to another across the node.",
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 6,
+ "x": 0,
+ "y": 9
+ },
+ "id": 5,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "value",
+ "wideLayout": true
+ },
+ "pluginVersion": "13.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr":
"rate(disk_balancer_service_metrics_success_bytes{instance=~\"$datanode\"}[1m])",
+ "refId": "A"
+ }
+ ],
+ "title": "Active Balancing Throughput Speed",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "description": "The remaining amount of data the balancer still needs to
queue up to reach perfect balance.",
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 8,
+ "x": 6,
+ "y": 9
+ },
+ "id": 7,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "value",
+ "wideLayout": true
+ },
+ "pluginVersion": "13.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr": "max(sum({__name__=~\"volume_info_metrics_.*_ozone_used\",
instance=~\"$datanode\"}) by (instance, storagedirectory)) by (instance) -
avg(sum({__name__=~\"volume_info_metrics_.*_ozone_used\",
instance=~\"$datanode\"}) by (instance, storagedirectory)) by (instance)",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Unscheduled Target Backlog",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "description": "A rough estimate of how many minutes are left before the
balancer finishes balancing.",
+ "fieldConfig": {
+ "defaults": {
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "m"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 5,
+ "w": 10,
+ "x": 14,
+ "y": 9
+ },
+ "id": 8,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "value",
+ "wideLayout": true
+ },
+ "pluginVersion": "13.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr":
"((sum(max(sum({__name__=~\"volume_info_metrics_.*_ozone_used\",
instance=~\"$datanode\"}) by (instance, storagedirectory)) by (instance) -
avg(sum({__name__=~\"volume_info_metrics_.*_ozone_used\",
instance=~\"$datanode\"}) by (instance, storagedirectory)) by (instance)) /
(sum(rate(disk_balancer_service_metrics_success_bytes{instance=~\"$datanode\"}[1m]))
> 0)) / 60) or vector(0)",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Est. Time for Unscheduled Backlog",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "description": "The percentage of container moves that finished
perfectly.",
+ "fieldConfig": {
+ "defaults": {
+ "max": 100,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 5,
+ "x": 0,
+ "y": 14
+ },
+ "id": 11,
+ "options": {
+ "barShape": "flat",
+ "barWidthFactor": 0.5,
+ "effects": {
+ "barGlow": false,
+ "centerGlow": false,
+ "gradient": true
+ },
+ "endpointMarker": "point",
+ "minVizHeight": 75,
+ "minVizWidth": 75,
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "segmentCount": 1,
+ "segmentSpacing": 0.3,
+ "shape": "gauge",
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true,
+ "sizing": "auto",
+ "sparkline": true,
+ "textMode": "auto"
+ },
+ "pluginVersion": "13.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr":
"(sum(disk_balancer_service_metrics_success_count{instance=~\"$datanode\"}) /
(sum(disk_balancer_service_metrics_success_count{instance=~\"$datanode\"}) +
sum(disk_balancer_service_metrics_failure_count{instance=~\"$datanode\"}) +
0.00001)) * 100",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Overall Move Success Rate",
+ "type": "gauge"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "description": "The total number of individual data blocks (containers)
successfully relocated by the balancer.",
+ "fieldConfig": {
+ "defaults": {
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 4,
+ "x": 5,
+ "y": 14
+ },
+ "id": 12,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "value",
+ "wideLayout": true
+ },
+ "pluginVersion": "13.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr":
"max_over_time(disk_balancer_service_metrics_success_count{instance=~\"$datanode\"}[1h])",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Total Lifetime Container Moves",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "description": "The total physical weight of all the data successfully
shifted across the disks since the balancer started.",
+ "fieldConfig": {
+ "defaults": {
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 5,
+ "x": 9,
+ "y": 14
+ },
+ "id": 14,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "13.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr":
"max_over_time(disk_balancer_service_metrics_success_bytes{instance=~\"$datanode\"}[1h])",
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Total Lifetime Data Balanced",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "description": "The number of times a move was aborted due to error.
This should ideally stay at 0.",
+ "fieldConfig": {
+ "defaults": {
+ "noValue": "0",
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": 0
+ },
+ {
+ "color": "red",
+ "value": 1
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 4,
+ "x": 14,
+ "y": 14
+ },
+ "id": 15,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "13.0.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${datasource}"
+ },
+ "expr":
"max_over_time(disk_balancer_service_metrics_failure_count{instance=~\"$datanode\"}[1h])",
Review Comment:
@navinko Cursor has suggested some fix for this:
Recommended fix (most reliable in Grafana Stat panels)
Use a range query + Last (not null) — no max_over_time, no [1h].
Total Lifetime Container Moves
`sum(disk_balancer_service_metrics_success_count{instance=~"$datanode"})`
Total Lifetime Data Balanced
`sum(disk_balancer_service_metrics_success_bytes{instance=~"$datanode"})`
Failed Container Moves
`sum(disk_balancer_service_metrics_failure_count{instance=~"$datanode"})`
Grafana target settings (JSON)
```
{
"expr":
"sum(disk_balancer_service_metrics_success_count{instance=~\"$datanode\"})",
"instant": false,
"range": true,
"refId": "A"
}
```
Keep the Stat panel reduce option:
```
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
}
```
Could u please try this ? I am not sure if it works or not. If this fixes
it's well and good else when I have an idea will open PR to fix this as well.
Not too critical blocking merge of this PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
