This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new e5d85ea0f feat(q-dev): add Kiro AI Model ROI dashboard (#8795)
e5d85ea0f is described below
commit e5d85ea0fe0a1e654585ca6e12a2ae96f181fcea
Author: Warren Chen <[email protected]>
AuthorDate: Sun Mar 22 22:36:02 2026 +0800
feat(q-dev): add Kiro AI Model ROI dashboard (#8795)
Add a Grafana dashboard analyzing per-model performance from chat logs:
- Model Performance Summary table (requests, share%, avg prompt/response
length, response/prompt ratio, steering/spec mode usage)
- Daily Model Usage Distribution (stacked bar chart)
- Avg Response Length by Model trend (output quality proxy)
Data source: _tool_q_dev_chat_log grouped by model_id.
---
grafana/dashboards/AIModelROI.json | 124 +++++++++++++++++++++++++++++++++++++
1 file changed, 124 insertions(+)
diff --git a/grafana/dashboards/AIModelROI.json
b/grafana/dashboards/AIModelROI.json
new file mode 100644
index 000000000..4ee15620b
--- /dev/null
+++ b/grafana/dashboards/AIModelROI.json
@@ -0,0 +1,124 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 1,
+ "id": null,
+ "links": [],
+ "panels": [
+ {
+ "datasource": "mysql",
+ "description": "Requests, avg prompt/response length, and usage share
per model",
+ "fieldConfig": {
+ "defaults": {
+ "color": { "mode": "thresholds" },
+ "custom": { "align": "auto", "cellOptions": { "type": "auto" },
"filterable": true },
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }] }
+ },
+ "overrides": []
+ },
+ "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 },
+ "id": 1,
+ "options": { "cellHeight": "sm", "showHeader": true, "sortBy": [] },
+ "targets": [
+ {
+ "datasource": "mysql",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "SELECT\n CASE WHEN model_id = '' OR model_id IS NULL
THEN '(unknown)' ELSE model_id END AS 'Model',\n COUNT(*) AS 'Requests',\n
ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM _tool_q_dev_chat_log WHERE
$__timeFilter(timestamp)), 1) AS 'Share %',\n ROUND(AVG(prompt_length)) AS
'Avg Prompt Len',\n ROUND(AVG(response_length)) AS 'Avg Response Len',\n
ROUND(AVG(response_length) / NULLIF(AVG(prompt_length), 0), 2) AS
'Response/Prompt Ratio',\n SUM(CASE WHEN has_ste [...]
+ "refId": "A"
+ }
+ ],
+ "title": "Model Performance Summary",
+ "type": "table"
+ },
+ {
+ "datasource": "mysql",
+ "description": "Request volume by model over time",
+ "fieldConfig": {
+ "defaults": {
+ "color": { "mode": "palette-classic" },
+ "custom": {
+ "drawStyle": "bars", "fillOpacity": 80, "lineWidth": 1,
+ "stacking": { "mode": "normal" }, "thresholdsStyle": { "mode":
"off" }
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": { "h": 8, "w": 24, "x": 0, "y": 8 },
+ "id": 2,
+ "options": {
+ "legend": { "calcs": ["sum"], "displayMode": "table", "placement":
"right", "showLegend": true },
+ "tooltip": { "mode": "multi" }
+ },
+ "targets": [
+ {
+ "datasource": "mysql",
+ "format": "time_series",
+ "rawQuery": true,
+ "rawSql": "SELECT DATE(timestamp) AS time,\n CASE WHEN model_id =
'' OR model_id IS NULL THEN '(unknown)' ELSE model_id END AS metric,\n
COUNT(*) AS value\nFROM _tool_q_dev_chat_log\nWHERE
$__timeFilter(timestamp)\nGROUP BY DATE(timestamp), model_id\nORDER BY time",
+ "refId": "A"
+ }
+ ],
+ "title": "Daily Model Usage Distribution",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "mysql",
+ "description": "Average response length per model over time — proxy for
output quality",
+ "fieldConfig": {
+ "defaults": {
+ "color": { "mode": "palette-classic" },
+ "custom": {
+ "drawStyle": "line", "fillOpacity": 10, "lineInterpolation":
"smooth", "lineWidth": 2,
+ "showPoints": "never", "spanNulls": true,
+ "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off"
}
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 },
+ "id": 3,
+ "options": {
+ "legend": { "calcs": ["mean"], "displayMode": "table", "placement":
"right", "showLegend": true },
+ "tooltip": { "mode": "multi" }
+ },
+ "targets": [
+ {
+ "datasource": "mysql",
+ "format": "time_series",
+ "rawQuery": true,
+ "rawSql": "SELECT DATE(timestamp) AS time,\n CASE WHEN model_id =
'' OR model_id IS NULL THEN '(unknown)' ELSE model_id END AS metric,\n
ROUND(AVG(response_length)) AS value\nFROM _tool_q_dev_chat_log\nWHERE
$__timeFilter(timestamp)\nGROUP BY DATE(timestamp), model_id\nORDER BY time",
+ "refId": "A"
+ }
+ ],
+ "title": "Avg Response Length by Model (Daily)",
+ "type": "timeseries"
+ }
+ ],
+ "preload": false,
+ "refresh": "5m",
+ "schemaVersion": 41,
+ "tags": ["q_dev", "kiro", "model", "roi"],
+ "templating": { "list": [] },
+ "time": { "from": "now-90d", "to": "now" },
+ "timepicker": {},
+ "timezone": "utc",
+ "title": "Kiro AI Model ROI",
+ "uid": "kiro_model_roi",
+ "version": 1
+}
