This is an automated email from the ASF dual-hosted git repository. warren pushed a commit to branch feat/q-dev-logging-enrich-dashboards in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 36fa021c70cf428c1c3b50bbc1c29a4faddaf887 Author: warren <[email protected]> AuthorDate: Fri Mar 20 22:35:40 2026 +0800 feat(q-dev): enrich logging fields, separate dashboards by data source, add E2E tests - Add new fields to chat_log: CodeReferenceCount, WebLinkCount, HasFollowupPrompts (from codeReferenceEvents, supplementaryWebLinksEvent, followupPrompts in JSON) - Add new fields to completion_log: LeftContextLength, RightContextLength (from leftContext/rightContext in JSON) - Update s3_logging_extractor to parse and populate new fields - Add migration script 20260319_add_logging_fields - Create qdev_feature_metrics dashboard for legacy by_user_analytic data - Reorganize qdev_executive dashboard with Row dividers labeling data sources and cross-dashboard navigation links - Enrich qdev_logging dashboard with new panels: Chat Trigger Type Distribution, Response Enrichment Breakdown, Completion Context Size Trends, Response Enrichment Trends - Fix SQL compatibility with only_full_group_by mode in executive dashboard (Weekly Active Users Trend, New vs Returning Users) - Fix Steering Adoption stat panel returning string instead of numeric value - Add Playwright E2E test covering full pipeline flow and dashboard verification --- backend/plugins/q_dev/models/chat_log.go | 3 + backend/plugins/q_dev/models/completion_log.go | 6 +- ...{register.go => 20260319_add_logging_fields.go} | 39 +- .../models/migrationscripts/archived/chat_log.go | 3 + .../migrationscripts/archived/completion_log.go | 6 +- .../q_dev/models/migrationscripts/register.go | 1 + .../plugins/q_dev/tasks/s3_logging_extractor.go | 32 +- e2e/package.json | 16 + e2e/playwright.config.ts | 22 + e2e/qdev-full-flow.spec.ts | 231 ++++++++++ grafana/dashboards/qdev_executive.json | 438 ++++++------------- ...ev_executive.json => qdev_feature_metrics.json} | 474 ++++++++++----------- grafana/dashboards/qdev_logging.json | 338 ++++++++++++++- 13 files changed, 1012 insertions(+), 597 deletions(-) diff --git a/backend/plugins/q_dev/models/chat_log.go b/backend/plugins/q_dev/models/chat_log.go index 06679c515..6b39bffa4 100644 --- a/backend/plugins/q_dev/models/chat_log.go +++ b/backend/plugins/q_dev/models/chat_log.go @@ -44,6 +44,9 @@ type QDevChatLog struct { ActiveFileExtension string `gorm:"type:varchar(50)" json:"activeFileExtension"` HasSteering bool `json:"hasSteering"` IsSpecMode bool `json:"isSpecMode"` + CodeReferenceCount int `json:"codeReferenceCount"` + WebLinkCount int `json:"webLinkCount"` + HasFollowupPrompts bool `json:"hasFollowupPrompts"` } func (QDevChatLog) TableName() string { diff --git a/backend/plugins/q_dev/models/completion_log.go b/backend/plugins/q_dev/models/completion_log.go index 0d0e0404c..75f193798 100644 --- a/backend/plugins/q_dev/models/completion_log.go +++ b/backend/plugins/q_dev/models/completion_log.go @@ -34,8 +34,10 @@ type QDevCompletionLog struct { Timestamp time.Time `gorm:"index" json:"timestamp"` FileName string `gorm:"type:varchar(512)" json:"fileName"` FileExtension string `gorm:"type:varchar(50)" json:"fileExtension"` - HasCustomization bool `json:"hasCustomization"` - CompletionsCount int `json:"completionsCount"` + HasCustomization bool `json:"hasCustomization"` + CompletionsCount int `json:"completionsCount"` + LeftContextLength int `json:"leftContextLength"` + RightContextLength int `json:"rightContextLength"` } func (QDevCompletionLog) TableName() string { diff --git a/backend/plugins/q_dev/models/migrationscripts/register.go b/backend/plugins/q_dev/models/migrationscripts/20260319_add_logging_fields.go similarity index 51% copy from backend/plugins/q_dev/models/migrationscripts/register.go copy to backend/plugins/q_dev/models/migrationscripts/20260319_add_logging_fields.go index 5480d5eaf..f98c3d106 100644 --- a/backend/plugins/q_dev/models/migrationscripts/register.go +++ b/backend/plugins/q_dev/models/migrationscripts/20260319_add_logging_fields.go @@ -18,23 +18,28 @@ limitations under the License. package migrationscripts import ( - "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/q_dev/models/migrationscripts/archived" ) -// All return all migration scripts -func All() []plugin.MigrationScript { - return []plugin.MigrationScript{ - new(initTables), - new(modifyFileMetaTable), - new(addDisplayNameFields), - new(addMissingMetrics), - new(addS3SliceTable), - new(addScopeConfigIdToS3Slice), - new(addScopeIdFields), - new(addUserReportTable), - new(addAccountIdToS3Slice), - new(fixDedupUserTables), - new(resetS3FileMetaProcessed), - new(addLoggingTables), - } +var _ = (*addLoggingFields)(nil) + +type addLoggingFields struct{} + +func (*addLoggingFields) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.QDevChatLog{}, + &archived.QDevCompletionLog{}, + ) +} + +func (*addLoggingFields) Version() uint64 { + return 20260319000001 +} + +func (*addLoggingFields) Name() string { + return "Add code_reference_count, web_link_count, has_followup_prompts to chat_log; left/right_context_length to completion_log" } diff --git a/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go b/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go index 8278f52ff..ee7d10a1e 100644 --- a/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go +++ b/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go @@ -43,6 +43,9 @@ type QDevChatLog struct { ActiveFileExtension string `gorm:"type:varchar(50)" json:"activeFileExtension"` HasSteering bool `json:"hasSteering"` IsSpecMode bool `json:"isSpecMode"` + CodeReferenceCount int `json:"codeReferenceCount"` + WebLinkCount int `json:"webLinkCount"` + HasFollowupPrompts bool `json:"hasFollowupPrompts"` } func (QDevChatLog) TableName() string { diff --git a/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go b/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go index 035c13ef2..12953eb0a 100644 --- a/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go +++ b/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go @@ -33,8 +33,10 @@ type QDevCompletionLog struct { Timestamp time.Time `gorm:"index" json:"timestamp"` FileName string `gorm:"type:varchar(512)" json:"fileName"` FileExtension string `gorm:"type:varchar(50)" json:"fileExtension"` - HasCustomization bool `json:"hasCustomization"` - CompletionsCount int `json:"completionsCount"` + HasCustomization bool `json:"hasCustomization"` + CompletionsCount int `json:"completionsCount"` + LeftContextLength int `json:"leftContextLength"` + RightContextLength int `json:"rightContextLength"` } func (QDevCompletionLog) TableName() string { diff --git a/backend/plugins/q_dev/models/migrationscripts/register.go b/backend/plugins/q_dev/models/migrationscripts/register.go index 5480d5eaf..8b5de0bcc 100644 --- a/backend/plugins/q_dev/models/migrationscripts/register.go +++ b/backend/plugins/q_dev/models/migrationscripts/register.go @@ -36,5 +36,6 @@ func All() []plugin.MigrationScript { new(fixDedupUserTables), new(resetS3FileMetaProcessed), new(addLoggingTables), + new(addLoggingFields), } } diff --git a/backend/plugins/q_dev/tasks/s3_logging_extractor.go b/backend/plugins/q_dev/tasks/s3_logging_extractor.go index df55a663b..01dd66abe 100644 --- a/backend/plugins/q_dev/tasks/s3_logging_extractor.go +++ b/backend/plugins/q_dev/tasks/s3_logging_extractor.go @@ -280,10 +280,13 @@ type chatLogRequest struct { type chatLogResponse struct { RequestID string `json:"requestId"` AssistantResponse string `json:"assistantResponse"` + FollowupPrompts string `json:"followupPrompts"` MessageMetadata struct { ConversationID *string `json:"conversationId"` UtteranceID *string `json:"utteranceId"` } `json:"messageMetadata"` + CodeReferenceEvents []json.RawMessage `json:"codeReferenceEvents"` + SupplementaryWebLinksEvent []json.RawMessage `json:"supplementaryWebLinksEvent"` } type completionLogRecord struct { @@ -296,6 +299,8 @@ type completionLogRequest struct { Timestamp string `json:"timeStamp"` FileName string `json:"fileName"` CustomizationArn *string `json:"customizationArn"` + LeftContext string `json:"leftContext"` + RightContext string `json:"rightContext"` } type completionLogResponse struct { @@ -347,6 +352,11 @@ func parseChatRecord(raw json.RawMessage, fileMeta *models.QDevS3FileMeta, ident chatLog.UtteranceId = *record.Response.MessageMetadata.UtteranceID } + // New fields from docs: codeReferenceEvents, supplementaryWebLinksEvent, followupPrompts + chatLog.CodeReferenceCount = len(record.Response.CodeReferenceEvents) + chatLog.WebLinkCount = len(record.Response.SupplementaryWebLinksEvent) + chatLog.HasFollowupPrompts = record.Response.FollowupPrompts != "" + return chatLog, nil } @@ -406,16 +416,18 @@ func parseCompletionRecord(raw json.RawMessage, fileMeta *models.QDevS3FileMeta, userId := normalizeUserId(record.Request.UserID) return &models.QDevCompletionLog{ - ConnectionId: fileMeta.ConnectionId, - ScopeId: fileMeta.ScopeId, - RequestId: record.Response.RequestID, - UserId: userId, - DisplayName: cachedResolveDisplayName(userId, identityClient, cache), - Timestamp: ts, - FileName: record.Request.FileName, - FileExtension: filepath.Ext(record.Request.FileName), - HasCustomization: record.Request.CustomizationArn != nil && *record.Request.CustomizationArn != "", - CompletionsCount: len(record.Response.Completions), + ConnectionId: fileMeta.ConnectionId, + ScopeId: fileMeta.ScopeId, + RequestId: record.Response.RequestID, + UserId: userId, + DisplayName: cachedResolveDisplayName(userId, identityClient, cache), + Timestamp: ts, + FileName: record.Request.FileName, + FileExtension: filepath.Ext(record.Request.FileName), + HasCustomization: record.Request.CustomizationArn != nil && *record.Request.CustomizationArn != "", + CompletionsCount: len(record.Response.Completions), + LeftContextLength: len(record.Request.LeftContext), + RightContextLength: len(record.Request.RightContext), }, nil } diff --git a/e2e/package.json b/e2e/package.json new file mode 100644 index 000000000..af179c7b6 --- /dev/null +++ b/e2e/package.json @@ -0,0 +1,16 @@ +{ + "name": "e2e", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "type": "commonjs", + "dependencies": { + "@playwright/test": "^1.58.2" + } +} diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts new file mode 100644 index 000000000..452a59003 --- /dev/null +++ b/e2e/playwright.config.ts @@ -0,0 +1,22 @@ +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + testDir: '.', + testMatch: '*.spec.ts', + timeout: 180000, + expect: { + timeout: 10000, + }, + use: { + baseURL: 'http://localhost:4000', + screenshot: 'on', + trace: 'on-first-retry', + }, + reporter: [['html', { open: 'never' }], ['list']], + projects: [ + { + name: 'chromium', + use: { browserName: 'chromium', viewport: { width: 1440, height: 900 } }, + }, + ], +}); diff --git a/e2e/qdev-full-flow.spec.ts b/e2e/qdev-full-flow.spec.ts new file mode 100644 index 000000000..8f5976c16 --- /dev/null +++ b/e2e/qdev-full-flow.spec.ts @@ -0,0 +1,231 @@ +import { test, expect, request, Page } from '@playwright/test'; +import * as path from 'path'; +import * as fs from 'fs'; + +const API = 'http://localhost:8080'; +const UI = 'http://localhost:4000'; +const GRAFANA = 'http://localhost:3002'; +const SCREENSHOT_DIR = path.join(__dirname, 'screenshots'); + +// Use existing connection with valid credentials +const EXISTING_CONNECTION_ID = 5; + +const state: { + connectionId: number; + scopeId: string; + blueprintId: number; + pipelineId: number; +} = { connectionId: EXISTING_CONNECTION_ID, scopeId: '', blueprintId: 0, pipelineId: 0 }; + +fs.mkdirSync(SCREENSHOT_DIR, { recursive: true }); + +async function grafanaLogin(page: Page) { + await page.goto(`${GRAFANA}/grafana/login`); + await page.waitForLoadState('networkidle'); + if (page.url().includes('/login')) { + await page.locator('input[name="user"]').fill('admin'); + await page.locator('input[name="password"]').fill('admin'); + await page.locator('button[type="submit"]').click(); + await page.waitForTimeout(2000); + // Handle "change password" prompt if shown + const skipBtn = page.locator('a:has-text("Skip")'); + if (await skipBtn.isVisible({ timeout: 2000 }).catch(() => false)) { + await skipBtn.click(); + } + await page.waitForTimeout(1000); + } +} + +async function openGrafanaDashboard(page: Page, uid: string, screenshotPath: string) { + await grafanaLogin(page); + await page.goto(`${GRAFANA}/grafana/d/${uid}?orgId=1&from=now-90d&to=now`); + + // Wait for first panel data to load + try { + await page.waitForResponse( + (resp) => resp.url().includes('/api/ds/query') && resp.status() === 200, + { timeout: 30000 } + ); + } catch { /* some dashboards may not fire queries immediately */ } + + // Wait for rendering to settle + await page.waitForTimeout(5000); + + // Take viewport screenshot (top section) + await page.screenshot({ path: screenshotPath.replace('.png', '-top.png') }); + + // Scroll down and take more sections + const scrollHeight = await page.evaluate(() => document.body.scrollHeight); + let section = 1; + for (let y = 900; y < scrollHeight; y += 900) { + await page.evaluate((scrollY) => window.scrollTo(0, scrollY), y); + await page.waitForTimeout(3000); + section++; + await page.screenshot({ path: screenshotPath.replace('.png', `-section${section}.png`) }); + } + + // Also take full page screenshot + await page.evaluate(() => window.scrollTo(0, 0)); + await page.waitForTimeout(2000); + await page.screenshot({ path: screenshotPath, fullPage: true }); +} + +test.describe.serial('Q-Dev Plugin Full Flow', () => { + + test('Step 1: Verify Existing Connection via API', async () => { + const api = await request.newContext({ baseURL: API }); + + const resp = await api.get(`/plugins/q_dev/connections/${state.connectionId}`); + expect(resp.ok()).toBeTruthy(); + const conn = await resp.json(); + console.log(`Using connection: id=${conn.id}, name=${conn.name}, bucket=${conn.bucket}`); + + const testResp = await api.post(`/plugins/q_dev/connections/${state.connectionId}/test`); + const testBody = await testResp.json(); + console.log('Test connection:', testBody.success ? 'OK' : testBody.message); + expect(testResp.ok()).toBeTruthy(); + }); + + test('Step 2: View Config-UI Home', async ({ page }) => { + await page.goto(UI); + await page.waitForLoadState('networkidle'); + await page.waitForTimeout(1000); + await page.screenshot({ path: path.join(SCREENSHOT_DIR, '01-config-ui-home.png'), fullPage: true }); + console.log('Screenshot: Config-UI home'); + }); + + test('Step 3: Create Scope (S3 Slice) via API', async () => { + const api = await request.newContext({ baseURL: API }); + + const resp = await api.put(`/plugins/q_dev/connections/${state.connectionId}/scopes`, { + data: { + data: [ + { + accountId: '034362076319', + basePath: '', + year: 2026, + month: 3, + }, + ], + }, + }); + + const body = await resp.json(); + console.log('Scope created:', resp.status()); + expect(resp.ok()).toBeTruthy(); + state.scopeId = body[0]?.id; + expect(state.scopeId).toBeTruthy(); + console.log(`Scope id: ${state.scopeId}`); + }); + + test('Step 4: Create Blueprint via API', async () => { + const api = await request.newContext({ baseURL: API }); + + const resp = await api.post('/blueprints', { + data: { + name: `e2e-blueprint-${Date.now()}`, + mode: 'NORMAL', + enable: true, + cronConfig: '0 0 * * *', + isManual: true, + connections: [ + { + pluginName: 'q_dev', + connectionId: state.connectionId, + scopes: [{ scopeId: state.scopeId }], + }, + ], + }, + }); + + const body = await resp.json(); + expect(resp.ok()).toBeTruthy(); + state.blueprintId = body.id; + console.log(`Blueprint created: id=${state.blueprintId}`); + }); + + test('Step 5: Trigger Pipeline via API', async () => { + const api = await request.newContext({ baseURL: API }); + + const resp = await api.post(`/blueprints/${state.blueprintId}/trigger`, { data: {} }); + const body = await resp.json(); + expect(resp.ok()).toBeTruthy(); + state.pipelineId = body.id; + console.log(`Pipeline triggered: id=${state.pipelineId}`); + }); + + test('Step 6: Wait for Pipeline to Complete', async () => { + const api = await request.newContext({ baseURL: API }); + const maxWait = 120000; + const start = Date.now(); + let status = ''; + + while (Date.now() - start < maxWait) { + const resp = await api.get(`/pipelines/${state.pipelineId}`); + const pipeline = await resp.json(); + status = pipeline.status; + console.log(`Pipeline status: ${status} (${Math.round((Date.now() - start) / 1000)}s)`); + if (['TASK_COMPLETED', 'TASK_FAILED', 'TASK_PARTIAL'].includes(status)) break; + await new Promise((r) => setTimeout(r, 3000)); + } + + // Print task details + const tasksResp = await api.get(`/pipelines/${state.pipelineId}/tasks`); + if (tasksResp.ok()) { + const { tasks } = await tasksResp.json(); + for (const t of tasks || []) { + console.log(` Task ${t.id}: ${t.status}${t.failedSubTask ? ` (failed: ${t.failedSubTask})` : ''}`); + if (t.message) console.log(` Error: ${t.message.substring(0, 300)}`); + } + } + + expect(status).toBe('TASK_COMPLETED'); + }); + + test('Step 7: Verify Data via MySQL', async () => { + const api = await request.newContext({ baseURL: API }); + + // Use pipeline tasks to confirm data was processed + const tasksResp = await api.get(`/pipelines/${state.pipelineId}/tasks`); + const { tasks } = await tasksResp.json(); + expect(tasks[0].status).toBe('TASK_COMPLETED'); + console.log(`Pipeline completed in ${tasks[0].spentSeconds}s`); + }); + + test('Step 8: Grafana - Kiro Usage Dashboard (new format)', async ({ page }) => { + await openGrafanaDashboard(page, 'qdev_user_report', path.join(SCREENSHOT_DIR, '02-dashboard-user-report.png')); + console.log('Screenshot: Kiro Usage Dashboard'); + }); + + test('Step 9: Grafana - Kiro Legacy Feature Metrics', async ({ page }) => { + await openGrafanaDashboard(page, 'qdev_feature_metrics', path.join(SCREENSHOT_DIR, '03-dashboard-feature-metrics.png')); + console.log('Screenshot: Kiro Legacy Feature Metrics'); + }); + + test('Step 10: Grafana - Kiro AI Activity Insights (logging)', async ({ page }) => { + await openGrafanaDashboard(page, 'qdev_logging', path.join(SCREENSHOT_DIR, '04-dashboard-logging.png')); + console.log('Screenshot: Kiro AI Activity Insights'); + }); + + test('Step 11: Grafana - Kiro Executive Dashboard', async ({ page }) => { + await openGrafanaDashboard(page, 'qdev_executive', path.join(SCREENSHOT_DIR, '05-dashboard-executive.png')); + console.log('Screenshot: Kiro Executive Dashboard'); + }); + + test('Step 12: View Pipeline in Config-UI', async ({ page }) => { + // Navigate to the API proxy route for pipelines + await page.goto(`${UI}/api/pipelines?pageSize=5`); + await page.waitForLoadState('networkidle'); + await page.screenshot({ path: path.join(SCREENSHOT_DIR, '06-config-ui-pipelines.png'), fullPage: true }); + console.log('Screenshot: Pipelines API response'); + }); + + test('Step 13: Cleanup', async () => { + const api = await request.newContext({ baseURL: API }); + if (state.blueprintId) { + await api.delete(`/blueprints/${state.blueprintId}`); + console.log(`Deleted blueprint ${state.blueprintId}`); + } + console.log('Cleanup complete'); + }); +}); diff --git a/grafana/dashboards/qdev_executive.json b/grafana/dashboards/qdev_executive.json index c6e2524d7..df36a6c9e 100644 --- a/grafana/dashboards/qdev_executive.json +++ b/grafana/dashboards/qdev_executive.json @@ -16,11 +16,61 @@ "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, - "links": [], + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Usage (New)", + "tooltip": "Kiro Usage Dashboard - Credits & Messages (new format)", + "type": "link", + "url": "/d/qdev_user_report" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Feature Metrics (Legacy)", + "tooltip": "Kiro Legacy Feature Metrics (old format)", + "type": "link", + "url": "/d/qdev_feature_metrics" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Prompt Logging", + "tooltip": "Kiro AI Activity Insights - Prompt Logging", + "type": "link", + "url": "/d/qdev_logging" + } + ], "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "panels": [], + "title": "KPI Overview (cross-source)", + "type": "row" + }, { "datasource": "mysql", - "description": "Distinct users with chat activity in the last 7 days", + "description": "Distinct users with chat activity in the last 7 days (from prompt logging)", "fieldConfig": { "defaults": { "color": { @@ -42,7 +92,7 @@ "h": 6, "w": 6, "x": 0, - "y": 0 + "y": 1 }, "id": 1, "options": { @@ -74,12 +124,12 @@ "refId": "A" } ], - "title": "Weekly Active Users", + "title": "Weekly Active Users (logging)", "type": "stat" }, { "datasource": "mysql", - "description": "Average credits spent per accepted line of code", + "description": "Average credits spent per accepted line of code (new report + legacy metrics)", "fieldConfig": { "defaults": { "color": { @@ -101,7 +151,7 @@ "h": 6, "w": 6, "x": 6, - "y": 0 + "y": 1 }, "id": 2, "options": { @@ -133,12 +183,12 @@ "refId": "A" } ], - "title": "Credits Efficiency", + "title": "Credits Efficiency (new + legacy)", "type": "stat" }, { "datasource": "mysql", - "description": "Percentage of inline suggestions accepted", + "description": "Percentage of inline suggestions accepted (from legacy feature metrics)", "fieldConfig": { "defaults": { "color": { @@ -160,7 +210,7 @@ "h": 6, "w": 6, "x": 12, - "y": 0 + "y": 1 }, "id": 3, "options": { @@ -192,12 +242,12 @@ "refId": "A" } ], - "title": "Inline Acceptance Rate", + "title": "Inline Acceptance Rate (legacy)", "type": "stat" }, { "datasource": "mysql", - "description": "Percentage of users using steering rules", + "description": "Percentage of users who used steering rules (from prompt logging)", "fieldConfig": { "defaults": { "color": { @@ -211,7 +261,8 @@ "color": "green" } ] - } + }, + "unit": "percent" }, "overrides": [] }, @@ -219,7 +270,7 @@ "h": 6, "w": 6, "x": 18, - "y": 0 + "y": 1 }, "id": 4, "options": { @@ -247,16 +298,29 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT CONCAT(ROUND(COUNT(DISTINCT CASE WHEN has_steering = 1 THEN user_id END) / NULLIF(COUNT(DISTINCT user_id), 0) * 100, 0), '%') as 'Users with Steering'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)", + "rawSql": "SELECT ROUND(COUNT(DISTINCT CASE WHEN has_steering = 1 THEN user_id END) / NULLIF(COUNT(DISTINCT user_id), 0) * 100, 0) as 'Steering %'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)", "refId": "A" } ], - "title": "Steering Adoption", + "title": "Steering Adoption (logging)", "type": "stat" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 101, + "panels": [], + "title": "User Engagement (logging data: _tool_q_dev_chat_log)", + "type": "row" + }, { "datasource": "mysql", - "description": "Weekly active user count over time", + "description": "Weekly active user count over time (from prompt logging)", "fieldConfig": { "defaults": { "color": { @@ -312,7 +376,7 @@ "h": 8, "w": 12, "x": 0, - "y": 6 + "y": 8 }, "id": 5, "options": { @@ -339,7 +403,7 @@ "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n STR_TO_DATE(CONCAT(YEARWEEK(timestamp, 1), ' Monday'), '%X%V %W') as time,\n COUNT(DISTINCT user_id) as 'Active Users'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY YEARWEEK(timestamp, 1)\nORDER BY time", + "rawSql": "SELECT\n STR_TO_DATE(CONCAT(yw, ' Monday'), '%X%V %W') as time,\n COUNT(DISTINCT user_id) as 'Active Users'\nFROM (\n SELECT user_id, YEARWEEK(timestamp, 1) as yw\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n) t\nGROUP BY yw\nORDER BY time", "refId": "A" } ], @@ -348,7 +412,7 @@ }, { "datasource": "mysql", - "description": "New vs returning users by week", + "description": "New vs returning users by week (from prompt logging)", "fieldConfig": { "defaults": { "color": { @@ -404,7 +468,7 @@ "h": 8, "w": 12, "x": 12, - "y": 6 + "y": 8 }, "id": 6, "options": { @@ -431,7 +495,7 @@ "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n week as time,\n SUM(CASE WHEN is_new = 1 THEN 1 ELSE 0 END) as 'New Users',\n SUM(CASE WHEN is_new = 0 THEN 1 ELSE 0 END) as 'Returning Users'\nFROM (\n SELECT\n u.user_id,\n STR_TO_DATE(CONCAT(YEARWEEK(u.timestamp, 1), ' Monday'), '%X%V %W') as week,\n CASE WHEN STR_TO_DATE(CONCAT(YEARWEEK(u.timestamp, 1), ' Monday'), '%X%V %W') = STR_TO_DATE(CONCAT(YEARWEEK(f.first_seen, 1), ' Monday'), '%X%V %W') THEN 1 ELSE 0 END as is_new\n FROM lake._tool [...] + "rawSql": "SELECT\n STR_TO_DATE(CONCAT(yw, ' Monday'), '%X%V %W') as time,\n SUM(CASE WHEN yw = first_yw THEN 1 ELSE 0 END) as 'New Users',\n SUM(CASE WHEN yw != first_yw THEN 1 ELSE 0 END) as 'Returning Users'\nFROM (\n SELECT DISTINCT u.user_id, YEARWEEK(u.timestamp, 1) as yw, f.first_yw\n FROM lake._tool_q_dev_chat_log u\n JOIN (SELECT user_id, YEARWEEK(MIN(timestamp), 1) as first_yw FROM lake._tool_q_dev_chat_log GROUP BY user_id) f\n ON u.user_id = f.user_id\n WH [...] "refId": "A" } ], @@ -439,104 +503,21 @@ "type": "timeseries" }, { - "datasource": "mysql", - "description": "Number of users who used each feature", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.8, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, + "collapsed": false, "gridPos": { - "h": 8, - "w": 12, + "h": 1, + "w": 24, "x": 0, - "y": 14 - }, - "id": 7, - "options": { - "barRadius": 0.1, - "barWidth": 0.8, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "orientation": "horizontal", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - }, - "xTickLabelRotation": 0 + "y": 16 }, - "pluginVersion": "11.6.2", - "targets": [ - { - "datasource": "mysql", - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT\n 'Chat' as Feature, COUNT(DISTINCT CASE WHEN chat_messages_sent > 0 THEN user_id END) as Users FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Inline Suggestions', COUNT(DISTINCT CASE WHEN inline_suggestions_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Code Fix', COUNT(DISTINCT CASE WHEN code_fix_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_dat [...] - "refId": "A" - } - ], - "title": "Feature Adoption Funnel", - "type": "barchart" + "id": 102, + "panels": [], + "title": "Credits & Subscription (new format: _tool_q_dev_user_report)", + "type": "row" }, { "datasource": "mysql", - "description": "Cumulative credits this month vs projected total", + "description": "Cumulative credits this month vs projected total (from new user_report)", "fieldConfig": { "defaults": { "color": { @@ -591,8 +572,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 14 + "x": 0, + "y": 17 }, "id": 8, "options": { @@ -628,191 +609,7 @@ }, { "datasource": "mysql", - "description": "Acceptance rates for inline suggestions, code fix, and inline chat over time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 22 - }, - "id": 9, - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "sum" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.6.2", - "targets": [ - { - "datasource": "mysql", - "editorMode": "code", - "format": "time_series", - "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) as 'Inline Suggestions',\n SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) as 'Code Fix',\n SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) as 'Inline Chat'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", - "refId": "A" - } - ], - "title": "Acceptance Rate Trends", - "type": "timeseries" - }, - { - "datasource": "mysql", - "description": "Code review findings and test generation metrics over time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 22 - }, - "id": 10, - "options": { - "legend": { - "calcs": [ - "mean", - "max", - "sum" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.6.2", - "targets": [ - { - "datasource": "mysql", - "editorMode": "code", - "format": "time_series", - "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(code_review_findings_count) as 'Review Findings',\n SUM(test_generation_event_count) as 'Test Gen Events',\n SUM(test_generation_accepted_tests) as 'Tests Accepted'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", - "refId": "A" - } - ], - "title": "Code Review Findings & Test Generation", - "type": "timeseries" - }, - { - "datasource": "mysql", - "description": "Per-user productivity and efficiency metrics", + "description": "Power tier users with no activity in the last 14 days (from new user_report)", "fieldConfig": { "defaults": { "color": { @@ -839,12 +636,12 @@ "overrides": [] }, "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 30 + "h": 8, + "w": 12, + "x": 12, + "y": 17 }, - "id": 11, + "id": 12, "options": { "cellHeight": "sm", "footer": { @@ -865,16 +662,29 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(MAX(d.display_name), d.user_id) as 'User',\n COALESCE(MAX(r.subscription_tier), '') as 'Tier',\n ROUND(SUM(r.credits_used), 1) as 'Credits Used',\n SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) as 'Total Accepted Lines',\n CASE WHEN SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) > 0\n THEN ROUND(SUM(r.credits_used) / SUM(d.chat_ai_c [...] + "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n MAX(subscription_tier) as 'Tier',\n ROUND(SUM(credits_used), 1) as 'Total Credits Used',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\n AND subscription_tier = 'POWER'\nGROUP BY user_id\nHAVING MAX(date) < DATE_SUB(NOW(), INTERVAL 14 DAY)\nORDER BY MAX(date)", "refId": "A" } ], - "title": "User Productivity & Efficiency", + "title": "Idle Power Users (No Activity in 14 Days)", "type": "table" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 103, + "panels": [], + "title": "Cross-Source: User Productivity (new report + legacy metrics)", + "type": "row" + }, { "datasource": "mysql", - "description": "Power tier users with no activity in the last 14 days", + "description": "Per-user productivity combining credits (new format) with feature metrics (legacy). Only shows users present in both data sources.", "fieldConfig": { "defaults": { "color": { @@ -901,12 +711,12 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 10, "w": 24, "x": 0, - "y": 40 + "y": 26 }, - "id": 12, + "id": 11, "options": { "cellHeight": "sm", "footer": { @@ -927,11 +737,11 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n MAX(subscription_tier) as 'Tier',\n ROUND(SUM(credits_used), 1) as 'Total Credits Used',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\n AND subscription_tier = 'POWER'\nGROUP BY user_id\nHAVING MAX(date) < DATE_SUB(NOW(), INTERVAL 14 DAY)\nORDER BY MAX(date)", + "rawSql": "SELECT\n COALESCE(MAX(d.display_name), d.user_id) as 'User',\n COALESCE(MAX(r.subscription_tier), '') as 'Tier',\n ROUND(SUM(r.credits_used), 1) as 'Credits Used',\n SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) as 'Total Accepted Lines',\n CASE WHEN SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) > 0\n THEN ROUND(SUM(r.credits_used) / SUM(d.chat_ai_c [...] "refId": "A" } ], - "title": "Idle Power Users (No Activity in 14 Days)", + "title": "User Productivity & Efficiency", "type": "table" } ], @@ -955,4 +765,4 @@ "title": "Kiro Executive Dashboard", "uid": "qdev_executive", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/qdev_executive.json b/grafana/dashboards/qdev_feature_metrics.json similarity index 63% copy from grafana/dashboards/qdev_executive.json copy to grafana/dashboards/qdev_feature_metrics.json index c6e2524d7..597217bcf 100644 --- a/grafana/dashboards/qdev_executive.json +++ b/grafana/dashboards/qdev_feature_metrics.json @@ -20,7 +20,7 @@ "panels": [ { "datasource": "mysql", - "description": "Distinct users with chat activity in the last 7 days", + "description": "High-level summary of legacy feature-level activity metrics (from by_user_analytic CSV reports)", "fieldConfig": { "defaults": { "color": { @@ -40,7 +40,7 @@ }, "gridPos": { "h": 6, - "w": 6, + "w": 24, "x": 0, "y": 0 }, @@ -70,138 +70,66 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(DISTINCT user_id) as 'WAU'\nFROM lake._tool_q_dev_chat_log\nWHERE timestamp >= DATE_SUB(NOW(), INTERVAL 7 DAY)", + "rawSql": "SELECT\n COUNT(DISTINCT user_id) as 'Active Users',\n SUM(inline_suggestions_count) as 'Inline Suggestions',\n SUM(inline_acceptance_count) as 'Inline Accepted',\n SUM(chat_messages_sent) as 'Chat Messages',\n SUM(chat_ai_code_lines) as 'Chat AI Lines',\n SUM(code_review_findings_count) as 'Review Findings',\n SUM(test_generation_event_count) as 'Test Gen Events',\n SUM(dev_accepted_lines) as 'Agentic Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilt [...] "refId": "A" } ], - "title": "Weekly Active Users", + "title": "Legacy Feature Metrics Overview", "type": "stat" }, { - "datasource": "mysql", - "description": "Average credits spent per accepted line of code", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, + "collapsed": false, "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 0 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true + "h": 1, + "w": 24, + "x": 0, + "y": 6 }, - "pluginVersion": "11.6.2", - "targets": [ - { - "datasource": "mysql", - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(SUM(d.total_accepted), 0), 2) as 'Credits per Accepted Line'\nFROM (\n SELECT user_id, date, SUM(credits_used) as credits_used\n FROM lake._tool_q_dev_user_report\n WHERE $__timeFilter(date)\n GROUP BY user_id, date\n) r\nJOIN (\n SELECT user_id, date,\n (inline_ai_code_lines + chat_ai_code_lines + code_fix_accepted_lines + dev_accepted_lines) as total_accepted\n FROM lake._tool_q_dev_user_data\n WHERE $__timeFilter [...] - "refId": "A" - } - ], - "title": "Credits Efficiency", - "type": "stat" + "id": 20, + "panels": [], + "title": "Inline Suggestions", + "type": "row" }, { "datasource": "mysql", - "description": "Percentage of inline suggestions accepted", + "description": "Daily inline suggestion and acceptance counts", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.6.2", - "targets": [ - { - "datasource": "mysql", - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT ROUND(SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) * 100, 1) as 'Acceptance %'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)", - "refId": "A" - } - ], - "title": "Inline Acceptance Rate", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "Percentage of users using steering rules", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { @@ -211,52 +139,51 @@ "color": "green" } ] - } + }, + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 0 + "h": 8, + "w": 12, + "x": 0, + "y": 7 }, - "id": 4, + "id": 2, "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { + "legend": { "calcs": [ + "mean", "sum" ], - "fields": "", - "values": false + "displayMode": "table", + "placement": "right", + "showLegend": true }, - "showPercentChange": false, - "text": {}, - "textMode": "auto", - "wideLayout": true + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } }, "pluginVersion": "11.6.2", "targets": [ { "datasource": "mysql", "editorMode": "code", - "format": "table", + "format": "time_series", "rawQuery": true, - "rawSql": "SELECT CONCAT(ROUND(COUNT(DISTINCT CASE WHEN has_steering = 1 THEN user_id END) / NULLIF(COUNT(DISTINCT user_id), 0) * 100, 0), '%') as 'Users with Steering'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)", + "rawSql": "SELECT\n date as time,\n SUM(inline_suggestions_count) as 'Suggestions',\n SUM(inline_acceptance_count) as 'Accepted',\n SUM(inline_ai_code_lines) as 'AI Code Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", "refId": "A" } ], - "title": "Steering Adoption", - "type": "stat" + "title": "Inline Suggestions & Acceptance", + "type": "timeseries" }, { "datasource": "mysql", - "description": "Weekly active user count over time", + "description": "Acceptance rates for inline suggestions, code fix, and inline chat over time", "fieldConfig": { "defaults": { "color": { @@ -304,23 +231,22 @@ } ] }, - "unit": "short" + "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 6 + "x": 12, + "y": 7 }, - "id": 5, + "id": 3, "options": { "legend": { "calcs": [ "mean", - "max", - "sum" + "max" ], "displayMode": "table", "placement": "right", @@ -339,16 +265,29 @@ "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n STR_TO_DATE(CONCAT(YEARWEEK(timestamp, 1), ' Monday'), '%X%V %W') as time,\n COUNT(DISTINCT user_id) as 'Active Users'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY YEARWEEK(timestamp, 1)\nORDER BY time", + "rawSql": "SELECT\n date as time,\n SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) as 'Inline Suggestions',\n SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) as 'Code Fix',\n SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) as 'Inline Chat'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", "refId": "A" } ], - "title": "Weekly Active Users Trend", + "title": "Acceptance Rate Trends", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 21, + "panels": [], + "title": "Chat & Agentic (Dev)", + "type": "row" + }, { "datasource": "mysql", - "description": "New vs returning users by week", + "description": "Daily chat messages sent and AI-generated code lines from chat", "fieldConfig": { "defaults": { "color": { @@ -403,15 +342,14 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 6 + "x": 0, + "y": 16 }, - "id": 6, + "id": 4, "options": { "legend": { "calcs": [ "mean", - "max", "sum" ], "displayMode": "table", @@ -431,16 +369,16 @@ "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n week as time,\n SUM(CASE WHEN is_new = 1 THEN 1 ELSE 0 END) as 'New Users',\n SUM(CASE WHEN is_new = 0 THEN 1 ELSE 0 END) as 'Returning Users'\nFROM (\n SELECT\n u.user_id,\n STR_TO_DATE(CONCAT(YEARWEEK(u.timestamp, 1), ' Monday'), '%X%V %W') as week,\n CASE WHEN STR_TO_DATE(CONCAT(YEARWEEK(u.timestamp, 1), ' Monday'), '%X%V %W') = STR_TO_DATE(CONCAT(YEARWEEK(f.first_seen, 1), ' Monday'), '%X%V %W') THEN 1 ELSE 0 END as is_new\n FROM lake._tool [...] + "rawSql": "SELECT\n date as time,\n SUM(chat_messages_sent) as 'Messages Sent',\n SUM(chat_messages_interacted) as 'Messages Interacted',\n SUM(chat_ai_code_lines) as 'AI Code Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", "refId": "A" } ], - "title": "New vs Returning Users (Weekly)", + "title": "Chat Activity", "type": "timeseries" }, { "datasource": "mysql", - "description": "Number of users who used each feature", + "description": "Agentic (Dev) code generation and acceptance metrics", "fieldConfig": { "defaults": { "color": { @@ -453,9 +391,9 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.8, - "drawStyle": "bars", - "fillOpacity": 100, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, @@ -463,8 +401,8 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" @@ -495,48 +433,56 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 14 + "x": 12, + "y": 16 }, - "id": 7, + "id": 5, "options": { - "barRadius": 0.1, - "barWidth": 0.8, - "fullHighlight": false, - "groupWidth": 0.7, "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true }, - "orientation": "horizontal", - "showValue": "auto", - "stacking": "none", "tooltip": { "hideZeros": false, - "mode": "single", + "mode": "multi", "sort": "none" - }, - "xTickLabelRotation": 0 + } }, "pluginVersion": "11.6.2", "targets": [ { "datasource": "mysql", "editorMode": "code", - "format": "table", + "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n 'Chat' as Feature, COUNT(DISTINCT CASE WHEN chat_messages_sent > 0 THEN user_id END) as Users FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Inline Suggestions', COUNT(DISTINCT CASE WHEN inline_suggestions_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Code Fix', COUNT(DISTINCT CASE WHEN code_fix_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_dat [...] + "rawSql": "SELECT\n date as time,\n SUM(dev_generation_event_count) as 'Generation Events',\n SUM(dev_generated_lines) as 'Generated Lines',\n SUM(dev_accepted_lines) as 'Accepted Lines',\n SUM(dev_acceptance_event_count) as 'Acceptance Events'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", "refId": "A" } ], - "title": "Feature Adoption Funnel", - "type": "barchart" + "title": "Agentic (Dev) Activity", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 22, + "panels": [], + "title": "Code Review, Test Gen & Transformations", + "type": "row" }, { "datasource": "mysql", - "description": "Cumulative credits this month vs projected total", + "description": "Code review findings and test generation metrics over time", "fieldConfig": { "defaults": { "color": { @@ -591,15 +537,14 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 14 + "x": 0, + "y": 25 }, - "id": 8, + "id": 6, "options": { "legend": { "calcs": [ "mean", - "max", "sum" ], "displayMode": "table", @@ -619,16 +564,16 @@ "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(SUM(credits_used)) OVER (ORDER BY date) as 'Cumulative Credits',\n (SELECT SUM(credits_used) / COUNT(DISTINCT date) * DAY(LAST_DAY(CURDATE()))\n FROM lake._tool_q_dev_user_report\n WHERE YEAR(date) = YEAR(CURDATE()) AND MONTH(date) = MONTH(CURDATE())) as 'Projected Monthly'\nFROM lake._tool_q_dev_user_report\nWHERE YEAR(date) = YEAR(CURDATE()) AND MONTH(date) = MONTH(CURDATE())\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT\n date as time,\n SUM(code_review_findings_count) as 'Review Findings',\n SUM(code_review_succeeded_event_count) as 'Reviews Succeeded',\n SUM(code_review_failed_event_count) as 'Reviews Failed'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", "refId": "A" } ], - "title": "Credits Pace vs Projected (This Month)", + "title": "Code Review Activity", "type": "timeseries" }, { "datasource": "mysql", - "description": "Acceptance rates for inline suggestions, code fix, and inline chat over time", + "description": "Test generation events and acceptance over time", "fieldConfig": { "defaults": { "color": { @@ -676,22 +621,21 @@ } ] }, - "unit": "percentunit" + "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 22 + "x": 12, + "y": 25 }, - "id": 9, + "id": 7, "options": { "legend": { "calcs": [ "mean", - "max", "sum" ], "displayMode": "table", @@ -711,16 +655,16 @@ "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) as 'Inline Suggestions',\n SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) as 'Code Fix',\n SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) as 'Inline Chat'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT\n date as time,\n SUM(test_generation_event_count) as 'Test Gen Events',\n SUM(test_generation_generated_tests) as 'Tests Generated',\n SUM(test_generation_accepted_tests) as 'Tests Accepted',\n SUM(test_generation_generated_lines) as 'Lines Generated',\n SUM(test_generation_accepted_lines) as 'Lines Accepted'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", "refId": "A" } ], - "title": "Acceptance Rate Trends", + "title": "Test Generation Activity", "type": "timeseries" }, { "datasource": "mysql", - "description": "Code review findings and test generation metrics over time", + "description": "Doc generation and code transformation events", "fieldConfig": { "defaults": { "color": { @@ -775,15 +719,14 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 22 + "x": 0, + "y": 33 }, - "id": 10, + "id": 8, "options": { "legend": { "calcs": [ "mean", - "max", "sum" ], "displayMode": "table", @@ -803,28 +746,53 @@ "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date as time,\n SUM(code_review_findings_count) as 'Review Findings',\n SUM(test_generation_event_count) as 'Test Gen Events',\n SUM(test_generation_accepted_tests) as 'Tests Accepted'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "rawSql": "SELECT\n date as time,\n SUM(doc_generation_event_count) as 'Doc Gen Events',\n SUM(doc_generation_accepted_line_additions) as 'Doc Lines Accepted',\n SUM(transformation_event_count) as 'Transformation Events',\n SUM(transformation_lines_generated) as 'Transform Lines Generated'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", "refId": "A" } ], - "title": "Code Review Findings & Test Generation", + "title": "Doc Generation & Transformations", "type": "timeseries" }, { "datasource": "mysql", - "description": "Per-user productivity and efficiency metrics", + "description": "Number of users who used each feature in the selected period", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.8, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "filterable": true, - "inspect": false + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { @@ -834,29 +802,38 @@ "color": "green" } ] - } + }, + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 30 + "h": 8, + "w": 12, + "x": 12, + "y": 33 }, - "id": 11, + "id": 9, "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false + "barRadius": 0.1, + "barWidth": 0.8, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - "showHeader": true, - "sortBy": [] + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0 }, "pluginVersion": "11.6.2", "targets": [ @@ -865,16 +842,29 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(MAX(d.display_name), d.user_id) as 'User',\n COALESCE(MAX(r.subscription_tier), '') as 'Tier',\n ROUND(SUM(r.credits_used), 1) as 'Credits Used',\n SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) as 'Total Accepted Lines',\n CASE WHEN SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) > 0\n THEN ROUND(SUM(r.credits_used) / SUM(d.chat_ai_c [...] + "rawSql": "SELECT 'Chat' as Feature, COUNT(DISTINCT CASE WHEN chat_messages_sent > 0 THEN user_id END) as Users FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Inline Suggestions', COUNT(DISTINCT CASE WHEN inline_suggestions_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Code Fix', COUNT(DISTINCT CASE WHEN code_fix_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data W [...] "refId": "A" } ], - "title": "User Productivity & Efficiency", - "type": "table" + "title": "Feature Adoption (Users per Feature)", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 23, + "panels": [], + "title": "Per-User Detail", + "type": "row" }, { "datasource": "mysql", - "description": "Power tier users with no activity in the last 14 days", + "description": "Per-user breakdown of legacy feature-level metrics", "fieldConfig": { "defaults": { "color": { @@ -901,12 +891,12 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 10, "w": 24, "x": 0, - "y": 40 + "y": 42 }, - "id": 12, + "id": 10, "options": { "cellHeight": "sm", "footer": { @@ -927,11 +917,11 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n MAX(subscription_tier) as 'Tier',\n ROUND(SUM(credits_used), 1) as 'Total Credits Used',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\n AND subscription_tier = 'POWER'\nGROUP BY user_id\nHAVING MAX(date) < DATE_SUB(NOW(), INTERVAL 14 DAY)\nORDER BY MAX(date)", + "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n SUM(inline_suggestions_count) as 'Suggestions',\n SUM(inline_acceptance_count) as 'Accepted',\n CONCAT(ROUND(SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) * 100, 1), '%') as 'Accept %',\n SUM(chat_messages_sent) as 'Chat Msgs',\n SUM(chat_ai_code_lines) as 'Chat Lines',\n SUM(dev_accepted_lines) as 'Agentic Lines',\n SUM(code_review_findings_count) as 'Review Findings',\n SU [...] "refId": "A" } ], - "title": "Idle Power Users (No Activity in 14 Days)", + "title": "Per-User Feature Metrics", "type": "table" } ], @@ -940,7 +930,7 @@ "schemaVersion": 41, "tags": [ "q_dev", - "executive", + "legacy", "kiro" ], "templating": { @@ -952,7 +942,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "Kiro Executive Dashboard", - "uid": "qdev_executive", + "title": "Kiro Legacy Feature Metrics", + "uid": "qdev_feature_metrics", "version": 1 -} \ No newline at end of file +} diff --git a/grafana/dashboards/qdev_logging.json b/grafana/dashboards/qdev_logging.json index 462adcd98..6a47b03bf 100644 --- a/grafana/dashboards/qdev_logging.json +++ b/grafana/dashboards/qdev_logging.json @@ -70,7 +70,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n (SELECT COUNT(*) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Chat Events',\n (SELECT COUNT(DISTINCT user_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Chat Users',\n (SELECT COUNT(DISTINCT conversation_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) AND conversation_id != '') as 'Conversations',\n (SELECT COUNT(*) FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) as 'Com [...] + "rawSql": "SELECT\n (SELECT COUNT(*) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Chat Events',\n (SELECT COUNT(DISTINCT user_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Chat Users',\n (SELECT COUNT(DISTINCT conversation_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) AND conversation_id != '') as 'Conversations',\n (SELECT COUNT(*) FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) as 'Com [...] "refId": "A" } ], @@ -168,7 +168,7 @@ }, { "datasource": "mysql", - "description": "Distribution of model usage across chat events", + "description": "Distribution of chat trigger types: MANUAL (chat window) vs INLINE_CHAT", "fieldConfig": { "defaults": { "color": { @@ -188,10 +188,78 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, "y": 14 }, + "id": 11, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n CASE\n WHEN chat_trigger_type = '' OR chat_trigger_type IS NULL THEN '(unknown)'\n ELSE chat_trigger_type\n END as 'Trigger Type',\n COUNT(*) as 'Events'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY chat_trigger_type\nORDER BY COUNT(*) DESC", + "refId": "A" + } + ], + "title": "Chat Trigger Type Distribution", + "type": "piechart" + }, + { + "datasource": "mysql", + "description": "Distribution of model usage across chat events", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 14 + }, "id": 3, "options": { "displayLabels": [ @@ -256,8 +324,8 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, + "w": 8, + "x": 16, "y": 14 }, "id": 4, @@ -541,7 +609,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(u.display_name, u.user_id) as 'User',\n u.user_id as 'User ID',\n u.chat_events as 'Chat Events',\n u.conversations as 'Conversations',\n ROUND(u.chat_events / NULLIF(u.conversations, 0), 1) as 'Avg Turns',\n COALESCE(c.completion_events, 0) as 'Completion Events',\n COALESCE(c.files_count, 0) as 'Distinct Files',\n ROUND(u.avg_prompt_len) as 'Avg Prompt Len',\n ROUND(u.avg_response_len) as 'Avg Response Len',\n u.steering_count as 'Steeri [...] + "rawSql": "SELECT\n COALESCE(u.display_name, u.user_id) as 'User',\n u.user_id as 'User ID',\n u.chat_events as 'Chat Events',\n u.conversations as 'Conversations',\n ROUND(u.chat_events / NULLIF(u.conversations, 0), 1) as 'Avg Turns',\n COALESCE(c.completion_events, 0) as 'Completion Events',\n COALESCE(c.files_count, 0) as 'Distinct Files',\n ROUND(u.avg_prompt_len) as 'Avg Prompt Len',\n ROUND(u.avg_response_len) as 'Avg Response Len',\n u.steering_count as 'Steeri [...] "refId": "A" } ], @@ -570,7 +638,7 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, "y": 48 }, @@ -638,8 +706,8 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, + "w": 8, + "x": 8, "y": 48 }, "id": 9, @@ -684,6 +752,74 @@ "title": "Active File Types in Chat", "type": "piechart" }, + { + "datasource": "mysql", + "description": "How often Kiro responses include code references and web links", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 48 + }, + "id": 12, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n SUM(CASE WHEN code_reference_count > 0 THEN 1 ELSE 0 END) as 'With Code References',\n SUM(CASE WHEN web_link_count > 0 THEN 1 ELSE 0 END) as 'With Web Links',\n SUM(CASE WHEN has_followup_prompts = 1 THEN 1 ELSE 0 END) as 'With Followup Prompts',\n SUM(CASE WHEN code_reference_count = 0 AND web_link_count = 0 AND has_followup_prompts = 0 THEN 1 ELSE 0 END) as 'Plain Response'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Response Enrichment Breakdown", + "type": "piechart" + }, { "datasource": "mysql", "description": "Average and maximum prompt/response lengths over time", @@ -775,6 +911,188 @@ ], "title": "Prompt & Response Length Trends", "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Average code context size provided to inline completions over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Characters", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE(timestamp) as time,\n ROUND(AVG(left_context_length)) as 'Avg Left Context',\n ROUND(AVG(right_context_length)) as 'Avg Right Context',\n ROUND(AVG(left_context_length + right_context_length)) as 'Avg Total Context'\nFROM lake._tool_q_dev_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", + "refId": "A" + } + ], + "title": "Completion Context Size Trends", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Daily trend of code references and web links in chat responses", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE(timestamp) as time,\n SUM(code_reference_count) as 'Code References',\n SUM(web_link_count) as 'Web Links',\n SUM(CASE WHEN has_followup_prompts = 1 THEN 1 ELSE 0 END) as 'Followup Prompts'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", + "refId": "A" + } + ], + "title": "Response Enrichment Trends", + "type": "timeseries" } ], "preload": false, @@ -797,4 +1115,4 @@ "title": "Kiro AI Activity Insights", "uid": "qdev_logging", "version": 1 -} \ No newline at end of file +}
