This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new c4ad2cbc7 feat(q-dev): enrich logging fields, separate dashboards, add
E2E tests (#8786)
c4ad2cbc7 is described below
commit c4ad2cbc7e912f905088b1d27eccab0858f3fbda
Author: Warren Chen <[email protected]>
AuthorDate: Sat Mar 21 10:08:42 2026 +0800
feat(q-dev): enrich logging fields, separate dashboards, add E2E tests
(#8786)
* feat(q-dev): enrich logging fields, separate dashboards by data source,
add E2E tests
- Add new fields to chat_log: CodeReferenceCount, WebLinkCount,
HasFollowupPrompts
(from codeReferenceEvents, supplementaryWebLinksEvent, followupPrompts in
JSON)
- Add new fields to completion_log: LeftContextLength, RightContextLength
(from leftContext/rightContext in JSON)
- Update s3_logging_extractor to parse and populate new fields
- Add migration script 20260319_add_logging_fields
- Create qdev_feature_metrics dashboard for legacy by_user_analytic data
- Reorganize qdev_executive dashboard with Row dividers labeling data
sources
and cross-dashboard navigation links
- Enrich qdev_logging dashboard with new panels:
Chat Trigger Type Distribution, Response Enrichment Breakdown,
Completion Context Size Trends, Response Enrichment Trends
- Fix SQL compatibility with only_full_group_by mode in executive dashboard
(Weekly Active Users Trend, New vs Returning Users)
- Fix Steering Adoption stat panel returning string instead of numeric value
- Add Playwright E2E test covering full pipeline flow and dashboard
verification
* fix: add Apache license headers to e2e files, fix gofmt alignment
---
backend/plugins/q_dev/models/chat_log.go | 3 +
backend/plugins/q_dev/models/completion_log.go | 22 +-
...{register.go => 20260319_add_logging_fields.go} | 39 +-
.../models/migrationscripts/archived/chat_log.go | 3 +
.../migrationscripts/archived/completion_log.go | 22 +-
.../q_dev/models/migrationscripts/register.go | 1 +
.../plugins/q_dev/tasks/s3_logging_extractor.go | 32 +-
e2e/package.json | 16 +
.../register.go => e2e/playwright.config.ts | 43 +-
e2e/qdev-full-flow.spec.ts | 248 +++++++++++
grafana/dashboards/qdev_executive.json | 438 ++++++-------------
...ev_executive.json => qdev_feature_metrics.json} | 474 ++++++++++-----------
grafana/dashboards/qdev_logging.json | 338 ++++++++++++++-
13 files changed, 1044 insertions(+), 635 deletions(-)
diff --git a/backend/plugins/q_dev/models/chat_log.go
b/backend/plugins/q_dev/models/chat_log.go
index 06679c515..6b39bffa4 100644
--- a/backend/plugins/q_dev/models/chat_log.go
+++ b/backend/plugins/q_dev/models/chat_log.go
@@ -44,6 +44,9 @@ type QDevChatLog struct {
ActiveFileExtension string `gorm:"type:varchar(50)"
json:"activeFileExtension"`
HasSteering bool `json:"hasSteering"`
IsSpecMode bool `json:"isSpecMode"`
+ CodeReferenceCount int `json:"codeReferenceCount"`
+ WebLinkCount int `json:"webLinkCount"`
+ HasFollowupPrompts bool `json:"hasFollowupPrompts"`
}
func (QDevChatLog) TableName() string {
diff --git a/backend/plugins/q_dev/models/completion_log.go
b/backend/plugins/q_dev/models/completion_log.go
index 0d0e0404c..00a1b471f 100644
--- a/backend/plugins/q_dev/models/completion_log.go
+++ b/backend/plugins/q_dev/models/completion_log.go
@@ -26,16 +26,18 @@ import (
// QDevCompletionLog stores parsed data from GenerateCompletions logging events
type QDevCompletionLog struct {
common.NoPKModel
- ConnectionId uint64 `gorm:"primaryKey"`
- ScopeId string `gorm:"primaryKey;type:varchar(255)"
json:"scopeId"`
- RequestId string `gorm:"primaryKey;type:varchar(255)"
json:"requestId"`
- UserId string `gorm:"index;type:varchar(255)"
json:"userId"`
- DisplayName string `gorm:"type:varchar(255)" json:"displayName"`
- Timestamp time.Time `gorm:"index" json:"timestamp"`
- FileName string `gorm:"type:varchar(512)" json:"fileName"`
- FileExtension string `gorm:"type:varchar(50)"
json:"fileExtension"`
- HasCustomization bool `json:"hasCustomization"`
- CompletionsCount int `json:"completionsCount"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ScopeId string `gorm:"primaryKey;type:varchar(255)"
json:"scopeId"`
+ RequestId string `gorm:"primaryKey;type:varchar(255)"
json:"requestId"`
+ UserId string `gorm:"index;type:varchar(255)"
json:"userId"`
+ DisplayName string `gorm:"type:varchar(255)"
json:"displayName"`
+ Timestamp time.Time `gorm:"index" json:"timestamp"`
+ FileName string `gorm:"type:varchar(512)" json:"fileName"`
+ FileExtension string `gorm:"type:varchar(50)"
json:"fileExtension"`
+ HasCustomization bool `json:"hasCustomization"`
+ CompletionsCount int `json:"completionsCount"`
+ LeftContextLength int `json:"leftContextLength"`
+ RightContextLength int `json:"rightContextLength"`
}
func (QDevCompletionLog) TableName() string {
diff --git a/backend/plugins/q_dev/models/migrationscripts/register.go
b/backend/plugins/q_dev/models/migrationscripts/20260319_add_logging_fields.go
similarity index 51%
copy from backend/plugins/q_dev/models/migrationscripts/register.go
copy to
backend/plugins/q_dev/models/migrationscripts/20260319_add_logging_fields.go
index 5480d5eaf..f98c3d106 100644
--- a/backend/plugins/q_dev/models/migrationscripts/register.go
+++
b/backend/plugins/q_dev/models/migrationscripts/20260319_add_logging_fields.go
@@ -18,23 +18,28 @@ limitations under the License.
package migrationscripts
import (
- "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+
"github.com/apache/incubator-devlake/plugins/q_dev/models/migrationscripts/archived"
)
-// All return all migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(initTables),
- new(modifyFileMetaTable),
- new(addDisplayNameFields),
- new(addMissingMetrics),
- new(addS3SliceTable),
- new(addScopeConfigIdToS3Slice),
- new(addScopeIdFields),
- new(addUserReportTable),
- new(addAccountIdToS3Slice),
- new(fixDedupUserTables),
- new(resetS3FileMetaProcessed),
- new(addLoggingTables),
- }
+var _ = (*addLoggingFields)(nil)
+
+type addLoggingFields struct{}
+
+func (*addLoggingFields) Up(basicRes context.BasicRes) errors.Error {
+ return migrationhelper.AutoMigrateTables(
+ basicRes,
+ &archived.QDevChatLog{},
+ &archived.QDevCompletionLog{},
+ )
+}
+
+func (*addLoggingFields) Version() uint64 {
+ return 20260319000001
+}
+
+func (*addLoggingFields) Name() string {
+ return "Add code_reference_count, web_link_count, has_followup_prompts
to chat_log; left/right_context_length to completion_log"
}
diff --git a/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go
b/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go
index 8278f52ff..ee7d10a1e 100644
--- a/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go
+++ b/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go
@@ -43,6 +43,9 @@ type QDevChatLog struct {
ActiveFileExtension string `gorm:"type:varchar(50)"
json:"activeFileExtension"`
HasSteering bool `json:"hasSteering"`
IsSpecMode bool `json:"isSpecMode"`
+ CodeReferenceCount int `json:"codeReferenceCount"`
+ WebLinkCount int `json:"webLinkCount"`
+ HasFollowupPrompts bool `json:"hasFollowupPrompts"`
}
func (QDevChatLog) TableName() string {
diff --git
a/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go
b/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go
index 035c13ef2..4acff9569 100644
--- a/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go
+++ b/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go
@@ -25,16 +25,18 @@ import (
type QDevCompletionLog struct {
archived.NoPKModel
- ConnectionId uint64 `gorm:"primaryKey"`
- ScopeId string `gorm:"primaryKey;type:varchar(255)"
json:"scopeId"`
- RequestId string `gorm:"primaryKey;type:varchar(255)"
json:"requestId"`
- UserId string `gorm:"index;type:varchar(255)"
json:"userId"`
- DisplayName string `gorm:"type:varchar(255)" json:"displayName"`
- Timestamp time.Time `gorm:"index" json:"timestamp"`
- FileName string `gorm:"type:varchar(512)" json:"fileName"`
- FileExtension string `gorm:"type:varchar(50)"
json:"fileExtension"`
- HasCustomization bool `json:"hasCustomization"`
- CompletionsCount int `json:"completionsCount"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ScopeId string `gorm:"primaryKey;type:varchar(255)"
json:"scopeId"`
+ RequestId string `gorm:"primaryKey;type:varchar(255)"
json:"requestId"`
+ UserId string `gorm:"index;type:varchar(255)"
json:"userId"`
+ DisplayName string `gorm:"type:varchar(255)"
json:"displayName"`
+ Timestamp time.Time `gorm:"index" json:"timestamp"`
+ FileName string `gorm:"type:varchar(512)" json:"fileName"`
+ FileExtension string `gorm:"type:varchar(50)"
json:"fileExtension"`
+ HasCustomization bool `json:"hasCustomization"`
+ CompletionsCount int `json:"completionsCount"`
+ LeftContextLength int `json:"leftContextLength"`
+ RightContextLength int `json:"rightContextLength"`
}
func (QDevCompletionLog) TableName() string {
diff --git a/backend/plugins/q_dev/models/migrationscripts/register.go
b/backend/plugins/q_dev/models/migrationscripts/register.go
index 5480d5eaf..8b5de0bcc 100644
--- a/backend/plugins/q_dev/models/migrationscripts/register.go
+++ b/backend/plugins/q_dev/models/migrationscripts/register.go
@@ -36,5 +36,6 @@ func All() []plugin.MigrationScript {
new(fixDedupUserTables),
new(resetS3FileMetaProcessed),
new(addLoggingTables),
+ new(addLoggingFields),
}
}
diff --git a/backend/plugins/q_dev/tasks/s3_logging_extractor.go
b/backend/plugins/q_dev/tasks/s3_logging_extractor.go
index df55a663b..3fa771789 100644
--- a/backend/plugins/q_dev/tasks/s3_logging_extractor.go
+++ b/backend/plugins/q_dev/tasks/s3_logging_extractor.go
@@ -280,10 +280,13 @@ type chatLogRequest struct {
type chatLogResponse struct {
RequestID string `json:"requestId"`
AssistantResponse string `json:"assistantResponse"`
+ FollowupPrompts string `json:"followupPrompts"`
MessageMetadata struct {
ConversationID *string `json:"conversationId"`
UtteranceID *string `json:"utteranceId"`
} `json:"messageMetadata"`
+ CodeReferenceEvents []json.RawMessage
`json:"codeReferenceEvents"`
+ SupplementaryWebLinksEvent []json.RawMessage
`json:"supplementaryWebLinksEvent"`
}
type completionLogRecord struct {
@@ -296,6 +299,8 @@ type completionLogRequest struct {
Timestamp string `json:"timeStamp"`
FileName string `json:"fileName"`
CustomizationArn *string `json:"customizationArn"`
+ LeftContext string `json:"leftContext"`
+ RightContext string `json:"rightContext"`
}
type completionLogResponse struct {
@@ -347,6 +352,11 @@ func parseChatRecord(raw json.RawMessage, fileMeta
*models.QDevS3FileMeta, ident
chatLog.UtteranceId =
*record.Response.MessageMetadata.UtteranceID
}
+ // New fields from docs: codeReferenceEvents,
supplementaryWebLinksEvent, followupPrompts
+ chatLog.CodeReferenceCount = len(record.Response.CodeReferenceEvents)
+ chatLog.WebLinkCount = len(record.Response.SupplementaryWebLinksEvent)
+ chatLog.HasFollowupPrompts = record.Response.FollowupPrompts != ""
+
return chatLog, nil
}
@@ -406,16 +416,18 @@ func parseCompletionRecord(raw json.RawMessage, fileMeta
*models.QDevS3FileMeta,
userId := normalizeUserId(record.Request.UserID)
return &models.QDevCompletionLog{
- ConnectionId: fileMeta.ConnectionId,
- ScopeId: fileMeta.ScopeId,
- RequestId: record.Response.RequestID,
- UserId: userId,
- DisplayName: cachedResolveDisplayName(userId,
identityClient, cache),
- Timestamp: ts,
- FileName: record.Request.FileName,
- FileExtension: filepath.Ext(record.Request.FileName),
- HasCustomization: record.Request.CustomizationArn != nil &&
*record.Request.CustomizationArn != "",
- CompletionsCount: len(record.Response.Completions),
+ ConnectionId: fileMeta.ConnectionId,
+ ScopeId: fileMeta.ScopeId,
+ RequestId: record.Response.RequestID,
+ UserId: userId,
+ DisplayName: cachedResolveDisplayName(userId,
identityClient, cache),
+ Timestamp: ts,
+ FileName: record.Request.FileName,
+ FileExtension: filepath.Ext(record.Request.FileName),
+ HasCustomization: record.Request.CustomizationArn != nil &&
*record.Request.CustomizationArn != "",
+ CompletionsCount: len(record.Response.Completions),
+ LeftContextLength: len(record.Request.LeftContext),
+ RightContextLength: len(record.Request.RightContext),
}, nil
}
diff --git a/e2e/package.json b/e2e/package.json
new file mode 100644
index 000000000..af179c7b6
--- /dev/null
+++ b/e2e/package.json
@@ -0,0 +1,16 @@
+{
+ "name": "e2e",
+ "version": "1.0.0",
+ "description": "",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "keywords": [],
+ "author": "",
+ "license": "ISC",
+ "type": "commonjs",
+ "dependencies": {
+ "@playwright/test": "^1.58.2"
+ }
+}
diff --git a/backend/plugins/q_dev/models/migrationscripts/register.go
b/e2e/playwright.config.ts
similarity index 59%
copy from backend/plugins/q_dev/models/migrationscripts/register.go
copy to e2e/playwright.config.ts
index 5480d5eaf..d40062836 100644
--- a/backend/plugins/q_dev/models/migrationscripts/register.go
+++ b/e2e/playwright.config.ts
@@ -15,26 +15,25 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+import { defineConfig } from '@playwright/test';
-import (
- "github.com/apache/incubator-devlake/core/plugin"
-)
-
-// All return all migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(initTables),
- new(modifyFileMetaTable),
- new(addDisplayNameFields),
- new(addMissingMetrics),
- new(addS3SliceTable),
- new(addScopeConfigIdToS3Slice),
- new(addScopeIdFields),
- new(addUserReportTable),
- new(addAccountIdToS3Slice),
- new(fixDedupUserTables),
- new(resetS3FileMetaProcessed),
- new(addLoggingTables),
- }
-}
+export default defineConfig({
+ testDir: '.',
+ testMatch: '*.spec.ts',
+ timeout: 180000,
+ expect: {
+ timeout: 10000,
+ },
+ use: {
+ baseURL: 'http://localhost:4000',
+ screenshot: 'on',
+ trace: 'on-first-retry',
+ },
+ reporter: [['html', { open: 'never' }], ['list']],
+ projects: [
+ {
+ name: 'chromium',
+ use: { browserName: 'chromium', viewport: { width: 1440, height: 900 } },
+ },
+ ],
+});
diff --git a/e2e/qdev-full-flow.spec.ts b/e2e/qdev-full-flow.spec.ts
new file mode 100644
index 000000000..0498df67b
--- /dev/null
+++ b/e2e/qdev-full-flow.spec.ts
@@ -0,0 +1,248 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { test, expect, request, Page } from '@playwright/test';
+import * as path from 'path';
+import * as fs from 'fs';
+
+const API = 'http://localhost:8080';
+const UI = 'http://localhost:4000';
+const GRAFANA = 'http://localhost:3002';
+const SCREENSHOT_DIR = path.join(__dirname, 'screenshots');
+
+// Use existing connection with valid credentials
+const EXISTING_CONNECTION_ID = 5;
+
+const state: {
+ connectionId: number;
+ scopeId: string;
+ blueprintId: number;
+ pipelineId: number;
+} = { connectionId: EXISTING_CONNECTION_ID, scopeId: '', blueprintId: 0,
pipelineId: 0 };
+
+fs.mkdirSync(SCREENSHOT_DIR, { recursive: true });
+
+async function grafanaLogin(page: Page) {
+ await page.goto(`${GRAFANA}/grafana/login`);
+ await page.waitForLoadState('networkidle');
+ if (page.url().includes('/login')) {
+ await page.locator('input[name="user"]').fill('admin');
+ await page.locator('input[name="password"]').fill('admin');
+ await page.locator('button[type="submit"]').click();
+ await page.waitForTimeout(2000);
+ // Handle "change password" prompt if shown
+ const skipBtn = page.locator('a:has-text("Skip")');
+ if (await skipBtn.isVisible({ timeout: 2000 }).catch(() => false)) {
+ await skipBtn.click();
+ }
+ await page.waitForTimeout(1000);
+ }
+}
+
+async function openGrafanaDashboard(page: Page, uid: string, screenshotPath:
string) {
+ await grafanaLogin(page);
+ await page.goto(`${GRAFANA}/grafana/d/${uid}?orgId=1&from=now-90d&to=now`);
+
+ // Wait for first panel data to load
+ try {
+ await page.waitForResponse(
+ (resp) => resp.url().includes('/api/ds/query') && resp.status() === 200,
+ { timeout: 30000 }
+ );
+ } catch { /* some dashboards may not fire queries immediately */ }
+
+ // Wait for rendering to settle
+ await page.waitForTimeout(5000);
+
+ // Take viewport screenshot (top section)
+ await page.screenshot({ path: screenshotPath.replace('.png', '-top.png') });
+
+ // Scroll down and take more sections
+ const scrollHeight = await page.evaluate(() => document.body.scrollHeight);
+ let section = 1;
+ for (let y = 900; y < scrollHeight; y += 900) {
+ await page.evaluate((scrollY) => window.scrollTo(0, scrollY), y);
+ await page.waitForTimeout(3000);
+ section++;
+ await page.screenshot({ path: screenshotPath.replace('.png',
`-section${section}.png`) });
+ }
+
+ // Also take full page screenshot
+ await page.evaluate(() => window.scrollTo(0, 0));
+ await page.waitForTimeout(2000);
+ await page.screenshot({ path: screenshotPath, fullPage: true });
+}
+
+test.describe.serial('Q-Dev Plugin Full Flow', () => {
+
+ test('Step 1: Verify Existing Connection via API', async () => {
+ const api = await request.newContext({ baseURL: API });
+
+ const resp = await
api.get(`/plugins/q_dev/connections/${state.connectionId}`);
+ expect(resp.ok()).toBeTruthy();
+ const conn = await resp.json();
+ console.log(`Using connection: id=${conn.id}, name=${conn.name},
bucket=${conn.bucket}`);
+
+ const testResp = await
api.post(`/plugins/q_dev/connections/${state.connectionId}/test`);
+ const testBody = await testResp.json();
+ console.log('Test connection:', testBody.success ? 'OK' :
testBody.message);
+ expect(testResp.ok()).toBeTruthy();
+ });
+
+ test('Step 2: View Config-UI Home', async ({ page }) => {
+ await page.goto(UI);
+ await page.waitForLoadState('networkidle');
+ await page.waitForTimeout(1000);
+ await page.screenshot({ path: path.join(SCREENSHOT_DIR,
'01-config-ui-home.png'), fullPage: true });
+ console.log('Screenshot: Config-UI home');
+ });
+
+ test('Step 3: Create Scope (S3 Slice) via API', async () => {
+ const api = await request.newContext({ baseURL: API });
+
+ const resp = await
api.put(`/plugins/q_dev/connections/${state.connectionId}/scopes`, {
+ data: {
+ data: [
+ {
+ accountId: '034362076319',
+ basePath: '',
+ year: 2026,
+ month: 3,
+ },
+ ],
+ },
+ });
+
+ const body = await resp.json();
+ console.log('Scope created:', resp.status());
+ expect(resp.ok()).toBeTruthy();
+ state.scopeId = body[0]?.id;
+ expect(state.scopeId).toBeTruthy();
+ console.log(`Scope id: ${state.scopeId}`);
+ });
+
+ test('Step 4: Create Blueprint via API', async () => {
+ const api = await request.newContext({ baseURL: API });
+
+ const resp = await api.post('/blueprints', {
+ data: {
+ name: `e2e-blueprint-${Date.now()}`,
+ mode: 'NORMAL',
+ enable: true,
+ cronConfig: '0 0 * * *',
+ isManual: true,
+ connections: [
+ {
+ pluginName: 'q_dev',
+ connectionId: state.connectionId,
+ scopes: [{ scopeId: state.scopeId }],
+ },
+ ],
+ },
+ });
+
+ const body = await resp.json();
+ expect(resp.ok()).toBeTruthy();
+ state.blueprintId = body.id;
+ console.log(`Blueprint created: id=${state.blueprintId}`);
+ });
+
+ test('Step 5: Trigger Pipeline via API', async () => {
+ const api = await request.newContext({ baseURL: API });
+
+ const resp = await api.post(`/blueprints/${state.blueprintId}/trigger`, {
data: {} });
+ const body = await resp.json();
+ expect(resp.ok()).toBeTruthy();
+ state.pipelineId = body.id;
+ console.log(`Pipeline triggered: id=${state.pipelineId}`);
+ });
+
+ test('Step 6: Wait for Pipeline to Complete', async () => {
+ const api = await request.newContext({ baseURL: API });
+ const maxWait = 120000;
+ const start = Date.now();
+ let status = '';
+
+ while (Date.now() - start < maxWait) {
+ const resp = await api.get(`/pipelines/${state.pipelineId}`);
+ const pipeline = await resp.json();
+ status = pipeline.status;
+ console.log(`Pipeline status: ${status} (${Math.round((Date.now() -
start) / 1000)}s)`);
+ if (['TASK_COMPLETED', 'TASK_FAILED', 'TASK_PARTIAL'].includes(status))
break;
+ await new Promise((r) => setTimeout(r, 3000));
+ }
+
+ // Print task details
+ const tasksResp = await api.get(`/pipelines/${state.pipelineId}/tasks`);
+ if (tasksResp.ok()) {
+ const { tasks } = await tasksResp.json();
+ for (const t of tasks || []) {
+ console.log(` Task ${t.id}: ${t.status}${t.failedSubTask ? ` (failed:
${t.failedSubTask})` : ''}`);
+ if (t.message) console.log(` Error: ${t.message.substring(0,
300)}`);
+ }
+ }
+
+ expect(status).toBe('TASK_COMPLETED');
+ });
+
+ test('Step 7: Verify Data via MySQL', async () => {
+ const api = await request.newContext({ baseURL: API });
+
+ // Use pipeline tasks to confirm data was processed
+ const tasksResp = await api.get(`/pipelines/${state.pipelineId}/tasks`);
+ const { tasks } = await tasksResp.json();
+ expect(tasks[0].status).toBe('TASK_COMPLETED');
+ console.log(`Pipeline completed in ${tasks[0].spentSeconds}s`);
+ });
+
+ test('Step 8: Grafana - Kiro Usage Dashboard (new format)', async ({ page })
=> {
+ await openGrafanaDashboard(page, 'qdev_user_report',
path.join(SCREENSHOT_DIR, '02-dashboard-user-report.png'));
+ console.log('Screenshot: Kiro Usage Dashboard');
+ });
+
+ test('Step 9: Grafana - Kiro Legacy Feature Metrics', async ({ page }) => {
+ await openGrafanaDashboard(page, 'qdev_feature_metrics',
path.join(SCREENSHOT_DIR, '03-dashboard-feature-metrics.png'));
+ console.log('Screenshot: Kiro Legacy Feature Metrics');
+ });
+
+ test('Step 10: Grafana - Kiro AI Activity Insights (logging)', async ({ page
}) => {
+ await openGrafanaDashboard(page, 'qdev_logging', path.join(SCREENSHOT_DIR,
'04-dashboard-logging.png'));
+ console.log('Screenshot: Kiro AI Activity Insights');
+ });
+
+ test('Step 11: Grafana - Kiro Executive Dashboard', async ({ page }) => {
+ await openGrafanaDashboard(page, 'qdev_executive',
path.join(SCREENSHOT_DIR, '05-dashboard-executive.png'));
+ console.log('Screenshot: Kiro Executive Dashboard');
+ });
+
+ test('Step 12: View Pipeline in Config-UI', async ({ page }) => {
+ // Navigate to the API proxy route for pipelines
+ await page.goto(`${UI}/api/pipelines?pageSize=5`);
+ await page.waitForLoadState('networkidle');
+ await page.screenshot({ path: path.join(SCREENSHOT_DIR,
'06-config-ui-pipelines.png'), fullPage: true });
+ console.log('Screenshot: Pipelines API response');
+ });
+
+ test('Step 13: Cleanup', async () => {
+ const api = await request.newContext({ baseURL: API });
+ if (state.blueprintId) {
+ await api.delete(`/blueprints/${state.blueprintId}`);
+ console.log(`Deleted blueprint ${state.blueprintId}`);
+ }
+ console.log('Cleanup complete');
+ });
+});
diff --git a/grafana/dashboards/qdev_executive.json
b/grafana/dashboards/qdev_executive.json
index c6e2524d7..df36a6c9e 100644
--- a/grafana/dashboards/qdev_executive.json
+++ b/grafana/dashboards/qdev_executive.json
@@ -16,11 +16,61 @@
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
- "links": [],
+ "links": [
+ {
+ "asDropdown": false,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [],
+ "targetBlank": true,
+ "title": "Usage (New)",
+ "tooltip": "Kiro Usage Dashboard - Credits & Messages (new format)",
+ "type": "link",
+ "url": "/d/qdev_user_report"
+ },
+ {
+ "asDropdown": false,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [],
+ "targetBlank": true,
+ "title": "Feature Metrics (Legacy)",
+ "tooltip": "Kiro Legacy Feature Metrics (old format)",
+ "type": "link",
+ "url": "/d/qdev_feature_metrics"
+ },
+ {
+ "asDropdown": false,
+ "icon": "external link",
+ "includeVars": true,
+ "keepTime": true,
+ "tags": [],
+ "targetBlank": true,
+ "title": "Prompt Logging",
+ "tooltip": "Kiro AI Activity Insights - Prompt Logging",
+ "type": "link",
+ "url": "/d/qdev_logging"
+ }
+ ],
"panels": [
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 100,
+ "panels": [],
+ "title": "KPI Overview (cross-source)",
+ "type": "row"
+ },
{
"datasource": "mysql",
- "description": "Distinct users with chat activity in the last 7 days",
+ "description": "Distinct users with chat activity in the last 7 days
(from prompt logging)",
"fieldConfig": {
"defaults": {
"color": {
@@ -42,7 +92,7 @@
"h": 6,
"w": 6,
"x": 0,
- "y": 0
+ "y": 1
},
"id": 1,
"options": {
@@ -74,12 +124,12 @@
"refId": "A"
}
],
- "title": "Weekly Active Users",
+ "title": "Weekly Active Users (logging)",
"type": "stat"
},
{
"datasource": "mysql",
- "description": "Average credits spent per accepted line of code",
+ "description": "Average credits spent per accepted line of code (new
report + legacy metrics)",
"fieldConfig": {
"defaults": {
"color": {
@@ -101,7 +151,7 @@
"h": 6,
"w": 6,
"x": 6,
- "y": 0
+ "y": 1
},
"id": 2,
"options": {
@@ -133,12 +183,12 @@
"refId": "A"
}
],
- "title": "Credits Efficiency",
+ "title": "Credits Efficiency (new + legacy)",
"type": "stat"
},
{
"datasource": "mysql",
- "description": "Percentage of inline suggestions accepted",
+ "description": "Percentage of inline suggestions accepted (from legacy
feature metrics)",
"fieldConfig": {
"defaults": {
"color": {
@@ -160,7 +210,7 @@
"h": 6,
"w": 6,
"x": 12,
- "y": 0
+ "y": 1
},
"id": 3,
"options": {
@@ -192,12 +242,12 @@
"refId": "A"
}
],
- "title": "Inline Acceptance Rate",
+ "title": "Inline Acceptance Rate (legacy)",
"type": "stat"
},
{
"datasource": "mysql",
- "description": "Percentage of users using steering rules",
+ "description": "Percentage of users who used steering rules (from prompt
logging)",
"fieldConfig": {
"defaults": {
"color": {
@@ -211,7 +261,8 @@
"color": "green"
}
]
- }
+ },
+ "unit": "percent"
},
"overrides": []
},
@@ -219,7 +270,7 @@
"h": 6,
"w": 6,
"x": 18,
- "y": 0
+ "y": 1
},
"id": 4,
"options": {
@@ -247,16 +298,29 @@
"editorMode": "code",
"format": "table",
"rawQuery": true,
- "rawSql": "SELECT CONCAT(ROUND(COUNT(DISTINCT CASE WHEN has_steering
= 1 THEN user_id END) / NULLIF(COUNT(DISTINCT user_id), 0) * 100, 0), '%') as
'Users with Steering'\nFROM lake._tool_q_dev_chat_log\nWHERE
$__timeFilter(timestamp)",
+ "rawSql": "SELECT ROUND(COUNT(DISTINCT CASE WHEN has_steering = 1
THEN user_id END) / NULLIF(COUNT(DISTINCT user_id), 0) * 100, 0) as 'Steering
%'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)",
"refId": "A"
}
],
- "title": "Steering Adoption",
+ "title": "Steering Adoption (logging)",
"type": "stat"
},
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 7
+ },
+ "id": 101,
+ "panels": [],
+ "title": "User Engagement (logging data: _tool_q_dev_chat_log)",
+ "type": "row"
+ },
{
"datasource": "mysql",
- "description": "Weekly active user count over time",
+ "description": "Weekly active user count over time (from prompt
logging)",
"fieldConfig": {
"defaults": {
"color": {
@@ -312,7 +376,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 6
+ "y": 8
},
"id": 5,
"options": {
@@ -339,7 +403,7 @@
"editorMode": "code",
"format": "time_series",
"rawQuery": true,
- "rawSql": "SELECT\n STR_TO_DATE(CONCAT(YEARWEEK(timestamp, 1), '
Monday'), '%X%V %W') as time,\n COUNT(DISTINCT user_id) as 'Active
Users'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP
BY YEARWEEK(timestamp, 1)\nORDER BY time",
+ "rawSql": "SELECT\n STR_TO_DATE(CONCAT(yw, ' Monday'), '%X%V %W')
as time,\n COUNT(DISTINCT user_id) as 'Active Users'\nFROM (\n SELECT
user_id, YEARWEEK(timestamp, 1) as yw\n FROM lake._tool_q_dev_chat_log\n
WHERE $__timeFilter(timestamp)\n) t\nGROUP BY yw\nORDER BY time",
"refId": "A"
}
],
@@ -348,7 +412,7 @@
},
{
"datasource": "mysql",
- "description": "New vs returning users by week",
+ "description": "New vs returning users by week (from prompt logging)",
"fieldConfig": {
"defaults": {
"color": {
@@ -404,7 +468,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 6
+ "y": 8
},
"id": 6,
"options": {
@@ -431,7 +495,7 @@
"editorMode": "code",
"format": "time_series",
"rawQuery": true,
- "rawSql": "SELECT\n week as time,\n SUM(CASE WHEN is_new = 1 THEN
1 ELSE 0 END) as 'New Users',\n SUM(CASE WHEN is_new = 0 THEN 1 ELSE 0 END) as
'Returning Users'\nFROM (\n SELECT\n u.user_id,\n
STR_TO_DATE(CONCAT(YEARWEEK(u.timestamp, 1), ' Monday'), '%X%V %W') as week,\n
CASE WHEN STR_TO_DATE(CONCAT(YEARWEEK(u.timestamp, 1), ' Monday'), '%X%V %W')
= STR_TO_DATE(CONCAT(YEARWEEK(f.first_seen, 1), ' Monday'), '%X%V %W') THEN 1
ELSE 0 END as is_new\n FROM lake._tool [...]
+ "rawSql": "SELECT\n STR_TO_DATE(CONCAT(yw, ' Monday'), '%X%V %W')
as time,\n SUM(CASE WHEN yw = first_yw THEN 1 ELSE 0 END) as 'New Users',\n
SUM(CASE WHEN yw != first_yw THEN 1 ELSE 0 END) as 'Returning Users'\nFROM (\n
SELECT DISTINCT u.user_id, YEARWEEK(u.timestamp, 1) as yw, f.first_yw\n FROM
lake._tool_q_dev_chat_log u\n JOIN (SELECT user_id, YEARWEEK(MIN(timestamp),
1) as first_yw FROM lake._tool_q_dev_chat_log GROUP BY user_id) f\n ON
u.user_id = f.user_id\n WH [...]
"refId": "A"
}
],
@@ -439,104 +503,21 @@
"type": "timeseries"
},
{
- "datasource": "mysql",
- "description": "Number of users who used each feature",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.8,
- "drawStyle": "bars",
- "fillOpacity": 100,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
+ "collapsed": false,
"gridPos": {
- "h": 8,
- "w": 12,
+ "h": 1,
+ "w": 24,
"x": 0,
- "y": 14
- },
- "id": 7,
- "options": {
- "barRadius": 0.1,
- "barWidth": 0.8,
- "fullHighlight": false,
- "groupWidth": 0.7,
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "orientation": "horizontal",
- "showValue": "auto",
- "stacking": "none",
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- },
- "xTickLabelRotation": 0
+ "y": 16
},
- "pluginVersion": "11.6.2",
- "targets": [
- {
- "datasource": "mysql",
- "editorMode": "code",
- "format": "table",
- "rawQuery": true,
- "rawSql": "SELECT\n 'Chat' as Feature, COUNT(DISTINCT CASE WHEN
chat_messages_sent > 0 THEN user_id END) as Users FROM
lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Inline
Suggestions', COUNT(DISTINCT CASE WHEN inline_suggestions_count > 0 THEN
user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION
ALL SELECT 'Code Fix', COUNT(DISTINCT CASE WHEN code_fix_generation_event_count
> 0 THEN user_id END) FROM lake._tool_q_dev_user_dat [...]
- "refId": "A"
- }
- ],
- "title": "Feature Adoption Funnel",
- "type": "barchart"
+ "id": 102,
+ "panels": [],
+ "title": "Credits & Subscription (new format: _tool_q_dev_user_report)",
+ "type": "row"
},
{
"datasource": "mysql",
- "description": "Cumulative credits this month vs projected total",
+ "description": "Cumulative credits this month vs projected total (from
new user_report)",
"fieldConfig": {
"defaults": {
"color": {
@@ -591,8 +572,8 @@
"gridPos": {
"h": 8,
"w": 12,
- "x": 12,
- "y": 14
+ "x": 0,
+ "y": 17
},
"id": 8,
"options": {
@@ -628,191 +609,7 @@
},
{
"datasource": "mysql",
- "description": "Acceptance rates for inline suggestions, code fix, and
inline chat over time",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "smooth",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "percentunit"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 22
- },
- "id": 9,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "max",
- "sum"
- ],
- "displayMode": "table",
- "placement": "right",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.2",
- "targets": [
- {
- "datasource": "mysql",
- "editorMode": "code",
- "format": "time_series",
- "rawQuery": true,
- "rawSql": "SELECT\n date as time,\n SUM(inline_acceptance_count) /
NULLIF(SUM(inline_suggestions_count), 0) as 'Inline Suggestions',\n
SUM(code_fix_acceptance_event_count) /
NULLIF(SUM(code_fix_generation_event_count), 0) as 'Code Fix',\n
SUM(inline_chat_acceptance_event_count) /
NULLIF(SUM(inline_chat_total_event_count), 0) as 'Inline Chat'\nFROM
lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY
date",
- "refId": "A"
- }
- ],
- "title": "Acceptance Rate Trends",
- "type": "timeseries"
- },
- {
- "datasource": "mysql",
- "description": "Code review findings and test generation metrics over
time",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 10,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "smooth",
- "lineWidth": 2,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "never",
- "spanNulls": true,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 22
- },
- "id": 10,
- "options": {
- "legend": {
- "calcs": [
- "mean",
- "max",
- "sum"
- ],
- "displayMode": "table",
- "placement": "right",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.2",
- "targets": [
- {
- "datasource": "mysql",
- "editorMode": "code",
- "format": "time_series",
- "rawQuery": true,
- "rawSql": "SELECT\n date as time,\n
SUM(code_review_findings_count) as 'Review Findings',\n
SUM(test_generation_event_count) as 'Test Gen Events',\n
SUM(test_generation_accepted_tests) as 'Tests Accepted'\nFROM
lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY
date",
- "refId": "A"
- }
- ],
- "title": "Code Review Findings & Test Generation",
- "type": "timeseries"
- },
- {
- "datasource": "mysql",
- "description": "Per-user productivity and efficiency metrics",
+ "description": "Power tier users with no activity in the last 14 days
(from new user_report)",
"fieldConfig": {
"defaults": {
"color": {
@@ -839,12 +636,12 @@
"overrides": []
},
"gridPos": {
- "h": 10,
- "w": 24,
- "x": 0,
- "y": 30
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 17
},
- "id": 11,
+ "id": 12,
"options": {
"cellHeight": "sm",
"footer": {
@@ -865,16 +662,29 @@
"editorMode": "code",
"format": "table",
"rawQuery": true,
- "rawSql": "SELECT\n COALESCE(MAX(d.display_name), d.user_id) as
'User',\n COALESCE(MAX(r.subscription_tier), '') as 'Tier',\n
ROUND(SUM(r.credits_used), 1) as 'Credits Used',\n SUM(d.chat_ai_code_lines +
d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) as
'Total Accepted Lines',\n CASE WHEN SUM(d.chat_ai_code_lines +
d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) >
0\n THEN ROUND(SUM(r.credits_used) / SUM(d.chat_ai_c [...]
+ "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as
'User',\n MAX(subscription_tier) as 'Tier',\n ROUND(SUM(credits_used), 1) as
'Total Credits Used',\n MAX(date) as 'Last Activity'\nFROM
lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\n AND
subscription_tier = 'POWER'\nGROUP BY user_id\nHAVING MAX(date) <
DATE_SUB(NOW(), INTERVAL 14 DAY)\nORDER BY MAX(date)",
"refId": "A"
}
],
- "title": "User Productivity & Efficiency",
+ "title": "Idle Power Users (No Activity in 14 Days)",
"type": "table"
},
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 25
+ },
+ "id": 103,
+ "panels": [],
+ "title": "Cross-Source: User Productivity (new report + legacy metrics)",
+ "type": "row"
+ },
{
"datasource": "mysql",
- "description": "Power tier users with no activity in the last 14 days",
+ "description": "Per-user productivity combining credits (new format)
with feature metrics (legacy). Only shows users present in both data sources.",
"fieldConfig": {
"defaults": {
"color": {
@@ -901,12 +711,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
+ "h": 10,
"w": 24,
"x": 0,
- "y": 40
+ "y": 26
},
- "id": 12,
+ "id": 11,
"options": {
"cellHeight": "sm",
"footer": {
@@ -927,11 +737,11 @@
"editorMode": "code",
"format": "table",
"rawQuery": true,
- "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as
'User',\n MAX(subscription_tier) as 'Tier',\n ROUND(SUM(credits_used), 1) as
'Total Credits Used',\n MAX(date) as 'Last Activity'\nFROM
lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\n AND
subscription_tier = 'POWER'\nGROUP BY user_id\nHAVING MAX(date) <
DATE_SUB(NOW(), INTERVAL 14 DAY)\nORDER BY MAX(date)",
+ "rawSql": "SELECT\n COALESCE(MAX(d.display_name), d.user_id) as
'User',\n COALESCE(MAX(r.subscription_tier), '') as 'Tier',\n
ROUND(SUM(r.credits_used), 1) as 'Credits Used',\n SUM(d.chat_ai_code_lines +
d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) as
'Total Accepted Lines',\n CASE WHEN SUM(d.chat_ai_code_lines +
d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) >
0\n THEN ROUND(SUM(r.credits_used) / SUM(d.chat_ai_c [...]
"refId": "A"
}
],
- "title": "Idle Power Users (No Activity in 14 Days)",
+ "title": "User Productivity & Efficiency",
"type": "table"
}
],
@@ -955,4 +765,4 @@
"title": "Kiro Executive Dashboard",
"uid": "qdev_executive",
"version": 1
-}
\ No newline at end of file
+}
diff --git a/grafana/dashboards/qdev_executive.json
b/grafana/dashboards/qdev_feature_metrics.json
similarity index 63%
copy from grafana/dashboards/qdev_executive.json
copy to grafana/dashboards/qdev_feature_metrics.json
index c6e2524d7..597217bcf 100644
--- a/grafana/dashboards/qdev_executive.json
+++ b/grafana/dashboards/qdev_feature_metrics.json
@@ -20,7 +20,7 @@
"panels": [
{
"datasource": "mysql",
- "description": "Distinct users with chat activity in the last 7 days",
+ "description": "High-level summary of legacy feature-level activity
metrics (from by_user_analytic CSV reports)",
"fieldConfig": {
"defaults": {
"color": {
@@ -40,7 +40,7 @@
},
"gridPos": {
"h": 6,
- "w": 6,
+ "w": 24,
"x": 0,
"y": 0
},
@@ -70,138 +70,66 @@
"editorMode": "code",
"format": "table",
"rawQuery": true,
- "rawSql": "SELECT COUNT(DISTINCT user_id) as 'WAU'\nFROM
lake._tool_q_dev_chat_log\nWHERE timestamp >= DATE_SUB(NOW(), INTERVAL 7 DAY)",
+ "rawSql": "SELECT\n COUNT(DISTINCT user_id) as 'Active Users',\n
SUM(inline_suggestions_count) as 'Inline Suggestions',\n
SUM(inline_acceptance_count) as 'Inline Accepted',\n SUM(chat_messages_sent)
as 'Chat Messages',\n SUM(chat_ai_code_lines) as 'Chat AI Lines',\n
SUM(code_review_findings_count) as 'Review Findings',\n
SUM(test_generation_event_count) as 'Test Gen Events',\n
SUM(dev_accepted_lines) as 'Agentic Lines'\nFROM
lake._tool_q_dev_user_data\nWHERE $__timeFilt [...]
"refId": "A"
}
],
- "title": "Weekly Active Users",
+ "title": "Legacy Feature Metrics Overview",
"type": "stat"
},
{
- "datasource": "mysql",
- "description": "Average credits spent per accepted line of code",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- }
- },
- "overrides": []
- },
+ "collapsed": false,
"gridPos": {
- "h": 6,
- "w": 6,
- "x": 6,
- "y": 0
- },
- "id": 2,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "sum"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "text": {},
- "textMode": "auto",
- "wideLayout": true
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 6
},
- "pluginVersion": "11.6.2",
- "targets": [
- {
- "datasource": "mysql",
- "editorMode": "code",
- "format": "table",
- "rawQuery": true,
- "rawSql": "SELECT ROUND(SUM(r.credits_used) /
NULLIF(SUM(d.total_accepted), 0), 2) as 'Credits per Accepted Line'\nFROM (\n
SELECT user_id, date, SUM(credits_used) as credits_used\n FROM
lake._tool_q_dev_user_report\n WHERE $__timeFilter(date)\n GROUP BY user_id,
date\n) r\nJOIN (\n SELECT user_id, date,\n (inline_ai_code_lines +
chat_ai_code_lines + code_fix_accepted_lines + dev_accepted_lines) as
total_accepted\n FROM lake._tool_q_dev_user_data\n WHERE $__timeFilter [...]
- "refId": "A"
- }
- ],
- "title": "Credits Efficiency",
- "type": "stat"
+ "id": 20,
+ "panels": [],
+ "title": "Inline Suggestions",
+ "type": "row"
},
{
"datasource": "mysql",
- "description": "Percentage of inline suggestions accepted",
+ "description": "Daily inline suggestion and acceptance counts",
"fieldConfig": {
"defaults": {
"color": {
- "mode": "thresholds"
+ "mode": "palette-classic"
},
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 6,
- "w": 6,
- "x": 12,
- "y": 0
- },
- "id": 3,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "sum"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "text": {},
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.6.2",
- "targets": [
- {
- "datasource": "mysql",
- "editorMode": "code",
- "format": "table",
- "rawQuery": true,
- "rawSql": "SELECT ROUND(SUM(inline_acceptance_count) /
NULLIF(SUM(inline_suggestions_count), 0) * 100, 1) as 'Acceptance %'\nFROM
lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)",
- "refId": "A"
- }
- ],
- "title": "Inline Acceptance Rate",
- "type": "stat"
- },
- {
- "datasource": "mysql",
- "description": "Percentage of users using steering rules",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "smooth",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
},
"mappings": [],
"thresholds": {
@@ -211,52 +139,51 @@
"color": "green"
}
]
- }
+ },
+ "unit": "short"
},
"overrides": []
},
"gridPos": {
- "h": 6,
- "w": 6,
- "x": 18,
- "y": 0
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 7
},
- "id": 4,
+ "id": 2,
"options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
+ "legend": {
"calcs": [
+ "mean",
"sum"
],
- "fields": "",
- "values": false
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
},
- "showPercentChange": false,
- "text": {},
- "textMode": "auto",
- "wideLayout": true
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "none"
+ }
},
"pluginVersion": "11.6.2",
"targets": [
{
"datasource": "mysql",
"editorMode": "code",
- "format": "table",
+ "format": "time_series",
"rawQuery": true,
- "rawSql": "SELECT CONCAT(ROUND(COUNT(DISTINCT CASE WHEN has_steering
= 1 THEN user_id END) / NULLIF(COUNT(DISTINCT user_id), 0) * 100, 0), '%') as
'Users with Steering'\nFROM lake._tool_q_dev_chat_log\nWHERE
$__timeFilter(timestamp)",
+ "rawSql": "SELECT\n date as time,\n SUM(inline_suggestions_count)
as 'Suggestions',\n SUM(inline_acceptance_count) as 'Accepted',\n
SUM(inline_ai_code_lines) as 'AI Code Lines'\nFROM
lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY
date",
"refId": "A"
}
],
- "title": "Steering Adoption",
- "type": "stat"
+ "title": "Inline Suggestions & Acceptance",
+ "type": "timeseries"
},
{
"datasource": "mysql",
- "description": "Weekly active user count over time",
+ "description": "Acceptance rates for inline suggestions, code fix, and
inline chat over time",
"fieldConfig": {
"defaults": {
"color": {
@@ -304,23 +231,22 @@
}
]
},
- "unit": "short"
+ "unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
- "x": 0,
- "y": 6
+ "x": 12,
+ "y": 7
},
- "id": 5,
+ "id": 3,
"options": {
"legend": {
"calcs": [
"mean",
- "max",
- "sum"
+ "max"
],
"displayMode": "table",
"placement": "right",
@@ -339,16 +265,29 @@
"editorMode": "code",
"format": "time_series",
"rawQuery": true,
- "rawSql": "SELECT\n STR_TO_DATE(CONCAT(YEARWEEK(timestamp, 1), '
Monday'), '%X%V %W') as time,\n COUNT(DISTINCT user_id) as 'Active
Users'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP
BY YEARWEEK(timestamp, 1)\nORDER BY time",
+ "rawSql": "SELECT\n date as time,\n SUM(inline_acceptance_count) /
NULLIF(SUM(inline_suggestions_count), 0) as 'Inline Suggestions',\n
SUM(code_fix_acceptance_event_count) /
NULLIF(SUM(code_fix_generation_event_count), 0) as 'Code Fix',\n
SUM(inline_chat_acceptance_event_count) /
NULLIF(SUM(inline_chat_total_event_count), 0) as 'Inline Chat'\nFROM
lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY
date",
"refId": "A"
}
],
- "title": "Weekly Active Users Trend",
+ "title": "Acceptance Rate Trends",
"type": "timeseries"
},
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 15
+ },
+ "id": 21,
+ "panels": [],
+ "title": "Chat & Agentic (Dev)",
+ "type": "row"
+ },
{
"datasource": "mysql",
- "description": "New vs returning users by week",
+ "description": "Daily chat messages sent and AI-generated code lines
from chat",
"fieldConfig": {
"defaults": {
"color": {
@@ -403,15 +342,14 @@
"gridPos": {
"h": 8,
"w": 12,
- "x": 12,
- "y": 6
+ "x": 0,
+ "y": 16
},
- "id": 6,
+ "id": 4,
"options": {
"legend": {
"calcs": [
"mean",
- "max",
"sum"
],
"displayMode": "table",
@@ -431,16 +369,16 @@
"editorMode": "code",
"format": "time_series",
"rawQuery": true,
- "rawSql": "SELECT\n week as time,\n SUM(CASE WHEN is_new = 1 THEN
1 ELSE 0 END) as 'New Users',\n SUM(CASE WHEN is_new = 0 THEN 1 ELSE 0 END) as
'Returning Users'\nFROM (\n SELECT\n u.user_id,\n
STR_TO_DATE(CONCAT(YEARWEEK(u.timestamp, 1), ' Monday'), '%X%V %W') as week,\n
CASE WHEN STR_TO_DATE(CONCAT(YEARWEEK(u.timestamp, 1), ' Monday'), '%X%V %W')
= STR_TO_DATE(CONCAT(YEARWEEK(f.first_seen, 1), ' Monday'), '%X%V %W') THEN 1
ELSE 0 END as is_new\n FROM lake._tool [...]
+ "rawSql": "SELECT\n date as time,\n SUM(chat_messages_sent) as
'Messages Sent',\n SUM(chat_messages_interacted) as 'Messages Interacted',\n
SUM(chat_ai_code_lines) as 'AI Code Lines'\nFROM
lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY
date",
"refId": "A"
}
],
- "title": "New vs Returning Users (Weekly)",
+ "title": "Chat Activity",
"type": "timeseries"
},
{
"datasource": "mysql",
- "description": "Number of users who used each feature",
+ "description": "Agentic (Dev) code generation and acceptance metrics",
"fieldConfig": {
"defaults": {
"color": {
@@ -453,9 +391,9 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
- "barWidthFactor": 0.8,
- "drawStyle": "bars",
- "fillOpacity": 100,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
@@ -463,8 +401,8 @@
"viz": false
},
"insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
+ "lineInterpolation": "smooth",
+ "lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
@@ -495,48 +433,56 @@
"gridPos": {
"h": 8,
"w": 12,
- "x": 0,
- "y": 14
+ "x": 12,
+ "y": 16
},
- "id": 7,
+ "id": 5,
"options": {
- "barRadius": 0.1,
- "barWidth": 0.8,
- "fullHighlight": false,
- "groupWidth": 0.7,
"legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
+ "calcs": [
+ "mean",
+ "sum"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
},
- "orientation": "horizontal",
- "showValue": "auto",
- "stacking": "none",
"tooltip": {
"hideZeros": false,
- "mode": "single",
+ "mode": "multi",
"sort": "none"
- },
- "xTickLabelRotation": 0
+ }
},
"pluginVersion": "11.6.2",
"targets": [
{
"datasource": "mysql",
"editorMode": "code",
- "format": "table",
+ "format": "time_series",
"rawQuery": true,
- "rawSql": "SELECT\n 'Chat' as Feature, COUNT(DISTINCT CASE WHEN
chat_messages_sent > 0 THEN user_id END) as Users FROM
lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Inline
Suggestions', COUNT(DISTINCT CASE WHEN inline_suggestions_count > 0 THEN
user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION
ALL SELECT 'Code Fix', COUNT(DISTINCT CASE WHEN code_fix_generation_event_count
> 0 THEN user_id END) FROM lake._tool_q_dev_user_dat [...]
+ "rawSql": "SELECT\n date as time,\n
SUM(dev_generation_event_count) as 'Generation Events',\n
SUM(dev_generated_lines) as 'Generated Lines',\n SUM(dev_accepted_lines) as
'Accepted Lines',\n SUM(dev_acceptance_event_count) as 'Acceptance
Events'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY
date\nORDER BY date",
"refId": "A"
}
],
- "title": "Feature Adoption Funnel",
- "type": "barchart"
+ "title": "Agentic (Dev) Activity",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 24
+ },
+ "id": 22,
+ "panels": [],
+ "title": "Code Review, Test Gen & Transformations",
+ "type": "row"
},
{
"datasource": "mysql",
- "description": "Cumulative credits this month vs projected total",
+ "description": "Code review findings and test generation metrics over
time",
"fieldConfig": {
"defaults": {
"color": {
@@ -591,15 +537,14 @@
"gridPos": {
"h": 8,
"w": 12,
- "x": 12,
- "y": 14
+ "x": 0,
+ "y": 25
},
- "id": 8,
+ "id": 6,
"options": {
"legend": {
"calcs": [
"mean",
- "max",
"sum"
],
"displayMode": "table",
@@ -619,16 +564,16 @@
"editorMode": "code",
"format": "time_series",
"rawQuery": true,
- "rawSql": "SELECT\n date as time,\n SUM(SUM(credits_used)) OVER
(ORDER BY date) as 'Cumulative Credits',\n (SELECT SUM(credits_used) /
COUNT(DISTINCT date) * DAY(LAST_DAY(CURDATE()))\n FROM
lake._tool_q_dev_user_report\n WHERE YEAR(date) = YEAR(CURDATE()) AND
MONTH(date) = MONTH(CURDATE())) as 'Projected Monthly'\nFROM
lake._tool_q_dev_user_report\nWHERE YEAR(date) = YEAR(CURDATE()) AND
MONTH(date) = MONTH(CURDATE())\nGROUP BY date\nORDER BY date",
+ "rawSql": "SELECT\n date as time,\n
SUM(code_review_findings_count) as 'Review Findings',\n
SUM(code_review_succeeded_event_count) as 'Reviews Succeeded',\n
SUM(code_review_failed_event_count) as 'Reviews Failed'\nFROM
lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY
date",
"refId": "A"
}
],
- "title": "Credits Pace vs Projected (This Month)",
+ "title": "Code Review Activity",
"type": "timeseries"
},
{
"datasource": "mysql",
- "description": "Acceptance rates for inline suggestions, code fix, and
inline chat over time",
+ "description": "Test generation events and acceptance over time",
"fieldConfig": {
"defaults": {
"color": {
@@ -676,22 +621,21 @@
}
]
},
- "unit": "percentunit"
+ "unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
- "x": 0,
- "y": 22
+ "x": 12,
+ "y": 25
},
- "id": 9,
+ "id": 7,
"options": {
"legend": {
"calcs": [
"mean",
- "max",
"sum"
],
"displayMode": "table",
@@ -711,16 +655,16 @@
"editorMode": "code",
"format": "time_series",
"rawQuery": true,
- "rawSql": "SELECT\n date as time,\n SUM(inline_acceptance_count) /
NULLIF(SUM(inline_suggestions_count), 0) as 'Inline Suggestions',\n
SUM(code_fix_acceptance_event_count) /
NULLIF(SUM(code_fix_generation_event_count), 0) as 'Code Fix',\n
SUM(inline_chat_acceptance_event_count) /
NULLIF(SUM(inline_chat_total_event_count), 0) as 'Inline Chat'\nFROM
lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY
date",
+ "rawSql": "SELECT\n date as time,\n
SUM(test_generation_event_count) as 'Test Gen Events',\n
SUM(test_generation_generated_tests) as 'Tests Generated',\n
SUM(test_generation_accepted_tests) as 'Tests Accepted',\n
SUM(test_generation_generated_lines) as 'Lines Generated',\n
SUM(test_generation_accepted_lines) as 'Lines Accepted'\nFROM
lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY
date",
"refId": "A"
}
],
- "title": "Acceptance Rate Trends",
+ "title": "Test Generation Activity",
"type": "timeseries"
},
{
"datasource": "mysql",
- "description": "Code review findings and test generation metrics over
time",
+ "description": "Doc generation and code transformation events",
"fieldConfig": {
"defaults": {
"color": {
@@ -775,15 +719,14 @@
"gridPos": {
"h": 8,
"w": 12,
- "x": 12,
- "y": 22
+ "x": 0,
+ "y": 33
},
- "id": 10,
+ "id": 8,
"options": {
"legend": {
"calcs": [
"mean",
- "max",
"sum"
],
"displayMode": "table",
@@ -803,28 +746,53 @@
"editorMode": "code",
"format": "time_series",
"rawQuery": true,
- "rawSql": "SELECT\n date as time,\n
SUM(code_review_findings_count) as 'Review Findings',\n
SUM(test_generation_event_count) as 'Test Gen Events',\n
SUM(test_generation_accepted_tests) as 'Tests Accepted'\nFROM
lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY
date",
+ "rawSql": "SELECT\n date as time,\n
SUM(doc_generation_event_count) as 'Doc Gen Events',\n
SUM(doc_generation_accepted_line_additions) as 'Doc Lines Accepted',\n
SUM(transformation_event_count) as 'Transformation Events',\n
SUM(transformation_lines_generated) as 'Transform Lines Generated'\nFROM
lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY
date",
"refId": "A"
}
],
- "title": "Code Review Findings & Test Generation",
+ "title": "Doc Generation & Transformations",
"type": "timeseries"
},
{
"datasource": "mysql",
- "description": "Per-user productivity and efficiency metrics",
+ "description": "Number of users who used each feature in the selected
period",
"fieldConfig": {
"defaults": {
"color": {
- "mode": "thresholds"
+ "mode": "palette-classic"
},
"custom": {
- "align": "auto",
- "cellOptions": {
- "type": "auto"
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.8,
+ "drawStyle": "bars",
+ "fillOpacity": 100,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "filterable": true,
- "inspect": false
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
},
"mappings": [],
"thresholds": {
@@ -834,29 +802,38 @@
"color": "green"
}
]
- }
+ },
+ "unit": "short"
},
"overrides": []
},
"gridPos": {
- "h": 10,
- "w": 24,
- "x": 0,
- "y": 30
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 33
},
- "id": 11,
+ "id": 9,
"options": {
- "cellHeight": "sm",
- "footer": {
- "countRows": false,
- "fields": "",
- "reducer": [
- "sum"
- ],
- "show": false
+ "barRadius": 0.1,
+ "barWidth": 0.8,
+ "fullHighlight": false,
+ "groupWidth": 0.7,
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
},
- "showHeader": true,
- "sortBy": []
+ "orientation": "horizontal",
+ "showValue": "auto",
+ "stacking": "none",
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ },
+ "xTickLabelRotation": 0
},
"pluginVersion": "11.6.2",
"targets": [
@@ -865,16 +842,29 @@
"editorMode": "code",
"format": "table",
"rawQuery": true,
- "rawSql": "SELECT\n COALESCE(MAX(d.display_name), d.user_id) as
'User',\n COALESCE(MAX(r.subscription_tier), '') as 'Tier',\n
ROUND(SUM(r.credits_used), 1) as 'Credits Used',\n SUM(d.chat_ai_code_lines +
d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) as
'Total Accepted Lines',\n CASE WHEN SUM(d.chat_ai_code_lines +
d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) >
0\n THEN ROUND(SUM(r.credits_used) / SUM(d.chat_ai_c [...]
+ "rawSql": "SELECT 'Chat' as Feature, COUNT(DISTINCT CASE WHEN
chat_messages_sent > 0 THEN user_id END) as Users FROM
lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Inline
Suggestions', COUNT(DISTINCT CASE WHEN inline_suggestions_count > 0 THEN
user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION
ALL SELECT 'Code Fix', COUNT(DISTINCT CASE WHEN code_fix_generation_event_count
> 0 THEN user_id END) FROM lake._tool_q_dev_user_data W [...]
"refId": "A"
}
],
- "title": "User Productivity & Efficiency",
- "type": "table"
+ "title": "Feature Adoption (Users per Feature)",
+ "type": "barchart"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 41
+ },
+ "id": 23,
+ "panels": [],
+ "title": "Per-User Detail",
+ "type": "row"
},
{
"datasource": "mysql",
- "description": "Power tier users with no activity in the last 14 days",
+ "description": "Per-user breakdown of legacy feature-level metrics",
"fieldConfig": {
"defaults": {
"color": {
@@ -901,12 +891,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
+ "h": 10,
"w": 24,
"x": 0,
- "y": 40
+ "y": 42
},
- "id": 12,
+ "id": 10,
"options": {
"cellHeight": "sm",
"footer": {
@@ -927,11 +917,11 @@
"editorMode": "code",
"format": "table",
"rawQuery": true,
- "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as
'User',\n MAX(subscription_tier) as 'Tier',\n ROUND(SUM(credits_used), 1) as
'Total Credits Used',\n MAX(date) as 'Last Activity'\nFROM
lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\n AND
subscription_tier = 'POWER'\nGROUP BY user_id\nHAVING MAX(date) <
DATE_SUB(NOW(), INTERVAL 14 DAY)\nORDER BY MAX(date)",
+ "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as
'User',\n SUM(inline_suggestions_count) as 'Suggestions',\n
SUM(inline_acceptance_count) as 'Accepted',\n
CONCAT(ROUND(SUM(inline_acceptance_count) /
NULLIF(SUM(inline_suggestions_count), 0) * 100, 1), '%') as 'Accept %',\n
SUM(chat_messages_sent) as 'Chat Msgs',\n SUM(chat_ai_code_lines) as 'Chat
Lines',\n SUM(dev_accepted_lines) as 'Agentic Lines',\n
SUM(code_review_findings_count) as 'Review Findings',\n SU [...]
"refId": "A"
}
],
- "title": "Idle Power Users (No Activity in 14 Days)",
+ "title": "Per-User Feature Metrics",
"type": "table"
}
],
@@ -940,7 +930,7 @@
"schemaVersion": 41,
"tags": [
"q_dev",
- "executive",
+ "legacy",
"kiro"
],
"templating": {
@@ -952,7 +942,7 @@
},
"timepicker": {},
"timezone": "utc",
- "title": "Kiro Executive Dashboard",
- "uid": "qdev_executive",
+ "title": "Kiro Legacy Feature Metrics",
+ "uid": "qdev_feature_metrics",
"version": 1
-}
\ No newline at end of file
+}
diff --git a/grafana/dashboards/qdev_logging.json
b/grafana/dashboards/qdev_logging.json
index 462adcd98..6a47b03bf 100644
--- a/grafana/dashboards/qdev_logging.json
+++ b/grafana/dashboards/qdev_logging.json
@@ -70,7 +70,7 @@
"editorMode": "code",
"format": "table",
"rawQuery": true,
- "rawSql": "SELECT\n (SELECT COUNT(*) FROM lake._tool_q_dev_chat_log
WHERE $__timeFilter(timestamp)) as 'Chat Events',\n (SELECT COUNT(DISTINCT
user_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as
'Chat Users',\n (SELECT COUNT(DISTINCT conversation_id) FROM
lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) AND conversation_id !=
'') as 'Conversations',\n (SELECT COUNT(*) FROM
lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) as 'Com [...]
+ "rawSql": "SELECT\n (SELECT COUNT(*) FROM lake._tool_q_dev_chat_log
WHERE $__timeFilter(timestamp)) as 'Chat Events',\n (SELECT COUNT(DISTINCT
user_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as
'Chat Users',\n (SELECT COUNT(DISTINCT conversation_id) FROM
lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) AND conversation_id !=
'') as 'Conversations',\n (SELECT COUNT(*) FROM
lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) as 'Com [...]
"refId": "A"
}
],
@@ -168,7 +168,7 @@
},
{
"datasource": "mysql",
- "description": "Distribution of model usage across chat events",
+ "description": "Distribution of chat trigger types: MANUAL (chat window)
vs INLINE_CHAT",
"fieldConfig": {
"defaults": {
"color": {
@@ -188,10 +188,78 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
"y": 14
},
+ "id": 11,
+ "options": {
+ "displayLabels": [
+ "name",
+ "percent"
+ ],
+ "legend": {
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true,
+ "values": [
+ "value",
+ "percent"
+ ]
+ },
+ "pieType": "donut",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.2",
+ "targets": [
+ {
+ "datasource": "mysql",
+ "editorMode": "code",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "SELECT\n CASE\n WHEN chat_trigger_type = '' OR
chat_trigger_type IS NULL THEN '(unknown)'\n ELSE chat_trigger_type\n END
as 'Trigger Type',\n COUNT(*) as 'Events'\nFROM
lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY
chat_trigger_type\nORDER BY COUNT(*) DESC",
+ "refId": "A"
+ }
+ ],
+ "title": "Chat Trigger Type Distribution",
+ "type": "piechart"
+ },
+ {
+ "datasource": "mysql",
+ "description": "Distribution of model usage across chat events",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 14
+ },
"id": 3,
"options": {
"displayLabels": [
@@ -256,8 +324,8 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
+ "w": 8,
+ "x": 16,
"y": 14
},
"id": 4,
@@ -541,7 +609,7 @@
"editorMode": "code",
"format": "table",
"rawQuery": true,
- "rawSql": "SELECT\n COALESCE(u.display_name, u.user_id) as
'User',\n u.user_id as 'User ID',\n u.chat_events as 'Chat Events',\n
u.conversations as 'Conversations',\n ROUND(u.chat_events /
NULLIF(u.conversations, 0), 1) as 'Avg Turns',\n COALESCE(c.completion_events,
0) as 'Completion Events',\n COALESCE(c.files_count, 0) as 'Distinct Files',\n
ROUND(u.avg_prompt_len) as 'Avg Prompt Len',\n ROUND(u.avg_response_len) as
'Avg Response Len',\n u.steering_count as 'Steeri [...]
+ "rawSql": "SELECT\n COALESCE(u.display_name, u.user_id) as
'User',\n u.user_id as 'User ID',\n u.chat_events as 'Chat Events',\n
u.conversations as 'Conversations',\n ROUND(u.chat_events /
NULLIF(u.conversations, 0), 1) as 'Avg Turns',\n COALESCE(c.completion_events,
0) as 'Completion Events',\n COALESCE(c.files_count, 0) as 'Distinct Files',\n
ROUND(u.avg_prompt_len) as 'Avg Prompt Len',\n ROUND(u.avg_response_len) as
'Avg Response Len',\n u.steering_count as 'Steeri [...]
"refId": "A"
}
],
@@ -570,7 +638,7 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
"y": 48
},
@@ -638,8 +706,8 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
+ "w": 8,
+ "x": 8,
"y": 48
},
"id": 9,
@@ -684,6 +752,74 @@
"title": "Active File Types in Chat",
"type": "piechart"
},
+ {
+ "datasource": "mysql",
+ "description": "How often Kiro responses include code references and web
links",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 48
+ },
+ "id": 12,
+ "options": {
+ "displayLabels": [
+ "name",
+ "percent"
+ ],
+ "legend": {
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true,
+ "values": [
+ "value",
+ "percent"
+ ]
+ },
+ "pieType": "donut",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.2",
+ "targets": [
+ {
+ "datasource": "mysql",
+ "editorMode": "code",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "SELECT\n SUM(CASE WHEN code_reference_count > 0 THEN 1
ELSE 0 END) as 'With Code References',\n SUM(CASE WHEN web_link_count > 0 THEN
1 ELSE 0 END) as 'With Web Links',\n SUM(CASE WHEN has_followup_prompts = 1
THEN 1 ELSE 0 END) as 'With Followup Prompts',\n SUM(CASE WHEN
code_reference_count = 0 AND web_link_count = 0 AND has_followup_prompts = 0
THEN 1 ELSE 0 END) as 'Plain Response'\nFROM lake._tool_q_dev_chat_log\nWHERE
$__timeFilter(timestamp)",
+ "refId": "A"
+ }
+ ],
+ "title": "Response Enrichment Breakdown",
+ "type": "piechart"
+ },
{
"datasource": "mysql",
"description": "Average and maximum prompt/response lengths over time",
@@ -775,6 +911,188 @@
],
"title": "Prompt & Response Length Trends",
"type": "timeseries"
+ },
+ {
+ "datasource": "mysql",
+ "description": "Average code context size provided to inline completions
over time",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "Characters",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "smooth",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 64
+ },
+ "id": 13,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.2",
+ "targets": [
+ {
+ "datasource": "mysql",
+ "editorMode": "code",
+ "format": "time_series",
+ "rawQuery": true,
+ "rawSql": "SELECT\n DATE(timestamp) as time,\n
ROUND(AVG(left_context_length)) as 'Avg Left Context',\n
ROUND(AVG(right_context_length)) as 'Avg Right Context',\n
ROUND(AVG(left_context_length + right_context_length)) as 'Avg Total
Context'\nFROM lake._tool_q_dev_completion_log\nWHERE
$__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)",
+ "refId": "A"
+ }
+ ],
+ "title": "Completion Context Size Trends",
+ "type": "timeseries"
+ },
+ {
+ "datasource": "mysql",
+ "description": "Daily trend of code references and web links in chat
responses",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "smooth",
+ "lineWidth": 2,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "never",
+ "spanNulls": true,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 72
+ },
+ "id": 14,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "sum"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.2",
+ "targets": [
+ {
+ "datasource": "mysql",
+ "editorMode": "code",
+ "format": "time_series",
+ "rawQuery": true,
+ "rawSql": "SELECT\n DATE(timestamp) as time,\n
SUM(code_reference_count) as 'Code References',\n SUM(web_link_count) as 'Web
Links',\n SUM(CASE WHEN has_followup_prompts = 1 THEN 1 ELSE 0 END) as
'Followup Prompts'\nFROM lake._tool_q_dev_chat_log\nWHERE
$__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)",
+ "refId": "A"
+ }
+ ],
+ "title": "Response Enrichment Trends",
+ "type": "timeseries"
}
],
"preload": false,
@@ -797,4 +1115,4 @@
"title": "Kiro AI Activity Insights",
"uid": "qdev_logging",
"version": 1
-}
\ No newline at end of file
+}
