This is an automated email from the ASF dual-hosted git repository. warren pushed a commit to branch feat/q-dev-logging-dashboard-enrichment in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 048ce43c3aa3d7ed0b8fcab92a1355e0c75d0c76 Author: warren <[email protected]> AuthorDate: Sat Mar 14 23:46:09 2026 +0800 feat(q-dev): add logging data ingestion and enrich Kiro dashboards Add support for ingesting S3 logging data (GenerateAssistantResponse and GenerateCompletions events) into new database tables, and enrich all three Kiro Grafana dashboards with additional metrics. Changes: - New models: QDevChatLog and QDevCompletionLog for logging event data - New extractor: s3_logging_extractor.go parses JSON.gz logging files - Updated S3 collector to also handle .json.gz files - Added logging S3 prefixes (GenerateAssistantResponse, GenerateCompletions) - New dashboard: "Kiro AI Activity Insights" with 10 panels including model usage distribution, active hours, conversation depth, feature adoption (Steering/Spec), file type usage, and prompt/response trends - Enriched "Kiro Code Metrics Dashboard" with DocGeneration, TestGeneration, and Dev (Agentic) metric panels - Fixed "Kiro Usage Dashboard" per-user table to sort by user_id - Migration script for new tables --- backend/plugins/q_dev/impl/impl.go | 5 + backend/plugins/q_dev/impl/impl_test.go | 4 +- backend/plugins/q_dev/models/chat_log.go | 51 ++ backend/plugins/q_dev/models/completion_log.go | 43 ++ ...{register.go => 20260314_add_logging_tables.go} | 36 +- .../models/migrationscripts/archived/chat_log.go | 50 ++ .../migrationscripts/archived/completion_log.go | 42 ++ .../q_dev/models/migrationscripts/register.go | 1 + backend/plugins/q_dev/tasks/s3_file_collector.go | 6 +- .../plugins/q_dev/tasks/s3_logging_extractor.go | 308 ++++++++ grafana/dashboards/qdev_logging.json | 800 +++++++++++++++++++++ grafana/dashboards/qdev_user_data.json | 389 +++++++++- grafana/dashboards/qdev_user_report.json | 2 +- 13 files changed, 1714 insertions(+), 23 deletions(-) diff --git a/backend/plugins/q_dev/impl/impl.go b/backend/plugins/q_dev/impl/impl.go index e38fe7ad7..a0435e58b 100644 --- a/backend/plugins/q_dev/impl/impl.go +++ b/backend/plugins/q_dev/impl/impl.go @@ -58,6 +58,8 @@ func (p QDev) GetTablesInfo() []dal.Tabler { &models.QDevS3FileMeta{}, &models.QDevS3Slice{}, &models.QDevUserReport{}, + &models.QDevChatLog{}, + &models.QDevCompletionLog{}, } } @@ -85,6 +87,7 @@ func (p QDev) SubTaskMetas() []plugin.SubTaskMeta { return []plugin.SubTaskMeta{ tasks.CollectQDevS3FilesMeta, tasks.ExtractQDevS3DataMeta, + tasks.ExtractQDevLoggingDataMeta, } } @@ -131,6 +134,8 @@ func (p QDev) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int s3Prefixes = []string{ fmt.Sprintf("%s/by_user_analytic/%s/%s", base, region, timePart), fmt.Sprintf("%s/user_report/%s/%s", base, region, timePart), + fmt.Sprintf("%s/GenerateAssistantResponse/%s/%s", base, region, timePart), + fmt.Sprintf("%s/GenerateCompletions/%s/%s", base, region, timePart), } } else { // Legacy scope: use S3Prefix directly diff --git a/backend/plugins/q_dev/impl/impl_test.go b/backend/plugins/q_dev/impl/impl_test.go index e61b53251..7153617ab 100644 --- a/backend/plugins/q_dev/impl/impl_test.go +++ b/backend/plugins/q_dev/impl/impl_test.go @@ -34,11 +34,11 @@ func TestQDev_BasicPluginMethods(t *testing.T) { // Test table info tables := plugin.GetTablesInfo() - assert.Len(t, tables, 5) + assert.Len(t, tables, 7) // Test subtask metas subtasks := plugin.SubTaskMetas() - assert.Len(t, subtasks, 2) + assert.Len(t, subtasks, 3) // Test API resources apiResources := plugin.ApiResources() diff --git a/backend/plugins/q_dev/models/chat_log.go b/backend/plugins/q_dev/models/chat_log.go new file mode 100644 index 000000000..265722af5 --- /dev/null +++ b/backend/plugins/q_dev/models/chat_log.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// QDevChatLog stores parsed data from GenerateAssistantResponse logging events +type QDevChatLog struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"` + UserId string `gorm:"index;type:varchar(255)" json:"userId"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Timestamp time.Time `gorm:"index" json:"timestamp"` + ChatTriggerType string `gorm:"type:varchar(50)" json:"chatTriggerType"` + HasCustomization bool `json:"hasCustomization"` + ConversationId string `gorm:"type:varchar(255)" json:"conversationId"` + UtteranceId string `gorm:"type:varchar(255)" json:"utteranceId"` + ModelId string `gorm:"type:varchar(100)" json:"modelId"` + PromptLength int `json:"promptLength"` + ResponseLength int `json:"responseLength"` + OpenFileCount int `json:"openFileCount"` + ActiveFileName string `gorm:"type:varchar(512)" json:"activeFileName"` + ActiveFileExtension string `gorm:"type:varchar(50)" json:"activeFileExtension"` + HasSteering bool `json:"hasSteering"` + IsSpecMode bool `json:"isSpecMode"` +} + +func (QDevChatLog) TableName() string { + return "_tool_q_dev_chat_log" +} diff --git a/backend/plugins/q_dev/models/completion_log.go b/backend/plugins/q_dev/models/completion_log.go new file mode 100644 index 000000000..0d0e0404c --- /dev/null +++ b/backend/plugins/q_dev/models/completion_log.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// QDevCompletionLog stores parsed data from GenerateCompletions logging events +type QDevCompletionLog struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"` + UserId string `gorm:"index;type:varchar(255)" json:"userId"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Timestamp time.Time `gorm:"index" json:"timestamp"` + FileName string `gorm:"type:varchar(512)" json:"fileName"` + FileExtension string `gorm:"type:varchar(50)" json:"fileExtension"` + HasCustomization bool `json:"hasCustomization"` + CompletionsCount int `json:"completionsCount"` +} + +func (QDevCompletionLog) TableName() string { + return "_tool_q_dev_completion_log" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/register.go b/backend/plugins/q_dev/models/migrationscripts/20260314_add_logging_tables.go similarity index 54% copy from backend/plugins/q_dev/models/migrationscripts/register.go copy to backend/plugins/q_dev/models/migrationscripts/20260314_add_logging_tables.go index 9c68ae8f8..cbd5943ec 100644 --- a/backend/plugins/q_dev/models/migrationscripts/register.go +++ b/backend/plugins/q_dev/models/migrationscripts/20260314_add_logging_tables.go @@ -18,22 +18,26 @@ limitations under the License. package migrationscripts import ( - "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/q_dev/models/migrationscripts/archived" ) -// All return all migration scripts -func All() []plugin.MigrationScript { - return []plugin.MigrationScript{ - new(initTables), - new(modifyFileMetaTable), - new(addDisplayNameFields), - new(addMissingMetrics), - new(addS3SliceTable), - new(addScopeConfigIdToS3Slice), - new(addScopeIdFields), - new(addUserReportTable), - new(addAccountIdToS3Slice), - new(fixDedupUserTables), - new(resetS3FileMetaProcessed), - } +type addLoggingTables struct{} + +func (*addLoggingTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.QDevChatLog{}, + &archived.QDevCompletionLog{}, + ) +} + +func (*addLoggingTables) Version() uint64 { + return 20260314000001 +} + +func (*addLoggingTables) Name() string { + return "Add chat_log and completion_log tables for Kiro logging data" } diff --git a/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go b/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go new file mode 100644 index 000000000..ae766e301 --- /dev/null +++ b/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go @@ -0,0 +1,50 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type QDevChatLog struct { + archived.Model + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"` + UserId string `gorm:"index;type:varchar(255)" json:"userId"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Timestamp time.Time `gorm:"index" json:"timestamp"` + ChatTriggerType string `gorm:"type:varchar(50)" json:"chatTriggerType"` + HasCustomization bool `json:"hasCustomization"` + ConversationId string `gorm:"type:varchar(255)" json:"conversationId"` + UtteranceId string `gorm:"type:varchar(255)" json:"utteranceId"` + ModelId string `gorm:"type:varchar(100)" json:"modelId"` + PromptLength int `json:"promptLength"` + ResponseLength int `json:"responseLength"` + OpenFileCount int `json:"openFileCount"` + ActiveFileName string `gorm:"type:varchar(512)" json:"activeFileName"` + ActiveFileExtension string `gorm:"type:varchar(50)" json:"activeFileExtension"` + HasSteering bool `json:"hasSteering"` + IsSpecMode bool `json:"isSpecMode"` +} + +func (QDevChatLog) TableName() string { + return "_tool_q_dev_chat_log" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go b/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go new file mode 100644 index 000000000..1e915ff9b --- /dev/null +++ b/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type QDevCompletionLog struct { + archived.Model + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"` + UserId string `gorm:"index;type:varchar(255)" json:"userId"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Timestamp time.Time `gorm:"index" json:"timestamp"` + FileName string `gorm:"type:varchar(512)" json:"fileName"` + FileExtension string `gorm:"type:varchar(50)" json:"fileExtension"` + HasCustomization bool `json:"hasCustomization"` + CompletionsCount int `json:"completionsCount"` +} + +func (QDevCompletionLog) TableName() string { + return "_tool_q_dev_completion_log" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/register.go b/backend/plugins/q_dev/models/migrationscripts/register.go index 9c68ae8f8..5480d5eaf 100644 --- a/backend/plugins/q_dev/models/migrationscripts/register.go +++ b/backend/plugins/q_dev/models/migrationscripts/register.go @@ -35,5 +35,6 @@ func All() []plugin.MigrationScript { new(addAccountIdToS3Slice), new(fixDedupUserTables), new(resetS3FileMetaProcessed), + new(addLoggingTables), } } diff --git a/backend/plugins/q_dev/tasks/s3_file_collector.go b/backend/plugins/q_dev/tasks/s3_file_collector.go index 9d40919ae..1ab4f8f0a 100644 --- a/backend/plugins/q_dev/tasks/s3_file_collector.go +++ b/backend/plugins/q_dev/tasks/s3_file_collector.go @@ -59,9 +59,9 @@ func CollectQDevS3Files(taskCtx plugin.SubTaskContext) errors.Error { } for _, object := range result.Contents { - // Only process CSV files - if !strings.HasSuffix(*object.Key, ".csv") { - taskCtx.GetLogger().Debug("Skipping non-CSV file: %s", *object.Key) + // Only process CSV and JSON.gz files + if !strings.HasSuffix(*object.Key, ".csv") && !strings.HasSuffix(*object.Key, ".json.gz") { + taskCtx.GetLogger().Debug("Skipping unsupported file: %s", *object.Key) continue } diff --git a/backend/plugins/q_dev/tasks/s3_logging_extractor.go b/backend/plugins/q_dev/tasks/s3_logging_extractor.go new file mode 100644 index 000000000..f25cc9659 --- /dev/null +++ b/backend/plugins/q_dev/tasks/s3_logging_extractor.go @@ -0,0 +1,308 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "compress/gzip" + "encoding/json" + "fmt" + "io" + "path/filepath" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/q_dev/models" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" +) + +var _ plugin.SubTaskEntryPoint = ExtractQDevLoggingData + +// ExtractQDevLoggingData extracts logging data from S3 JSON.gz files +func ExtractQDevLoggingData(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*QDevTaskData) + db := taskCtx.GetDal() + + cursor, err := db.Cursor( + dal.From(&models.QDevS3FileMeta{}), + dal.Where("connection_id = ? AND processed = ? AND file_name LIKE ?", + data.Options.ConnectionId, false, "%.json.gz"), + ) + if err != nil { + return errors.Default.Wrap(err, "failed to get logging file metadata cursor") + } + defer cursor.Close() + + taskCtx.SetProgress(0, -1) + + for cursor.Next() { + fileMeta := &models.QDevS3FileMeta{} + err = db.Fetch(cursor, fileMeta) + if err != nil { + return errors.Default.Wrap(err, "failed to fetch file metadata") + } + + getInput := &s3.GetObjectInput{ + Bucket: aws.String(data.S3Client.Bucket), + Key: aws.String(fileMeta.S3Path), + } + + getResult, err := data.S3Client.S3.GetObject(getInput) + if err != nil { + return errors.Convert(err) + } + + tx := db.Begin() + processErr := processLoggingData(taskCtx, tx, getResult.Body, fileMeta) + if processErr != nil { + if rollbackErr := tx.Rollback(); rollbackErr != nil { + taskCtx.GetLogger().Error(rollbackErr, "failed to rollback transaction") + } + return errors.Default.Wrap(processErr, fmt.Sprintf("failed to process logging file %s", fileMeta.FileName)) + } + + fileMeta.Processed = true + now := time.Now() + fileMeta.ProcessedTime = &now + err = tx.Update(fileMeta) + if err != nil { + if rollbackErr := tx.Rollback(); rollbackErr != nil { + taskCtx.GetLogger().Error(rollbackErr, "failed to rollback transaction") + } + return errors.Default.Wrap(err, "failed to update file metadata") + } + + err = tx.Commit() + if err != nil { + return errors.Default.Wrap(err, "failed to commit transaction") + } + + taskCtx.IncProgress(1) + } + + return nil +} + +// JSON structures for logging data + +type loggingFile struct { + Records []json.RawMessage `json:"records"` +} + +type chatLogRecord struct { + Request *chatLogRequest `json:"generateAssistantResponseEventRequest"` + Response *chatLogResponse `json:"generateAssistantResponseEventResponse"` +} + +type chatLogRequest struct { + UserID string `json:"userId"` + Timestamp string `json:"timeStamp"` + ChatTriggerType string `json:"chatTriggerType"` + CustomizationArn *string `json:"customizationArn"` + ModelID string `json:"modelId"` + Prompt string `json:"prompt"` +} + +type chatLogResponse struct { + RequestID string `json:"requestId"` + AssistantResponse string `json:"assistantResponse"` + MessageMetadata struct { + ConversationID *string `json:"conversationId"` + UtteranceID *string `json:"utteranceId"` + } `json:"messageMetadata"` +} + +type completionLogRecord struct { + Request *completionLogRequest `json:"generateCompletionsEventRequest"` + Response *completionLogResponse `json:"generateCompletionsEventResponse"` +} + +type completionLogRequest struct { + UserID string `json:"userId"` + Timestamp string `json:"timeStamp"` + FileName string `json:"fileName"` + CustomizationArn *string `json:"customizationArn"` +} + +type completionLogResponse struct { + RequestID string `json:"requestId"` + Completions []json.RawMessage `json:"completions"` +} + +func processLoggingData(taskCtx plugin.SubTaskContext, db dal.Dal, reader io.ReadCloser, fileMeta *models.QDevS3FileMeta) errors.Error { + defer reader.Close() + + data := taskCtx.GetData().(*QDevTaskData) + + gzReader, err := gzip.NewReader(reader) + if err != nil { + return errors.Convert(err) + } + defer gzReader.Close() + + var logFile loggingFile + decoder := json.NewDecoder(gzReader) + if err := decoder.Decode(&logFile); err != nil { + return errors.Convert(err) + } + + isChatLog := strings.Contains(fileMeta.S3Path, "GenerateAssistantResponse") + + for _, rawRecord := range logFile.Records { + if isChatLog { + if err := processChatRecord(taskCtx, db, rawRecord, fileMeta, data.IdentityClient); err != nil { + return err + } + } else { + if err := processCompletionRecord(taskCtx, db, rawRecord, fileMeta, data.IdentityClient); err != nil { + return err + } + } + } + + return nil +} + +func processChatRecord(taskCtx plugin.SubTaskContext, db dal.Dal, raw json.RawMessage, fileMeta *models.QDevS3FileMeta, identityClient UserDisplayNameResolver) errors.Error { + var record chatLogRecord + if err := json.Unmarshal(raw, &record); err != nil { + return errors.Convert(err) + } + + if record.Request == nil || record.Response == nil { + return nil + } + + ts, err := time.Parse(time.RFC3339Nano, record.Request.Timestamp) + if err != nil { + ts = time.Now() + } + + chatLog := &models.QDevChatLog{ + ConnectionId: fileMeta.ConnectionId, + ScopeId: fileMeta.ScopeId, + RequestId: record.Response.RequestID, + UserId: record.Request.UserID, + DisplayName: resolveDisplayName(taskCtx.GetLogger(), record.Request.UserID, identityClient), + Timestamp: ts, + ChatTriggerType: record.Request.ChatTriggerType, + HasCustomization: record.Request.CustomizationArn != nil && *record.Request.CustomizationArn != "", + ModelId: record.Request.ModelID, + PromptLength: len(record.Request.Prompt), + ResponseLength: len(record.Response.AssistantResponse), + } + + // Parse structured info from prompt + prompt := record.Request.Prompt + chatLog.OpenFileCount = countOpenFiles(prompt) + chatLog.ActiveFileName, chatLog.ActiveFileExtension = parseActiveFile(prompt) + chatLog.HasSteering = strings.Contains(prompt, ".kiro/steering") + chatLog.IsSpecMode = strings.Contains(prompt, "implicit-rules") + + if record.Response.MessageMetadata.ConversationID != nil { + chatLog.ConversationId = *record.Response.MessageMetadata.ConversationID + } + if record.Response.MessageMetadata.UtteranceID != nil { + chatLog.UtteranceId = *record.Response.MessageMetadata.UtteranceID + } + + return errors.Default.Wrap(db.CreateOrUpdate(chatLog), "failed to save chat log") +} + +// countOpenFiles counts <file name="..."> tags within <OPEN-EDITOR-FILES> block +func countOpenFiles(prompt string) int { + start := strings.Index(prompt, "<OPEN-EDITOR-FILES>") + if start == -1 { + return 0 + } + end := strings.Index(prompt, "</OPEN-EDITOR-FILES>") + if end == -1 { + return 0 + } + block := prompt[start:end] + return strings.Count(block, "<file name=") +} + +// parseActiveFile extracts the active file name and extension from prompt +func parseActiveFile(prompt string) (string, string) { + start := strings.Index(prompt, "<ACTIVE-EDITOR-FILE>") + if start == -1 { + return "", "" + } + end := strings.Index(prompt[start:], "</ACTIVE-EDITOR-FILE>") + if end == -1 { + return "", "" + } + block := prompt[start : start+end] + // Find <file name="..." /> + nameStart := strings.Index(block, "name=\"") + if nameStart == -1 { + return "", "" + } + nameStart += len("name=\"") + nameEnd := strings.Index(block[nameStart:], "\"") + if nameEnd == -1 { + return "", "" + } + fileName := block[nameStart : nameStart+nameEnd] + ext := filepath.Ext(fileName) + return fileName, ext +} + +func processCompletionRecord(taskCtx plugin.SubTaskContext, db dal.Dal, raw json.RawMessage, fileMeta *models.QDevS3FileMeta, identityClient UserDisplayNameResolver) errors.Error { + var record completionLogRecord + if err := json.Unmarshal(raw, &record); err != nil { + return errors.Convert(err) + } + + if record.Request == nil || record.Response == nil { + return nil + } + + ts, err := time.Parse(time.RFC3339Nano, record.Request.Timestamp) + if err != nil { + ts = time.Now() + } + + completionLog := &models.QDevCompletionLog{ + ConnectionId: fileMeta.ConnectionId, + ScopeId: fileMeta.ScopeId, + RequestId: record.Response.RequestID, + UserId: record.Request.UserID, + DisplayName: resolveDisplayName(taskCtx.GetLogger(), record.Request.UserID, identityClient), + Timestamp: ts, + FileName: record.Request.FileName, + FileExtension: filepath.Ext(record.Request.FileName), + HasCustomization: record.Request.CustomizationArn != nil && *record.Request.CustomizationArn != "", + CompletionsCount: len(record.Response.Completions), + } + + return errors.Default.Wrap(db.CreateOrUpdate(completionLog), "failed to save completion log") +} + +var ExtractQDevLoggingDataMeta = plugin.SubTaskMeta{ + Name: "extractQDevLoggingData", + EntryPoint: ExtractQDevLoggingData, + EnabledByDefault: true, + Description: "Extract logging data from S3 JSON.gz files (chat and completion events)", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&CollectQDevS3FilesMeta}, +} diff --git a/grafana/dashboards/qdev_logging.json b/grafana/dashboards/qdev_logging.json new file mode 100644 index 000000000..b6d0627a5 --- /dev/null +++ b/grafana/dashboards/qdev_logging.json @@ -0,0 +1,800 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "mysql", + "description": "Overview of logging event metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n (SELECT COUNT(*) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Chat Events',\n (SELECT COUNT(DISTINCT user_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Chat Users',\n (SELECT COUNT(DISTINCT conversation_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) AND conversation_id != '') as 'Conversations',\n (SELECT COUNT(*) FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) as 'Com [...] + "refId": "A" + } + ], + "title": "Logging Overview", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Hourly distribution of AI usage activity (chat + completions)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Hour of Day", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.8, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n hour_of_day as 'Hour',\n SUM(chat_count) as 'Chat Events',\n SUM(completion_count) as 'Completion Events'\nFROM (\n SELECT HOUR(timestamp) as hour_of_day, COUNT(*) as chat_count, 0 as completion_count\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n UNION ALL\n SELECT HOUR(timestamp) as hour_of_day, 0 as chat_count, COUNT(*) as completion_count\n FROM lake._tool_q_dev_completion_log\n WHERE $__timeFilt [...] + "refId": "A" + } + ], + "title": "Active Hours Distribution", + "type": "barchart" + }, + { + "datasource": "mysql", + "description": "Distribution of model usage across chat events", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 3, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n CASE\n WHEN model_id = '' OR model_id IS NULL THEN '(unknown)'\n ELSE model_id\n END as 'Model',\n COUNT(*) as 'Requests'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY model_id\nORDER BY COUNT(*) DESC", + "refId": "A" + } + ], + "title": "Model Usage Distribution", + "type": "piechart" + }, + { + "datasource": "mysql", + "description": "Top file extensions used with inline completions", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 4, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n CASE\n WHEN file_extension = '' THEN '(unknown)'\n ELSE file_extension\n END as 'File Type',\n COUNT(*) as 'Completions'\nFROM lake._tool_q_dev_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY file_extension\nORDER BY COUNT(*) DESC\nLIMIT 15", + "refId": "A" + } + ], + "title": "File Type Usage (Completions)", + "type": "piechart" + }, + { + "datasource": "mysql", + "description": "Average number of chat events per conversation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE(timestamp) as time,\n COUNT(*) / NULLIF(COUNT(DISTINCT CASE WHEN conversation_id != '' THEN conversation_id END), 0) as 'Avg Turns per Conversation',\n COUNT(DISTINCT CASE WHEN conversation_id != '' THEN conversation_id END) as 'Unique Conversations',\n COUNT(*) as 'Total Chat Events'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", + "refId": "A" + } + ], + "title": "Conversation Depth Analysis", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Daily chat and completion events over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT time, SUM(chat) as 'Chat Events', SUM(completions) as 'Completion Events'\nFROM (\n SELECT DATE(timestamp) as time, COUNT(*) as chat, 0 as completions\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY DATE(timestamp)\n UNION ALL\n SELECT DATE(timestamp) as time, 0 as chat, COUNT(*) as completions\n FROM lake._tool_q_dev_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY DATE(timestamp)\n) combined\nGROUP BY time\nORD [...] + "refId": "A" + } + ], + "title": "Daily Event Trends", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Per-user logging activity summary", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n COALESCE(u.display_name, u.user_id) as 'User',\n u.user_id as 'User ID',\n u.chat_events as 'Chat Events',\n u.conversations as 'Conversations',\n ROUND(u.chat_events / NULLIF(u.conversations, 0), 1) as 'Avg Turns',\n COALESCE(c.completion_events, 0) as 'Completion Events',\n COALESCE(c.files_count, 0) as 'Distinct Files',\n ROUND(u.avg_prompt_len) as 'Avg Prompt Len',\n ROUND(u.avg_response_len) as 'Avg Response Len',\n u.steering_count as 'Steeri [...] + "refId": "A" + } + ], + "title": "Per-User Activity", + "type": "table" + }, + { + "datasource": "mysql", + "description": "Distribution of Kiro feature adoption: Steering, Spec Mode, and Plain Chat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 8, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) as 'Using Steering',\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) as 'Using Spec Mode',\n SUM(CASE WHEN has_steering = 0 AND is_spec_mode = 0 THEN 1 ELSE 0 END) as 'Plain Chat'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Kiro Feature Adoption", + "type": "piechart" + }, + { + "datasource": "mysql", + "description": "Top file extensions active during chat events", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 9, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n CASE\n WHEN active_file_extension = '' OR active_file_extension IS NULL THEN '(no file active)'\n ELSE active_file_extension\n END as 'File Type',\n COUNT(*) as 'Chat Events'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY active_file_extension\nORDER BY COUNT(*) DESC\nLIMIT 15", + "refId": "A" + } + ], + "title": "Active File Types in Chat", + "type": "piechart" + }, + { + "datasource": "mysql", + "description": "Average and maximum prompt/response lengths over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE(timestamp) as time,\n AVG(prompt_length) as 'Avg Prompt Length',\n AVG(response_length) as 'Avg Response Length',\n MAX(prompt_length) as 'Max Prompt Length',\n MAX(response_length) as 'Max Response Length'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", + "refId": "A" + } + ], + "title": "Prompt & Response Length Trends", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "logging", + "kiro" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro AI Activity Insights", + "uid": "qdev_logging", + "version": 1 +} diff --git a/grafana/dashboards/qdev_user_data.json b/grafana/dashboards/qdev_user_data.json index d80d57bab..e66901a5d 100644 --- a/grafana/dashboards/qdev_user_data.json +++ b/grafana/dashboards/qdev_user_data.json @@ -730,7 +730,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(display_name, user_id) as 'User',\n SUM(chat_ai_code_lines) as 'Accepted Lines (Chat)',\n SUM(transformation_lines_ingested) as 'Lines Ingested (Java Transform)',\n SUM(transformation_lines_generated) as 'Lines Generated (Java Transform)',\n SUM(transformation_event_count) as 'Event Count (Java Transform)',\n SUM(code_review_findings_count) as 'Findings (Code Review)',\n SUM(code_fix_accepted_lines) as 'Accepted Lines (Code Fix)',\n SUM(code [...] + "rawSql": "SELECT\n COALESCE(display_name, user_id) as 'User',\n SUM(chat_ai_code_lines) as 'Accepted Lines (Chat)',\n SUM(transformation_lines_ingested) as 'Lines Ingested (Java Transform)',\n SUM(transformation_lines_generated) as 'Lines Generated (Java Transform)',\n SUM(transformation_event_count) as 'Event Count (Java Transform)',\n SUM(code_review_findings_count) as 'Findings (Code Review)',\n SUM(code_fix_accepted_lines) as 'Accepted Lines (Code Fix)',\n SUM(code [...] "refId": "A", "select": [ [ @@ -771,6 +771,393 @@ ], "title": "User Interactions", "type": "table" + }, + { + "datasource": "mysql", + "description": "Daily doc generation events and accepted/rejected lines", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(doc_generation_event_count) as 'Doc Generation Events',\n SUM(doc_generation_accepted_line_additions) as 'Accepted Line Additions',\n SUM(doc_generation_accepted_line_updates) as 'Accepted Line Updates',\n SUM(doc_generation_rejected_line_additions) as 'Rejected Line Additions',\n SUM(doc_generation_rejected_line_updates) as 'Rejected Line Updates'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Doc Generation Metrics", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Daily test generation events and lines", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(test_generation_event_count) as 'Test Generation Events',\n SUM(test_generation_accepted_tests) as 'Accepted Tests',\n SUM(test_generation_generated_tests) as 'Generated Tests',\n SUM(test_generation_accepted_lines) as 'Accepted Lines',\n SUM(test_generation_generated_lines) as 'Generated Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Test Generation Metrics", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Daily agentic dev events and lines", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.2", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(dev_generation_event_count) as 'Dev Generation Events',\n SUM(dev_acceptance_event_count) as 'Dev Acceptance Events',\n SUM(dev_generated_lines) as 'Dev Generated Lines',\n SUM(dev_accepted_lines) as 'Dev Accepted Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Dev (Agentic) Metrics", + "type": "timeseries" } ], "preload": false, diff --git a/grafana/dashboards/qdev_user_report.json b/grafana/dashboards/qdev_user_report.json index e1a27bc53..68f4b8650 100644 --- a/grafana/dashboards/qdev_user_report.json +++ b/grafana/dashboards/qdev_user_report.json @@ -433,7 +433,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(display_name, user_id) as 'User',\n subscription_tier as 'Tier',\n client_type as 'Client',\n SUM(credits_used) as 'Credits Used',\n SUM(total_messages) as 'Messages',\n SUM(chat_conversations) as 'Conversations',\n SUM(overage_credits_used) as 'Overage Credits',\n CASE WHEN MAX(CAST(overage_enabled AS UNSIGNED)) = 1 THEN 'Yes' ELSE 'No' END as 'Overage',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_de [...] + "rawSql": "SELECT\n COALESCE(display_name, user_id) as 'User',\n subscription_tier as 'Tier',\n client_type as 'Client',\n SUM(credits_used) as 'Credits Used',\n SUM(total_messages) as 'Messages',\n SUM(chat_conversations) as 'Conversations',\n SUM(overage_credits_used) as 'Overage Credits',\n CASE WHEN MAX(CAST(overage_enabled AS UNSIGNED)) = 1 THEN 'Yes' ELSE 'No' END as 'Overage',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_de [...] "refId": "A" } ],
