This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 98c1c4593 [feat] q-dev-plugin-collect-s3 (#8454)
98c1c4593 is described below
commit 98c1c4593f453537b644f9a634552b84e3b555a1
Author: Warren Chen <[email protected]>
AuthorDate: Wed May 28 21:57:20 2025 +0800
[feat] q-dev-plugin-collect-s3 (#8454)
* feat: new plugin = q dev
* fix: tables to archived
* fix: archived
* fix: translate docs
* fix: modify test
---
backend/go.mod | 1 +
backend/go.sum | 1 +
backend/plugins/q_dev/Q_DEV_deploy_guide.md | 85 ++++++++++++
backend/plugins/q_dev/README.md | 97 +++++++++++++
backend/plugins/q_dev/api/connection.go | 96 +++++++++++++
backend/plugins/q_dev/api/init.go | 39 ++++++
backend/plugins/q_dev/api/test_connection.go | 67 +++++++++
backend/plugins/q_dev/img.png | Bin 0 -> 20568 bytes
backend/plugins/q_dev/impl/impl.go | 150 +++++++++++++++++++++
backend/plugins/q_dev/models/connection.go | 69 ++++++++++
.../q_dev/models/migrationscripts/20250319_init.go | 45 +++++++
.../migrationscripts/20250320_modify_file_meta.go | 46 +++++++
.../models/migrationscripts/archived/connection.go | 46 +++++++
.../migrationscripts/archived/s3_file_meta.go | 38 ++++++
.../models/migrationscripts/archived/user_data.go | 51 +++++++
.../migrationscripts/archived/user_metrics.go | 66 +++++++++
.../q_dev/models/migrationscripts/register.go | 29 ++++
backend/plugins/q_dev/models/s3_file_meta.go | 38 ++++++
backend/plugins/q_dev/models/user_data.go | 51 +++++++
backend/plugins/q_dev/models/user_metrics.go | 66 +++++++++
backend/plugins/q_dev/q_dev.go | 43 ++++++
backend/plugins/q_dev/tasks/s3_client.go | 47 +++++++
backend/plugins/q_dev/tasks/s3_file_collector.go | 103 ++++++++++++++
backend/plugins/q_dev/tasks/task_data.go | 45 +++++++
backend/plugins/table_info_test.go | 2 +
25 files changed, 1321 insertions(+)
diff --git a/backend/go.mod b/backend/go.mod
index 819fcfb6d..871cba963 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -3,6 +3,7 @@ module github.com/apache/incubator-devlake
go 1.20
require (
+ github.com/aws/aws-sdk-go v1.55.6
github.com/cockroachdb/errors v1.11.1
github.com/gin-contrib/cors v1.6.0
github.com/gin-gonic/gin v1.9.1
diff --git a/backend/go.sum b/backend/go.sum
index 70c2daa08..a6e2f17ba 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -54,6 +54,7 @@ github.com/armon/circbuf
v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod
h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod
h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/aws/aws-sdk-go v1.55.6/go.mod
h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/bgentry/speakeasy v0.1.0/go.mod
h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.4/go.mod
h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
github.com/bwesterb/go-ristretto v1.2.3/go.mod
h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
diff --git a/backend/plugins/q_dev/Q_DEV_deploy_guide.md
b/backend/plugins/q_dev/Q_DEV_deploy_guide.md
new file mode 100644
index 000000000..218c55c6e
--- /dev/null
+++ b/backend/plugins/q_dev/Q_DEV_deploy_guide.md
@@ -0,0 +1,85 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+
+# DevLake Development Environment Deployment Guide
+
+## Environment Requirements
+- Docker v19.03.10+
+- Golang v1.19+
+- GNU Make
+ - Mac (pre-installed)
+ - Windows: [Download](http://gnuwin32.sourceforge.net/packages/make.htm)
+ - Ubuntu: `sudo apt-get install build-essential libssl-dev`
+
+## How to Set Up the Development Environment
+The following guide will explain how to run DevLake's frontend (config-ui) and
backend in development mode.
+
+### Clone the Repository
+Navigate to where you want to install this project and clone the repository:
+
+```bash
+git clone https://github.com/apache/incubator-devlake.git
+cd incubator-devlake
+```
+
+### Install Plugin Dependencies
+
+RefDiff plugin:
+Install Go packages
+```bash
+cd backend
+go get
+cd ..
+```
+
+### Configure Environment File
+Copy the example configuration file to a new local file:
+
+```bash
+cp env.example .env
+```
+
+Update the following variables in the `.env` file:
+
+- `DB_URL`: Replace `mysql:3306` with `127.0.0.1:3306`
+- `DISABLED_REMOTE_PLUGINS`: Set to `True`
+
+### Start MySQL and Grafana Containers
+
+Make sure the Docker daemon is running before this step.
+
+> Grafana needs to rebuild the image, then change the image in
docker-compose.datasources.yml to `image: grafana:latest`
+
+```bash
+docker-compose -f docker-compose-dev.yml up -d mysql grafana
+```
+
+### Run in Development Mode
+Run devlake and config-ui in development mode in two separate terminals:
+
+```bash
+# Install poetry, follow the guide:
https://python-poetry.org/docs/#installation
+# Run devlake, only using the q dev plugin here
+DEVLAKE_PLUGINS=q_dev nohup make dev &
+# Run config-ui
+make configure-dev
+```
+
+For common errors, please refer to the troubleshooting documentation.
+
+Config UI runs on localhost:4000
\ No newline at end of file
diff --git a/backend/plugins/q_dev/README.md b/backend/plugins/q_dev/README.md
new file mode 100644
index 000000000..331359083
--- /dev/null
+++ b/backend/plugins/q_dev/README.md
@@ -0,0 +1,97 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Q Developer Plugin
+
+This plugin is used to retrieve AWS Q Developer usage data from AWS S3, and
process and analyze it.
+
+## Features
+
+- Retrieve CSV files from a specified prefix in AWS S3
+- Parse user usage data from CSV files
+- Aggregate data by user and calculate various metrics
+
+## Configuration
+
+Configuration items include:
+
+1. AWS Access Key ID
+2. AWS Secret Key
+3. AWS Region
+4. S3 Bucket Name
+5. Rate Limit (per hour)
+
+You can create a connection using the following curl command:
+```bash
+curl 'http://localhost:8080/plugins/q_dev/connections' \
+--header 'Content-Type: application/json' \
+--data-raw '{
+ "name": "q_dev_connection",
+ "accessKeyId": "<YOUR_ACCESS_KEY_ID>",
+ "secretAccessKey": "<YOUR_SECRET_ACCESS_KEY>",
+ "region": "<AWS_REGION>",
+ "bucket": "<YOUR_S3_BUCKET_NAME>",
+ "rateLimitPerHour": 20000
+}'
+```
+Please replace the following placeholders with actual values:
+<YOUR_ACCESS_KEY_ID>: Your AWS access key ID
+<YOUR_SECRET_ACCESS_KEY>: Your AWS secret access key
+<YOUR_S3_BUCKET_NAME>: The S3 bucket name you want to use
+<AWS_REGION>: The region where your S3 bucket is located
+
+You can get all connections using the following curl command:
+```bash
+curl Get 'http://localhost:8080/plugins/q_dev/connections'
+```
+
+## Data Flow
+
+The plugin includes the following tasks:
+
+1. `collectQDevS3Files`: Collects file metadata information from S3, without
downloading file content
+2. `extractQDevS3Data`: Uses S3 file metadata to download CSV data and parse
it into the database
+3. `convertQDevUserMetrics`: Converts user data into aggregated metrics,
calculating averages and totals
+
+## Data Tables
+
+- `_tool_q_dev_connections`: Stores AWS S3 connection information
+- `_tool_q_dev_s3_file_meta`: Stores S3 file metadata
+- `_tool_q_dev_user_data`: Stores user data parsed from CSV files
+- `_tool_q_dev_user_metrics`: Stores aggregated user metrics
+
+## Data Collection Configuration
+Steps to collect data:
+1. On the Config UI page, select `Advanced Mode` on the left, click
`Blueprints`
+2. Create a new Blueprint
+3.  Click the gear icon on the right
+4. Paste the following JSON configuration into `JSON Configuration`:
+
+```json
+[
+ [
+ {
+ "plugin": "q_dev",
+ "subtasks": null,
+ "options": {
+ "connectionId": 5,
+ "s3Prefix": ""
+ }
+ }
+ ]
+]
+```
\ No newline at end of file
diff --git a/backend/plugins/q_dev/api/connection.go
b/backend/plugins/q_dev/api/connection.go
new file mode 100644
index 000000000..24cf0ced4
--- /dev/null
+++ b/backend/plugins/q_dev/api/connection.go
@@ -0,0 +1,96 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "net/http"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/q_dev/models"
+)
+
+// 连接项目的CRUD API
+
+// PostConnections 创建新连接
+func PostConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ // 创建连接
+ connection := &models.QDevConnection{}
+ err := api.Decode(input.Body, connection, vld)
+ if err != nil {
+ return nil, err
+ }
+ // 验证
+ // 保存到数据库
+ err = connectionHelper.Create(connection, input)
+ if err != nil {
+ return nil, err
+ }
+ return &plugin.ApiResourceOutput{Body: connection.Sanitize(), Status:
http.StatusOK}, nil
+}
+
+// PatchConnection 更新现有连接
+func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ connection := &models.QDevConnection{}
+ if err := connectionHelper.First(&connection, input.Params); err != nil
{
+ return nil, err
+ }
+ if err := (&models.QDevConnection{}).MergeFromRequest(connection,
input.Body); err != nil {
+ return nil, errors.Convert(err)
+ }
+ if err := connectionHelper.SaveWithCreateOrUpdate(connection); err !=
nil {
+ return nil, err
+ }
+ return &plugin.ApiResourceOutput{Body: connection.Sanitize(), Status:
http.StatusOK}, nil
+}
+
+// DeleteConnection 删除连接
+func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ conn := &models.QDevConnection{}
+ output, err := connectionHelper.Delete(conn, input)
+ if err != nil {
+ return output, err
+ }
+ output.Body = conn.Sanitize()
+ return output, nil
+}
+
+// ListConnections 列出所有连接
+func ListConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ var connections []models.QDevConnection
+ err := connectionHelper.List(&connections)
+ if err != nil {
+ return nil, err
+ }
+ // 敏感信息脱敏
+ for i := 0; i < len(connections); i++ {
+ connections[i] = connections[i].Sanitize()
+ }
+ return &plugin.ApiResourceOutput{Body: connections}, nil
+}
+
+// GetConnection 获取单个连接详情
+func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ connection := &models.QDevConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+ return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, err
+}
diff --git a/backend/plugins/q_dev/api/init.go
b/backend/plugins/q_dev/api/init.go
new file mode 100644
index 000000000..7a16d5f0c
--- /dev/null
+++ b/backend/plugins/q_dev/api/init.go
@@ -0,0 +1,39 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/go-playground/validator/v10"
+)
+
+var vld *validator.Validate
+var connectionHelper *api.ConnectionApiHelper
+var basicRes context.BasicRes
+
+func Init(br context.BasicRes, p plugin.PluginMeta) {
+ basicRes = br
+ vld = validator.New()
+ connectionHelper = api.NewConnectionHelper(
+ basicRes,
+ vld,
+ p.Name(),
+ )
+}
diff --git a/backend/plugins/q_dev/api/test_connection.go
b/backend/plugins/q_dev/api/test_connection.go
new file mode 100644
index 000000000..b5cf35e29
--- /dev/null
+++ b/backend/plugins/q_dev/api/test_connection.go
@@ -0,0 +1,67 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/q_dev/models"
+ "github.com/apache/incubator-devlake/plugins/q_dev/tasks"
+
+ "net/http"
+)
+
+// TestConnection 测试连接
+func TestConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ // 解析连接参数
+ var connection models.QDevConnection
+ err := api.Decode(input.Body, &connection, vld)
+ if err != nil {
+ return nil, err
+ }
+
+ // 测试S3连接
+ _, err = tasks.NewQDevS3Client(nil, &connection)
+ if err != nil {
+ return nil, err
+ }
+
+ // 连接成功
+ return &plugin.ApiResourceOutput{Status: http.StatusOK}, nil
+}
+
+// TestExistingConnection 测试现有连接
+func TestExistingConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ connection := &models.QDevConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "find connection from db")
+ }
+ if err := api.DecodeMapStruct(input.Body, connection, false); err !=
nil {
+ return nil, err
+ }
+ // 测试连接
+ _, err = tasks.NewQDevS3Client(nil, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ // 连接成功
+ return &plugin.ApiResourceOutput{Status: http.StatusOK}, nil
+}
diff --git a/backend/plugins/q_dev/img.png b/backend/plugins/q_dev/img.png
new file mode 100644
index 000000000..16af8d1ad
Binary files /dev/null and b/backend/plugins/q_dev/img.png differ
diff --git a/backend/plugins/q_dev/impl/impl.go
b/backend/plugins/q_dev/impl/impl.go
new file mode 100644
index 000000000..c8ddea5c1
--- /dev/null
+++ b/backend/plugins/q_dev/impl/impl.go
@@ -0,0 +1,150 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package impl
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/q_dev/api"
+ "github.com/apache/incubator-devlake/plugins/q_dev/models"
+
"github.com/apache/incubator-devlake/plugins/q_dev/models/migrationscripts"
+ "github.com/apache/incubator-devlake/plugins/q_dev/tasks"
+)
+
+var _ interface {
+ plugin.PluginMeta
+ plugin.PluginInit
+ plugin.PluginTask
+ plugin.PluginApi
+ plugin.PluginModel
+ plugin.PluginSource
+ plugin.PluginMigration
+ plugin.CloseablePluginTask
+} = (*QDev)(nil)
+
+type QDev struct{}
+
+func (p QDev) Init(basicRes context.BasicRes) errors.Error {
+ api.Init(basicRes, p)
+ return nil
+}
+
+func (p QDev) GetTablesInfo() []dal.Tabler {
+ return []dal.Tabler{
+ &models.QDevConnection{},
+ &models.QDevUserData{},
+ &models.QDevUserMetrics{},
+ &models.QDevS3FileMeta{},
+ }
+}
+
+func (p QDev) Description() string {
+ return "To collect and enrich data from AWS Q Developer usage metrics"
+}
+
+func (p QDev) Name() string {
+ return "q_dev"
+}
+
+func (p QDev) Connection() dal.Tabler {
+ return &models.QDevConnection{}
+}
+
+func (p QDev) Scope() plugin.ToolLayerScope {
+ return nil
+}
+
+func (p QDev) ScopeConfig() dal.Tabler {
+ return nil
+}
+
+func (p QDev) SubTaskMetas() []plugin.SubTaskMeta {
+ return []plugin.SubTaskMeta{
+ tasks.CollectQDevS3FilesMeta,
+ }
+}
+
+func (p QDev) PrepareTaskData(taskCtx plugin.TaskContext, options
map[string]interface{}) (interface{}, errors.Error) {
+ var op tasks.QDevOptions
+ if err := helper.Decode(options, &op, nil); err != nil {
+ return nil, err
+ }
+
+ connectionHelper := helper.NewConnectionHelper(
+ taskCtx,
+ nil,
+ p.Name(),
+ )
+ connection := &models.QDevConnection{}
+ err := connectionHelper.FirstById(connection, op.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+
+ // 创建S3客户端，替代API客户端
+ s3Client, err := tasks.NewQDevS3Client(taskCtx, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ return &tasks.QDevTaskData{
+ Options: &op,
+ S3Client: s3Client,
+ }, nil
+}
+
+func (p QDev) RootPkgPath() string {
+ return "github.com/apache/incubator-devlake/plugins/q_dev"
+}
+
+func (p QDev) MigrationScripts() []plugin.MigrationScript {
+ return migrationscripts.All()
+}
+
+func (p QDev) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
+ return map[string]map[string]plugin.ApiResourceHandler{
+ "test": {
+ "POST": api.TestConnection,
+ },
+ "connections": {
+ "POST": api.PostConnections,
+ "GET": api.ListConnections,
+ },
+ "connections/:connectionId": {
+ "PATCH": api.PatchConnection,
+ "DELETE": api.DeleteConnection,
+ "GET": api.GetConnection,
+ },
+ "connections/:connectionId/test": {
+ "POST": api.TestExistingConnection,
+ },
+ }
+}
+
+func (p QDev) Close(taskCtx plugin.TaskContext) errors.Error {
+ data, ok := taskCtx.GetData().(*tasks.QDevTaskData)
+ if !ok {
+ return errors.Default.New(fmt.Sprintf("GetData failed when try
to close %+v", taskCtx))
+ }
+ data.S3Client.Close()
+ return nil
+}
diff --git a/backend/plugins/q_dev/models/connection.go
b/backend/plugins/q_dev/models/connection.go
new file mode 100644
index 000000000..5f56749a5
--- /dev/null
+++ b/backend/plugins/q_dev/models/connection.go
@@ -0,0 +1,69 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/core/utils"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+// QDevConn holds the essential information to connect to AWS S3
+type QDevConn struct {
+ // AccessKeyId for AWS
+ AccessKeyId string `mapstructure:"accessKeyId" json:"accessKeyId"`
+ // SecretAccessKey for AWS
+ SecretAccessKey string `mapstructure:"secretAccessKey"
json:"secretAccessKey"`
+ // Region for AWS
+ Region string `mapstructure:"region" json:"region"`
+ // Bucket for AWS S3
+ Bucket string `mapstructure:"bucket" json:"bucket"`
+ // RateLimitPerHour limits the API requests sent to AWS
+ RateLimitPerHour int `mapstructure:"rateLimitPerHour"
json:"rateLimitPerHour"`
+}
+
+func (conn *QDevConn) Sanitize() QDevConn {
+ conn.SecretAccessKey = utils.SanitizeString(conn.SecretAccessKey)
+ return *conn
+}
+
+// QDevConnection holds QDevConn plus ID/Name for database storage
+type QDevConnection struct {
+ helper.BaseConnection `mapstructure:",squash"`
+ QDevConn `mapstructure:",squash"`
+}
+
+func (QDevConnection) TableName() string {
+ return "_tool_q_dev_connections"
+}
+
+func (connection QDevConnection) Sanitize() QDevConnection {
+ connection.QDevConn = connection.QDevConn.Sanitize()
+ return connection
+}
+
+func (connection *QDevConnection) MergeFromRequest(target *QDevConnection,
body map[string]interface{}) error {
+ secretKey := target.SecretAccessKey
+ if err := helper.DecodeMapStruct(body, target, true); err != nil {
+ return err
+ }
+ modifiedSecretKey := target.SecretAccessKey
+ if modifiedSecretKey == "" || modifiedSecretKey ==
utils.SanitizeString(secretKey) {
+ target.SecretAccessKey = secretKey
+ }
+ return nil
+}
diff --git a/backend/plugins/q_dev/models/migrationscripts/20250319_init.go
b/backend/plugins/q_dev/models/migrationscripts/20250319_init.go
new file mode 100644
index 000000000..f73c50a90
--- /dev/null
+++ b/backend/plugins/q_dev/models/migrationscripts/20250319_init.go
@@ -0,0 +1,45 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+
"github.com/apache/incubator-devlake/plugins/q_dev/models/migrationscripts/archived"
+)
+
+type initTables struct{}
+
+func (*initTables) Name() string {
+ return "Init schema for Q Developer plugin"
+}
+
+func (*initTables) Up(basicRes context.BasicRes) errors.Error {
+ return migrationhelper.AutoMigrateTables(
+ basicRes,
+ &archived.QDevConnection{},
+ &archived.QDevUserData{},
+ &archived.QDevUserMetrics{},
+ &archived.QDevS3FileMeta{},
+ )
+}
+
+func (*initTables) Version() uint64 {
+ return 20250319
+}
diff --git
a/backend/plugins/q_dev/models/migrationscripts/20250320_modify_file_meta.go
b/backend/plugins/q_dev/models/migrationscripts/20250320_modify_file_meta.go
new file mode 100644
index 000000000..a847d041e
--- /dev/null
+++ b/backend/plugins/q_dev/models/migrationscripts/20250320_modify_file_meta.go
@@ -0,0 +1,46 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+)
+
+type modifyFileMetaTable struct{}
+
+func (*modifyFileMetaTable) Name() string {
+ return "Modify QDevS3FileMeta table to allow NULL processed_time"
+}
+
+func (*modifyFileMetaTable) Up(basicRes context.BasicRes) errors.Error {
+ db := basicRes.GetDal()
+
+ // 修改 processed_time 列允许为 NULL
+ sql := "ALTER TABLE _tool_q_dev_s3_file_meta MODIFY processed_time
DATETIME NULL"
+ err := db.Exec(sql)
+ if err != nil {
+ return errors.Default.Wrap(err, "failed to modify
processed_time column")
+ }
+
+ return nil
+}
+
+func (*modifyFileMetaTable) Version() uint64 {
+ return 20250320
+}
diff --git
a/backend/plugins/q_dev/models/migrationscripts/archived/connection.go
b/backend/plugins/q_dev/models/migrationscripts/archived/connection.go
new file mode 100644
index 000000000..5d03159cc
--- /dev/null
+++ b/backend/plugins/q_dev/models/migrationscripts/archived/connection.go
@@ -0,0 +1,46 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ commonArchived
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+)
+
+// QDevConn holds the essential information to connect to AWS S3
+type QDevConn struct {
+ // AccessKeyId for AWS
+ AccessKeyId string `mapstructure:"accessKeyId" json:"accessKeyId"`
+ // SecretAccessKey for AWS
+ SecretAccessKey string `mapstructure:"secretAccessKey"
json:"secretAccessKey"`
+ // Region for AWS
+ Region string `mapstructure:"region" json:"region"`
+ // Bucket for AWS S3
+ Bucket string `mapstructure:"bucket" json:"bucket"`
+ // RateLimitPerHour limits the API requests sent to AWS
+ RateLimitPerHour int `mapstructure:"rateLimitPerHour"
json:"rateLimitPerHour"`
+}
+
+// QDevConnection holds QDevConn plus ID/Name for database storage
+type QDevConnection struct {
+ commonArchived.BaseConnection `mapstructure:",squash"`
+ QDevConn `mapstructure:",squash"`
+}
+
+func (QDevConnection) TableName() string {
+ return "_tool_q_dev_connections"
+}
diff --git
a/backend/plugins/q_dev/models/migrationscripts/archived/s3_file_meta.go
b/backend/plugins/q_dev/models/migrationscripts/archived/s3_file_meta.go
new file mode 100644
index 000000000..a648563a5
--- /dev/null
+++ b/backend/plugins/q_dev/models/migrationscripts/archived/s3_file_meta.go
@@ -0,0 +1,38 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "time"
+
+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+)
+
+// QDevS3FileMeta 存储S3文件的元数据信息
+type QDevS3FileMeta struct {
+ archived.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"`
+ FileName string `gorm:"primaryKey;type:varchar(255)"`
+ S3Path string `gorm:"type:varchar(512)" json:"s3Path"`
+ Processed bool `gorm:"default:false"`
+ ProcessedTime *time.Time `gorm:"default:null"`
+}
+
+func (QDevS3FileMeta) TableName() string {
+ return "_tool_q_dev_s3_file_meta"
+}
diff --git
a/backend/plugins/q_dev/models/migrationscripts/archived/user_data.go
b/backend/plugins/q_dev/models/migrationscripts/archived/user_data.go
new file mode 100644
index 000000000..00e6a0a44
--- /dev/null
+++ b/backend/plugins/q_dev/models/migrationscripts/archived/user_data.go
@@ -0,0 +1,51 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+ "time"
+
+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+)
+
+// QDevUserData 存储从CSV中提取的原始数据
+type QDevUserData struct {
+ archived.Model
+ ConnectionId uint64 `gorm:"primaryKey"`
+ UserId string `gorm:"index" json:"userId"`
+ Date time.Time `gorm:"index" json:"date"`
+ CodeReview_FindingsCount int
+ CodeReview_SucceededEventCount int
+ InlineChat_AcceptanceEventCount int
+ InlineChat_AcceptedLineAdditions int
+ InlineChat_AcceptedLineDeletions int
+ InlineChat_DismissalEventCount int
+ InlineChat_DismissedLineAdditions int
+ InlineChat_DismissedLineDeletions int
+ InlineChat_RejectedLineAdditions int
+ InlineChat_RejectedLineDeletions int
+ InlineChat_RejectionEventCount int
+ InlineChat_TotalEventCount int
+ Inline_AICodeLines int
+ Inline_AcceptanceCount int
+ Inline_SuggestionsCount int
+}
+
+func (QDevUserData) TableName() string {
+ return "_tool_q_dev_user_data"
+}
diff --git
a/backend/plugins/q_dev/models/migrationscripts/archived/user_metrics.go
b/backend/plugins/q_dev/models/migrationscripts/archived/user_metrics.go
new file mode 100644
index 000000000..d26b75aea
--- /dev/null
+++ b/backend/plugins/q_dev/models/migrationscripts/archived/user_metrics.go
@@ -0,0 +1,66 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+ "time"
+)
+
+// QDevUserMetrics 存储按用户聚合的指标数据
+type QDevUserMetrics struct {
+ archived.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"`
+ UserId string `gorm:"primaryKey"`
+ FirstDate time.Time
+ LastDate time.Time
+ TotalDays int
+
+ // 聚合指标
+ TotalCodeReview_FindingsCount int
+ TotalCodeReview_SucceededEventCount int
+ TotalInlineChat_AcceptanceEventCount int
+ TotalInlineChat_AcceptedLineAdditions int
+ TotalInlineChat_AcceptedLineDeletions int
+ TotalInlineChat_DismissalEventCount int
+ TotalInlineChat_DismissedLineAdditions int
+ TotalInlineChat_DismissedLineDeletions int
+ TotalInlineChat_RejectedLineAdditions int
+ TotalInlineChat_RejectedLineDeletions int
+ TotalInlineChat_RejectionEventCount int
+ TotalInlineChat_TotalEventCount int
+ TotalInline_AICodeLines int
+ TotalInline_AcceptanceCount int
+ TotalInline_SuggestionsCount int
+
+ // 平均指标
+ AvgCodeReview_FindingsCount float64
+ AvgCodeReview_SucceededEventCount float64
+ AvgInlineChat_AcceptanceEventCount float64
+ AvgInlineChat_TotalEventCount float64
+ AvgInline_AICodeLines float64
+ AvgInline_AcceptanceCount float64
+ AvgInline_SuggestionsCount float64
+
+ // 接受率指标
+ AcceptanceRate float64
+}
+
+func (QDevUserMetrics) TableName() string {
+ return "_tool_q_dev_user_metrics"
+}
diff --git a/backend/plugins/q_dev/models/migrationscripts/register.go
b/backend/plugins/q_dev/models/migrationscripts/register.go
new file mode 100644
index 000000000..885bdf2fc
--- /dev/null
+++ b/backend/plugins/q_dev/models/migrationscripts/register.go
@@ -0,0 +1,29 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/plugin"
+)
+
+// All return all migration scripts
+func All() []plugin.MigrationScript {
+ return []plugin.MigrationScript{
+ new(initTables),
+ }
+}
\ No newline at end of file
diff --git a/backend/plugins/q_dev/models/s3_file_meta.go
b/backend/plugins/q_dev/models/s3_file_meta.go
new file mode 100644
index 000000000..003cbc16a
--- /dev/null
+++ b/backend/plugins/q_dev/models/s3_file_meta.go
@@ -0,0 +1,38 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/core/models/common"
+)
+
+// QDevS3FileMeta 存储S3文件的元数据信息
+type QDevS3FileMeta struct {
+ common.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"`
+ FileName string `gorm:"primaryKey;type:varchar(255)"`
+ S3Path string `gorm:"type:varchar(512)" json:"s3Path"`
+ Processed bool `gorm:"default:false"`
+ ProcessedTime *time.Time `gorm:"default:null"`
+}
+
+func (QDevS3FileMeta) TableName() string {
+ return "_tool_q_dev_s3_file_meta"
+}
diff --git a/backend/plugins/q_dev/models/user_data.go
b/backend/plugins/q_dev/models/user_data.go
new file mode 100644
index 000000000..ce0692b9b
--- /dev/null
+++ b/backend/plugins/q_dev/models/user_data.go
@@ -0,0 +1,51 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "time"
+
+ "github.com/apache/incubator-devlake/core/models/common"
+)
+
+// QDevUserData 存储从CSV中提取的原始数据
+type QDevUserData struct {
+ common.Model
+ ConnectionId uint64 `gorm:"primaryKey"`
+ UserId string `gorm:"index" json:"userId"`
+ Date time.Time `gorm:"index" json:"date"`
+ CodeReview_FindingsCount int
+ CodeReview_SucceededEventCount int
+ InlineChat_AcceptanceEventCount int
+ InlineChat_AcceptedLineAdditions int
+ InlineChat_AcceptedLineDeletions int
+ InlineChat_DismissalEventCount int
+ InlineChat_DismissedLineAdditions int
+ InlineChat_DismissedLineDeletions int
+ InlineChat_RejectedLineAdditions int
+ InlineChat_RejectedLineDeletions int
+ InlineChat_RejectionEventCount int
+ InlineChat_TotalEventCount int
+ Inline_AICodeLines int
+ Inline_AcceptanceCount int
+ Inline_SuggestionsCount int
+}
+
+func (QDevUserData) TableName() string {
+ return "_tool_q_dev_user_data"
+}
diff --git a/backend/plugins/q_dev/models/user_metrics.go
b/backend/plugins/q_dev/models/user_metrics.go
new file mode 100644
index 000000000..baebc3ead
--- /dev/null
+++ b/backend/plugins/q_dev/models/user_metrics.go
@@ -0,0 +1,66 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/core/models/common"
+ "time"
+)
+
+// QDevUserMetrics 存储按用户聚合的指标数据
+type QDevUserMetrics struct {
+ common.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"`
+ UserId string `gorm:"primaryKey"`
+ FirstDate time.Time
+ LastDate time.Time
+ TotalDays int
+
+ // 聚合指标
+ TotalCodeReview_FindingsCount int
+ TotalCodeReview_SucceededEventCount int
+ TotalInlineChat_AcceptanceEventCount int
+ TotalInlineChat_AcceptedLineAdditions int
+ TotalInlineChat_AcceptedLineDeletions int
+ TotalInlineChat_DismissalEventCount int
+ TotalInlineChat_DismissedLineAdditions int
+ TotalInlineChat_DismissedLineDeletions int
+ TotalInlineChat_RejectedLineAdditions int
+ TotalInlineChat_RejectedLineDeletions int
+ TotalInlineChat_RejectionEventCount int
+ TotalInlineChat_TotalEventCount int
+ TotalInline_AICodeLines int
+ TotalInline_AcceptanceCount int
+ TotalInline_SuggestionsCount int
+
+ // 平均指标
+ AvgCodeReview_FindingsCount float64
+ AvgCodeReview_SucceededEventCount float64
+ AvgInlineChat_AcceptanceEventCount float64
+ AvgInlineChat_TotalEventCount float64
+ AvgInline_AICodeLines float64
+ AvgInline_AcceptanceCount float64
+ AvgInline_SuggestionsCount float64
+
+ // 接受率指标
+ AcceptanceRate float64
+}
+
+func (QDevUserMetrics) TableName() string {
+ return "_tool_q_dev_user_metrics"
+}
\ No newline at end of file
diff --git a/backend/plugins/q_dev/q_dev.go b/backend/plugins/q_dev/q_dev.go
new file mode 100644
index 000000000..2cb9a7eea
--- /dev/null
+++ b/backend/plugins/q_dev/q_dev.go
@@ -0,0 +1,43 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "github.com/apache/incubator-devlake/core/runner"
+ "github.com/apache/incubator-devlake/plugins/q_dev/impl"
+ "github.com/spf13/cobra"
+)
+
+var PluginEntry impl.QDev
+
+// standalone mode for debugging
+func main() {
+ cmd := &cobra.Command{Use: "q_dev"}
+ connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "q_dev
connection id")
+ s3Prefix := cmd.Flags().StringP("s3Prefix", "p", "", "s3 bucket prefix
for q_dev data")
+ timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data
that are created after specified time, ie 2006-01-02T15:04:05Z")
+
+ _ = cmd.MarkFlagRequired("connectionId")
+ cmd.Run = func(cmd *cobra.Command, args []string) {
+ runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
+ "connectionId": *connectionId,
+ "s3Prefix": *s3Prefix,
+ }, *timeAfter)
+ }
+ runner.RunCmd(cmd)
+}
\ No newline at end of file
diff --git a/backend/plugins/q_dev/tasks/s3_client.go
b/backend/plugins/q_dev/tasks/s3_client.go
new file mode 100644
index 000000000..679b99f11
--- /dev/null
+++ b/backend/plugins/q_dev/tasks/s3_client.go
@@ -0,0 +1,47 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/plugins/q_dev/models"
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/credentials"
+ "github.com/aws/aws-sdk-go/aws/session"
+ "github.com/aws/aws-sdk-go/service/s3"
+)
+
+func NewQDevS3Client(taskCtx plugin.TaskContext, connection
*models.QDevConnection) (*QDevS3Client, errors.Error) {
+ // 创建AWS session
+ sess, err := session.NewSession(&aws.Config{
+ Region: aws.String(connection.Region),
+ Credentials:
credentials.NewStaticCredentials(connection.AccessKeyId,
connection.SecretAccessKey, ""),
+ })
+ if err != nil {
+ return nil, errors.Convert(err)
+ }
+
+ // 创建S3服务客户端
+ s3Client := s3.New(sess)
+
+ return &QDevS3Client{
+ S3: s3Client,
+ Bucket: connection.Bucket,
+ }, nil
+}
\ No newline at end of file
diff --git a/backend/plugins/q_dev/tasks/s3_file_collector.go
b/backend/plugins/q_dev/tasks/s3_file_collector.go
new file mode 100644
index 000000000..d06f066e1
--- /dev/null
+++ b/backend/plugins/q_dev/tasks/s3_file_collector.go
@@ -0,0 +1,103 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/plugins/q_dev/models"
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/service/s3"
+ "strings"
+)
+
+var _ plugin.SubTaskEntryPoint = CollectQDevS3Files
+
+// CollectQDevS3Files 收集S3文件元数据
+func CollectQDevS3Files(taskCtx plugin.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*QDevTaskData)
+ db := taskCtx.GetDal()
+
+ // 列出指定前缀下的所有对象
+ var continuationToken *string
+ prefix := data.Options.S3Prefix
+ if prefix != "" && !strings.HasSuffix(prefix, "/") {
+ prefix = prefix + "/"
+ }
+
+ taskCtx.SetProgress(0, -1)
+
+ // 清空以前的元数据记录
+ err := db.Delete(&models.QDevS3FileMeta{}, dal.Where("connection_id =
?", data.Options.ConnectionId))
+ if err != nil {
+ return errors.Default.Wrap(err, "failed to clean previous file
metadata")
+ }
+
+ for {
+ input := &s3.ListObjectsV2Input{
+ Bucket: aws.String(data.S3Client.Bucket),
+ Prefix: aws.String(prefix),
+ ContinuationToken: continuationToken,
+ }
+
+ result, err := data.S3Client.S3.ListObjectsV2(input)
+ if err != nil {
+ return errors.Convert(err)
+ }
+
+ // 处理每个CSV文件
+ for _, object := range result.Contents {
+ // 只处理CSV文件
+ if !strings.HasSuffix(*object.Key, ".csv") {
+ continue
+ }
+
+ // 保存文件元数据
+ fileMeta := &models.QDevS3FileMeta{
+ ConnectionId: data.Options.ConnectionId,
+ FileName: *object.Key,
+ S3Path: *object.Key,
+ Processed: false,
+ }
+
+ err = db.Create(fileMeta)
+ if err != nil {
+ return errors.Default.Wrap(err, "failed to
create file metadata")
+ }
+
+ taskCtx.IncProgress(1)
+ }
+
+ // 如果没有更多对象，退出循环
+ if !*result.IsTruncated {
+ break
+ }
+
+ continuationToken = result.NextContinuationToken
+ }
+
+ return nil
+}
+
+var CollectQDevS3FilesMeta = plugin.SubTaskMeta{
+ Name: "collectQDevS3Files",
+ EntryPoint: CollectQDevS3Files,
+ EnabledByDefault: true,
+ Description: "Collect S3 file metadata from AWS S3 bucket",
+}
diff --git a/backend/plugins/q_dev/tasks/task_data.go
b/backend/plugins/q_dev/tasks/task_data.go
new file mode 100644
index 000000000..ce94a70ea
--- /dev/null
+++ b/backend/plugins/q_dev/tasks/task_data.go
@@ -0,0 +1,45 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/aws/aws-sdk-go/service/s3"
+)
+
+type QDevApiParams struct {
+ ConnectionId uint64 `json:"connectionId"`
+}
+
+type QDevOptions struct {
+ ConnectionId uint64 `json:"connectionId"`
+ S3Prefix string `json:"s3Prefix"`
+}
+
+type QDevTaskData struct {
+ Options *QDevOptions
+ S3Client *QDevS3Client
+}
+
+type QDevS3Client struct {
+ S3 *s3.S3
+ Bucket string
+}
+
+func (client *QDevS3Client) Close() {
+ // S3客户端不需要特别关闭操作
+}
\ No newline at end of file
diff --git a/backend/plugins/table_info_test.go
b/backend/plugins/table_info_test.go
index 83635fd58..ba83acd77 100644
--- a/backend/plugins/table_info_test.go
+++ b/backend/plugins/table_info_test.go
@@ -44,6 +44,7 @@ import (
opsgenie "github.com/apache/incubator-devlake/plugins/opsgenie/impl"
org "github.com/apache/incubator-devlake/plugins/org/impl"
pagerduty "github.com/apache/incubator-devlake/plugins/pagerduty/impl"
+ q_dev "github.com/apache/incubator-devlake/plugins/q_dev/impl"
refdiff "github.com/apache/incubator-devlake/plugins/refdiff/impl"
slack "github.com/apache/incubator-devlake/plugins/slack/impl"
sonarqube "github.com/apache/incubator-devlake/plugins/sonarqube/impl"
@@ -92,6 +93,7 @@ func Test_GetPluginTablesInfo(t *testing.T) {
checker.FeedIn("opsgenie/models", opsgenie.Opsgenie{}.GetTablesInfo)
checker.FeedIn("linker/models", linker.Linker{}.GetTablesInfo)
checker.FeedIn("issue_trace/models",
issueTrace.IssueTrace{}.GetTablesInfo)
+ checker.FeedIn("q_dev/models", q_dev.QDev{}.GetTablesInfo)
err := checker.Verify()
if err != nil {
t.Error(err)
