This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new ef73a6ca9 feat(plugins): add config-ui to slack (#8557)
ef73a6ca9 is described below
commit ef73a6ca9b7b03b32e141a42a47f330a39e3c089
Author: Richard Boisvert <[email protected]>
AuthorDate: Wed Aug 27 04:20:03 2025 -0400
feat(plugins): add config-ui to slack (#8557)
This change implements the frontend UI configuration for the Slack plugin,
enabling users to set up Slack connections through the DevLake dashboard
interface. Previously, the Slack plugin existed only as a backend
implementation without any user-facing configuration interface.
The implementation adds:
- Connection setup form with fields for name, endpoint URL, Slack bot
token, and rate limiting
- Channel selection interface supporting both public and private channels
based on bot permissions
- Integration with existing backend APIs for connection testing and scope
management
- Consistent UX patterns matching other DevLake plugins (GitHub, Jira, etc.)
This enables users to configure Slack data collection through the standard
DevLake workflow instead of requiring manual API calls or configuration file
editing.
https://github.com/apache/incubator-devlake/issues/8555
---
backend/plugins/slack/api/blueprint_v200.go | 62 ++++++++
backend/plugins/slack/api/init.go | 20 +++
backend/plugins/slack/api/remote_api.go | 162 +++++++++++++++++++++
backend/plugins/slack/api/scope_api.go | 107 ++++++++++++++
.../slack/api/{init.go => scope_state_api.go} | 29 ++--
backend/plugins/slack/impl/impl.go | 30 +++-
backend/plugins/slack/models/channel.go | 27 +++-
.../20250826_add_scope_config_id_to_channel.go} | 37 +++--
.../slack/models/migrationscripts/register.go | 1 +
backend/plugins/slack/slack.go | 2 +
.../slack/tasks/channel_message_collector.go | 39 ++---
.../slack/tasks/channel_message_extractor.go | 10 +-
backend/plugins/slack/tasks/task_data.go | 13 ++
backend/plugins/slack/tasks/thread_collector.go | 22 +--
backend/plugins/slack/tasks/thread_extractor.go | 10 +-
config-ui/src/plugins/register/index.ts | 2 +
.../src/plugins/register/slack/assets/icon.svg | 22 +++
config-ui/src/plugins/register/slack/config.tsx | 63 ++++++++
config-ui/src/plugins/register/slack/index.ts | 19 +++
config-ui/src/release/stable.ts | 4 +
20 files changed, 599 insertions(+), 82 deletions(-)
diff --git a/backend/plugins/slack/api/blueprint_v200.go
b/backend/plugins/slack/api/blueprint_v200.go
new file mode 100644
index 000000000..8c1c30f11
--- /dev/null
+++ b/backend/plugins/slack/api/blueprint_v200.go
@@ -0,0 +1,62 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helperapi "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/slack/tasks"
+)
+
+func MakeDataSourcePipelinePlanV200(
+ subtaskMetas []plugin.SubTaskMeta,
+ connectionId uint64,
+ bpScopes []*coreModels.BlueprintScope,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
+ // Map blueprint scopes to actual Slack channels
+ scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId,
bpScopes)
+ if err != nil {
+ return nil, nil, err
+ }
+ // Build one stage per selected channel
+ plan := make(coreModels.PipelinePlan, len(scopeDetails))
+ for i, scopeDetail := range scopeDetails {
+ stage := plan[i]
+ if stage == nil {
+ stage = coreModels.PipelineStage{}
+ }
+ // Only include CROSS domain subtasks; Slack subtasks define
DomainTypes accordingly.
+ entities := []string{plugin.DOMAIN_TYPE_CROSS}
+ scope := scopeDetail.Scope // *models.SlackChannel
+ task, err := helperapi.MakePipelinePlanTask(
+ "slack",
+ subtaskMetas,
+ entities,
+ tasks.SlackOptions{ConnectionId: connectionId,
ChannelId: scope.ScopeId()},
+ )
+ if err != nil {
+ return nil, nil, err
+ }
+ stage = append(stage, task)
+ plan[i] = stage
+ }
+ // No domain scopes emitted by Slack for now
+ return plan, nil, nil
+}
diff --git a/backend/plugins/slack/api/init.go
b/backend/plugins/slack/api/init.go
index 1da6ff414..7d2e0defb 100644
--- a/backend/plugins/slack/api/init.go
+++ b/backend/plugins/slack/api/init.go
@@ -21,12 +21,18 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/srvhelper"
+ "github.com/apache/incubator-devlake/plugins/slack/models"
"github.com/go-playground/validator/v10"
)
var vld *validator.Validate
var connectionHelper *api.ConnectionApiHelper
var basicRes context.BasicRes
+var dsHelper *api.DsHelper[models.SlackConnection, models.SlackChannel,
srvhelper.NoScopeConfig]
+var raProxy *api.DsRemoteApiProxyHelper[models.SlackConnection]
+var raScopeList *api.DsRemoteApiScopeListHelper[models.SlackConnection,
models.SlackChannel, SlackRemotePagination]
+var raScopeSearch *api.DsRemoteApiScopeSearchHelper[models.SlackConnection,
models.SlackChannel]
func Init(br context.BasicRes, p plugin.PluginMeta) {
@@ -37,4 +43,18 @@ func Init(br context.BasicRes, p plugin.PluginMeta) {
vld,
p.Name(),
)
+
+ dsHelper = api.NewDataSourceHelper[
+ models.SlackConnection, models.SlackChannel,
srvhelper.NoScopeConfig,
+ ](
+ br,
+ p.Name(),
+ []string{"name"},
+ func(c models.SlackConnection) models.SlackConnection { return
c.Sanitize() },
+ func(s models.SlackChannel) models.SlackChannel { return s },
+ nil,
+ )
+ raProxy =
api.NewDsRemoteApiProxyHelper[models.SlackConnection](dsHelper.ConnApi.ModelApiHelper)
+ raScopeList = api.NewDsRemoteApiScopeListHelper[models.SlackConnection,
models.SlackChannel, SlackRemotePagination](raProxy, listSlackRemoteScopes)
+ raScopeSearch =
api.NewDsRemoteApiScopeSearchHelper[models.SlackConnection,
models.SlackChannel](raProxy, searchSlackRemoteScopes)
}
diff --git a/backend/plugins/slack/api/remote_api.go
b/backend/plugins/slack/api/remote_api.go
new file mode 100644
index 000000000..273f36078
--- /dev/null
+++ b/backend/plugins/slack/api/remote_api.go
@@ -0,0 +1,162 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "encoding/json"
+ "net/url"
+ "strconv"
+ "strings"
+
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ dsmodels
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/models"
+ "github.com/apache/incubator-devlake/plugins/slack/models"
+)
+
+type SlackRemotePagination struct {
+ Cursor string `json:"cursor"`
+ Limit int `json:"limit"`
+}
+
+type slackConvListResp struct {
+ Ok bool `json:"ok"`
+ Error string `json:"error"`
+ Needed string `json:"needed"`
+ Provided string `json:"provided"`
+ Channels []json.RawMessage `json:"channels"`
+ ResponseMetadata struct {
+ NextCursor string `json:"next_cursor"`
+ } `json:"response_metadata"`
+}
+
+func listSlackRemoteScopes(
+ _ *models.SlackConnection,
+ apiClient plugin.ApiClient,
+ _ string,
+ page SlackRemotePagination,
+) (
+ children []dsmodels.DsRemoteApiScopeListEntry[models.SlackChannel],
+ nextPage *SlackRemotePagination,
+ err errors.Error,
+) {
+ if page.Limit == 0 {
+ page.Limit = 100
+ }
+ // helper to perform API call with given query
+ call := func(q url.Values) (*slackConvListResp, errors.Error) {
+ res, e := apiClient.Get("conversations.list", q, nil)
+ if e != nil {
+ return nil, e
+ }
+ resp := &slackConvListResp{}
+ if e = helper.UnmarshalResponse(res, resp); e != nil {
+ return nil, e
+ }
+ return resp, nil
+ }
+
+ q := url.Values{}
+ q.Set("limit", strconv.Itoa(page.Limit))
+ if page.Cursor != "" {
+ q.Set("cursor", page.Cursor)
+ }
+
+ q.Set("types", "public_channel,private_channel")
+ resp, e := call(q)
+ if e != nil {
+ err = e
+ return
+ }
+ // handle missing_scope gracefully by retrying with private_channel
only if channels:read is missing
+ if !resp.Ok && resp.Error == "missing_scope" {
+ if strings.Contains(resp.Needed, "channels:read") {
+ // retry with private channels only (requires
groups:read)
+ q.Set("types", "private_channel")
+ resp, e = call(q)
+ if e != nil {
+ err = e
+ return
+ }
+ }
+ }
+ if !resp.Ok {
+ err = errors.BadInput.New("slack conversations.list error: " +
resp.Error)
+ return
+ }
+
+ for _, raw := range resp.Channels {
+ var ch models.SlackChannel
+ if e := errors.Convert(json.Unmarshal(raw, &ch)); e != nil {
+ err = e
+ return
+ }
+ children = append(children,
dsmodels.DsRemoteApiScopeListEntry[models.SlackChannel]{
+ Type: helper.RAS_ENTRY_TYPE_SCOPE,
+ Id: ch.Id,
+ Name: ch.Name,
+ FullName: ch.Name,
+ Data: &ch,
+ })
+ }
+ if resp.ResponseMetadata.NextCursor != "" {
+ nextPage = &SlackRemotePagination{Cursor:
resp.ResponseMetadata.NextCursor, Limit: page.Limit}
+ }
+ return
+}
+
+func searchSlackRemoteScopes(
+ apiClient plugin.ApiClient,
+ params *dsmodels.DsRemoteApiScopeSearchParams,
+) (
+ children []dsmodels.DsRemoteApiScopeListEntry[models.SlackChannel],
+ err errors.Error,
+) {
+ cursor := ""
+ remaining := params.PageSize
+ for remaining > 0 {
+ list, next, e := listSlackRemoteScopes(nil, apiClient, "",
SlackRemotePagination{Cursor: cursor, Limit: 200})
+ if e != nil {
+ err = e
+ return
+ }
+ for _, it := range list {
+ if params.Search == "" || (it.Name != "" &&
strings.Contains(strings.ToLower(it.Name), strings.ToLower(params.Search))) {
+ children = append(children, it)
+ remaining--
+ if remaining == 0 {
+ return
+ }
+ }
+ }
+ if next == nil || next.Cursor == "" {
+ break
+ }
+ cursor = next.Cursor
+ }
+ return
+}
+
+func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ return raScopeList.Get(input)
+}
+
+func SearchRemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ return raScopeSearch.Get(input)
+}
diff --git a/backend/plugins/slack/api/scope_api.go
b/backend/plugins/slack/api/scope_api.go
new file mode 100644
index 000000000..68a264c71
--- /dev/null
+++ b/backend/plugins/slack/api/scope_api.go
@@ -0,0 +1,107 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/srvhelper"
+ "github.com/apache/incubator-devlake/plugins/slack/models"
+)
+
+type PutScopesReqBody api.PutScopesReqBody[models.SlackChannel]
+type ScopeDetail srvhelper.ScopeDetail[models.SlackChannel,
srvhelper.NoScopeConfig]
+
+// PutScopes create or update slack channels (scopes)
+// @Summary create or update Slack channels
+// @Description Create or update Slack channels
+// @Tags plugins/slack
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param scope body PutScopesReqBody true "json"
+// @Success 200 {object} []models.SlackChannel
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/slack/connections/{connectionId}/scopes [PUT]
+func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ return dsHelper.ScopeApi.PutMultiple(input)
+}
+
+// GetScopeList get Slack channels
+// @Summary get Slack channels
+// @Description get Slack channels
+// @Tags plugins/slack
+// @Param connectionId path int false "connection ID"
+// @Param pageSize query int false "page size, default 50"
+// @Param page query int false "page size, default 1"
+// @Param blueprints query bool false "also return blueprints using these
scopes as part of the payload"
+// @Success 200 {object} []ScopeDetail
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/slack/connections/{connectionId}/scopes [GET]
+func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ return dsHelper.ScopeApi.GetPage(input)
+}
+
+// GetScope get one Slack channel
+// @Summary get one Slack channel
+// @Description get one Slack channel
+// @Tags plugins/slack
+// @Param connectionId path int false "connection ID"
+// @Param scopeId path string false "channel id"
+// @Param blueprints query bool false "also return blueprints using this scope
as part of the payload"
+// @Success 200 {object} ScopeDetail
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/slack/connections/{connectionId}/scopes/{scopeId} [GET]
+func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ return dsHelper.ScopeApi.GetScopeDetail(input)
+}
+
+// PatchScope patch a Slack channel
+// @Summary patch a Slack channel
+// @Description patch a Slack channel
+// @Tags plugins/slack
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param scopeId path string false "channel id"
+// @Param scope body models.SlackChannel true "json"
+// @Success 200 {object} models.SlackChannel
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/slack/connections/{connectionId}/scopes/{scopeId} [PATCH]
+func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ return dsHelper.ScopeApi.Patch(input)
+}
+
+// DeleteScope delete plugin data associated with the scope and optionally the
scope itself
+// @Summary delete plugin data associated with the scope and optionally the
scope itself
+// @Description delete data associated with plugin scope
+// @Tags plugins/slack
+// @Param connectionId path int true "connection ID"
+// @Param scopeId path string true "channel id"
+// @Param delete_data_only query bool false "Only delete the scope data, not
the scope itself"
+// @Success 200
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 409 {object} srvhelper.DsRefs "References exist to this scope"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/slack/connections/{connectionId}/scopes/{scopeId} [DELETE]
+func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ return dsHelper.ScopeApi.Delete(input)
+}
diff --git a/backend/plugins/slack/api/init.go
b/backend/plugins/slack/api/scope_state_api.go
similarity index 51%
copy from backend/plugins/slack/api/init.go
copy to backend/plugins/slack/api/scope_state_api.go
index 1da6ff414..df8136e60 100644
--- a/backend/plugins/slack/api/init.go
+++ b/backend/plugins/slack/api/scope_state_api.go
@@ -18,23 +18,20 @@ limitations under the License.
package api
import (
- "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/go-playground/validator/v10"
)
-var vld *validator.Validate
-var connectionHelper *api.ConnectionApiHelper
-var basicRes context.BasicRes
-
-func Init(br context.BasicRes, p plugin.PluginMeta) {
-
- basicRes = br
- vld = validator.New()
- connectionHelper = api.NewConnectionHelper(
- basicRes,
- vld,
- p.Name(),
- )
+// GetScopeLatestSyncState get one Slack channel's latest sync state
+// @Summary get one Slack channel's latest sync state
+// @Description get one Slack channel's latest sync state
+// @Tags plugins/slack
+// @Param connectionId path int true "connection ID"
+// @Param scopeId path string true "channel id"
+// @Success 200 {object} []models.LatestSyncState
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router
/plugins/slack/connections/{connectionId}/scopes/{scopeId}/latest-sync-state
[GET]
+func GetScopeLatestSyncState(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ return dsHelper.ScopeApi.GetScopeLatestSyncState(input)
}
diff --git a/backend/plugins/slack/impl/impl.go
b/backend/plugins/slack/impl/impl.go
index e526d0642..47eb3084b 100644
--- a/backend/plugins/slack/impl/impl.go
+++ b/backend/plugins/slack/impl/impl.go
@@ -19,10 +19,12 @@ package impl
import (
"fmt"
+
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/slack/api"
@@ -38,6 +40,7 @@ var _ interface {
plugin.PluginApi
plugin.PluginModel
plugin.PluginMigration
+ plugin.DataSourcePluginBlueprintV200
plugin.CloseablePluginTask
plugin.PluginSource
} = (*Slack)(nil)
@@ -71,7 +74,7 @@ func (p Slack) Connection() dal.Tabler {
}
func (p Slack) Scope() plugin.ToolLayerScope {
- return nil
+ return &models.SlackChannel{}
}
func (p Slack) ScopeConfig() dal.Tabler {
@@ -126,6 +129,13 @@ func (p Slack) MigrationScripts() []plugin.MigrationScript
{
return migrationscripts.All()
}
+func (p Slack) MakeDataSourcePipelinePlanV200(
+ connectionId uint64,
+ scopes []*coreModels.BlueprintScope,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
+ return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(),
connectionId, scopes)
+}
+
func (p Slack) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
return map[string]map[string]plugin.ApiResourceHandler{
"test": {
@@ -143,6 +153,24 @@ func (p Slack) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
"connections/:connectionId/test": {
"POST": api.TestExistingConnection,
},
+ "connections/:connectionId/remote-scopes": {
+ "GET": api.RemoteScopes,
+ },
+ "connections/:connectionId/search-remote-scopes": {
+ "GET": api.SearchRemoteScopes,
+ },
+ "connections/:connectionId/scopes": {
+ "GET": api.GetScopeList,
+ "PUT": api.PutScopes,
+ },
+ "connections/:connectionId/scopes/:scopeId": {
+ "GET": api.GetScope,
+ "PATCH": api.PatchScope,
+ "DELETE": api.DeleteScope,
+ },
+ "connections/:connectionId/scopes/:scopeId/latest-sync-state": {
+ "GET": api.GetScopeLatestSyncState,
+ },
}
}
diff --git a/backend/plugins/slack/models/channel.go
b/backend/plugins/slack/models/channel.go
index ba8aa1724..edad51cc8 100644
--- a/backend/plugins/slack/models/channel.go
+++ b/backend/plugins/slack/models/channel.go
@@ -19,11 +19,11 @@ package models
import (
"github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/core/plugin"
)
type SlackChannel struct {
- common.NoPKModel `json:"-"`
- ConnectionId uint64 `gorm:"primaryKey"`
+ common.Scope `mapstructure:",squash"`
Id string `json:"id" gorm:"primaryKey"`
Name string `json:"name"`
IsChannel bool `json:"is_channel"`
@@ -50,3 +50,26 @@ type SlackChannel struct {
func (SlackChannel) TableName() string {
return "_tool_slack_channels"
}
+
+func (s SlackChannel) ScopeId() string {
+ return s.Id
+}
+
+func (s SlackChannel) ScopeName() string {
+ return s.Name
+}
+
+func (s SlackChannel) ScopeFullName() string {
+ return s.Name
+}
+
+func (s SlackChannel) ScopeParams() interface{} {
+ return &SlackParams{ConnectionId: s.ConnectionId, ScopeId: s.Id}
+}
+
+type SlackParams struct {
+ ConnectionId uint64
+ ScopeId string
+}
+
+var _ plugin.ToolLayerScope = (*SlackChannel)(nil)
diff --git a/backend/plugins/slack/api/init.go
b/backend/plugins/slack/models/migrationscripts/20250826_add_scope_config_id_to_channel.go
similarity index 51%
copy from backend/plugins/slack/api/init.go
copy to
backend/plugins/slack/models/migrationscripts/20250826_add_scope_config_id_to_channel.go
index 1da6ff414..1d0bc95aa 100644
--- a/backend/plugins/slack/api/init.go
+++
b/backend/plugins/slack/models/migrationscripts/20250826_add_scope_config_id_to_channel.go
@@ -15,26 +15,35 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package api
+package migrationscripts
import (
"github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/go-playground/validator/v10"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
)
-var vld *validator.Validate
-var connectionHelper *api.ConnectionApiHelper
-var basicRes context.BasicRes
+var _ plugin.MigrationScript = (*addScopeConfigIdToSlackChannel)(nil)
-func Init(br context.BasicRes, p plugin.PluginMeta) {
+type slackChannel20250826 struct {
+ ScopeConfigId uint64 `json:"scopeConfigId,omitempty"
mapstructure:"scopeConfigId,omitempty"`
+}
+
+func (slackChannel20250826) TableName() string {
+ return "_tool_slack_channels"
+}
+
+type addScopeConfigIdToSlackChannel struct{}
+
+func (script *addScopeConfigIdToSlackChannel) Up(basicRes context.BasicRes)
errors.Error {
+ return migrationhelper.AutoMigrateTables(basicRes,
&slackChannel20250826{})
+}
+
+func (*addScopeConfigIdToSlackChannel) Version() uint64 {
+ return 20250826000001
+}
- basicRes = br
- vld = validator.New()
- connectionHelper = api.NewConnectionHelper(
- basicRes,
- vld,
- p.Name(),
- )
+func (*addScopeConfigIdToSlackChannel) Name() string {
+ return "Add scope_config_id to _tool_slack_channels"
}
diff --git a/backend/plugins/slack/models/migrationscripts/register.go
b/backend/plugins/slack/models/migrationscripts/register.go
index ec054748c..d00f39a94 100644
--- a/backend/plugins/slack/models/migrationscripts/register.go
+++ b/backend/plugins/slack/models/migrationscripts/register.go
@@ -25,5 +25,6 @@ import (
func All() []plugin.MigrationScript {
return []plugin.MigrationScript{
new(addInitTables),
+ new(addScopeConfigIdToSlackChannel),
}
}
diff --git a/backend/plugins/slack/slack.go b/backend/plugins/slack/slack.go
index 19734bf96..a396cd038 100644
--- a/backend/plugins/slack/slack.go
+++ b/backend/plugins/slack/slack.go
@@ -29,11 +29,13 @@ var PluginEntry impl.Slack
func main() {
cmd := &cobra.Command{Use: "slack"}
connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "slack
connection id")
+ channelId := cmd.Flags().StringP("channelId", "s", "", "slack channel
id (scopeId)")
timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data
that are created after specified time, ie 2006-01-02T15:04:05Z")
_ = cmd.MarkFlagRequired("connectionId")
cmd.Run = func(cmd *cobra.Command, args []string) {
runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
"connectionId": *connectionId,
+ "channelId": *channelId,
}, *timeAfter)
}
runner.RunCmd(cmd)
diff --git a/backend/plugins/slack/tasks/channel_message_collector.go
b/backend/plugins/slack/tasks/channel_message_collector.go
index 7c12391d6..98bc34641 100644
--- a/backend/plugins/slack/tasks/channel_message_collector.go
+++ b/backend/plugins/slack/tasks/channel_message_collector.go
@@ -19,15 +19,14 @@ package tasks
import (
"encoding/json"
- "github.com/apache/incubator-devlake/core/dal"
+ "net/http"
+ "net/url"
+ "strconv"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/slack/apimodels"
- "net/http"
- "net/url"
- "reflect"
- "strconv"
)
const RAW_CHANNEL_MESSAGE_TABLE = "slack_channel_message"
@@ -40,33 +39,16 @@ type ChannelInput struct {
func CollectChannelMessage(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*SlackTaskData)
- db := taskCtx.GetDal()
-
- clauses := []dal.Clause{
- dal.Select("id as channel_id"),
- dal.From("_tool_slack_channels"),
- dal.Where("connection_id=?", data.Options.ConnectionId),
- }
-
- // construct the input iterator
- cursor, err := db.Cursor(clauses...)
- if err != nil {
- return err
- }
- // smaller struct can reduce memory footprint, we should try to avoid
using big struct
- iterator, err := api.NewDalCursorIterator(db, cursor,
reflect.TypeOf(ChannelInput{}))
- if err != nil {
- return err
- }
+ // Build a single-item iterator for the specific channel passed in
options
+ iterator := api.NewQueueIterator()
+ iterator.Push(&ChannelInput{ChannelId: data.Options.ChannelId})
pageSize := 100
collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: SlackApiParams{
- ConnectionId: data.Options.ConnectionId,
- },
- Table: RAW_CHANNEL_MESSAGE_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_CHANNEL_MESSAGE_TABLE,
},
ApiClient: data.ApiClient,
Incremental: false,
@@ -115,4 +97,5 @@ var CollectChannelMessageMeta = plugin.SubTaskMeta{
EntryPoint: CollectChannelMessage,
EnabledByDefault: true,
Description: "Collect channel message from Slack api",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
}
diff --git a/backend/plugins/slack/tasks/channel_message_extractor.go
b/backend/plugins/slack/tasks/channel_message_extractor.go
index 91a048133..690fc5635 100644
--- a/backend/plugins/slack/tasks/channel_message_extractor.go
+++ b/backend/plugins/slack/tasks/channel_message_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -32,11 +33,9 @@ func ExtractChannelMessage(taskCtx plugin.SubTaskContext)
errors.Error {
data := taskCtx.GetData().(*SlackTaskData)
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: SlackApiParams{
- ConnectionId: data.Options.ConnectionId,
- },
- Table: RAW_CHANNEL_MESSAGE_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_CHANNEL_MESSAGE_TABLE,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
channel := &ChannelInput{}
@@ -82,4 +81,5 @@ var ExtractChannelMessageMeta = plugin.SubTaskMeta{
EntryPoint: ExtractChannelMessage,
EnabledByDefault: true,
Description: "Extract raw channel messages data into tool layer
table",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
}
diff --git a/backend/plugins/slack/tasks/task_data.go
b/backend/plugins/slack/tasks/task_data.go
index 41c811fb6..a3366f0c7 100644
--- a/backend/plugins/slack/tasks/task_data.go
+++ b/backend/plugins/slack/tasks/task_data.go
@@ -27,9 +27,22 @@ type SlackApiParams struct {
type SlackOptions struct {
ConnectionId uint64 `json:"connectionId"`
+ ChannelId string `json:"channelId,omitempty"
mapstructure:"channelId,omitempty"`
}
type SlackTaskData struct {
Options *SlackOptions
ApiClient *helper.ApiAsyncClient
}
+
+// SlackParams defines the raw params shape used to tag raw tables for scoping
and latest-sync-state
+type SlackParams struct {
+ ConnectionId uint64 `json:"connectionId"`
+ ScopeId string `json:"scopeId"`
+}
+
+// GetParams implements api.TaskOptions to ensure raw_data_params include both
connection and scope
+func (p *SlackOptions) GetParams() any {
+ scopeId := p.ChannelId
+ return &SlackParams{ConnectionId: p.ConnectionId, ScopeId: scopeId}
+}
diff --git a/backend/plugins/slack/tasks/thread_collector.go
b/backend/plugins/slack/tasks/thread_collector.go
index 2019636a3..5c88a748e 100644
--- a/backend/plugins/slack/tasks/thread_collector.go
+++ b/backend/plugins/slack/tasks/thread_collector.go
@@ -19,15 +19,16 @@ package tasks
import (
"encoding/json"
+ "net/http"
+ "net/url"
+ "reflect"
+ "strconv"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/slack/apimodels"
- "net/http"
- "net/url"
- "reflect"
- "strconv"
)
const RAW_THREAD_TABLE = "slack_thread"
@@ -44,9 +45,9 @@ func CollectThread(taskCtx plugin.SubTaskContext)
errors.Error {
db := taskCtx.GetDal()
clauses := []dal.Clause{
- dal.Select("thread_ts, channel_id"),
+ dal.Select("DISTINCT CASE WHEN thread_ts = '' OR thread_ts IS
NULL THEN ts ELSE thread_ts END AS thread_ts, channel_id"),
dal.From("_tool_slack_channel_messages"),
- dal.Where("connection_id=? AND thread_ts!='' AND subtype=''",
data.Options.ConnectionId),
+ dal.Where("connection_id = ? AND channel_id = ? AND reply_count
> 0 AND (subtype = '' OR subtype IS NULL)", data.Options.ConnectionId,
data.Options.ChannelId),
}
// construct the input iterator
@@ -63,11 +64,9 @@ func CollectThread(taskCtx plugin.SubTaskContext)
errors.Error {
pageSize := 50
collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: SlackApiParams{
- ConnectionId: data.Options.ConnectionId,
- },
- Table: RAW_THREAD_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_THREAD_TABLE,
},
ApiClient: data.ApiClient,
Incremental: false,
@@ -118,4 +117,5 @@ var CollectThreadMeta = plugin.SubTaskMeta{
EntryPoint: CollectThread,
EnabledByDefault: true,
Description: "Collect thread from Slack api",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
}
diff --git a/backend/plugins/slack/tasks/thread_extractor.go
b/backend/plugins/slack/tasks/thread_extractor.go
index c9a9c08f0..3e2bb66a7 100644
--- a/backend/plugins/slack/tasks/thread_extractor.go
+++ b/backend/plugins/slack/tasks/thread_extractor.go
@@ -19,6 +19,7 @@ package tasks
import (
"encoding/json"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -32,11 +33,9 @@ func ExtractThread(taskCtx plugin.SubTaskContext)
errors.Error {
data := taskCtx.GetData().(*SlackTaskData)
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: SlackApiParams{
- ConnectionId: data.Options.ConnectionId,
- },
- Table: RAW_THREAD_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_THREAD_TABLE,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
threadInput := &ThreadInput{}
@@ -82,4 +81,5 @@ var ExtractThreadMeta = plugin.SubTaskMeta{
EntryPoint: ExtractThread,
EnabledByDefault: true,
Description: "Extract raw thread messages data into tool layer
table",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS},
}
diff --git a/config-ui/src/plugins/register/index.ts
b/config-ui/src/plugins/register/index.ts
index a68618fde..36f87f701 100644
--- a/config-ui/src/plugins/register/index.ts
+++ b/config-ui/src/plugins/register/index.ts
@@ -35,6 +35,7 @@ import { ZenTaoConfig } from './zentao';
import { OpsgenieConfig } from './opsgenie';
import { TeambitionConfig } from './teambition';
import { TestmoConfig } from './testmo';
+import { SlackConfig } from './slack/config';
export const pluginConfigs: IPluginConfig[] = [
AzureConfig,
@@ -48,6 +49,7 @@ export const pluginConfigs: IPluginConfig[] = [
JenkinsConfig,
JiraConfig,
PagerDutyConfig,
+ SlackConfig,
SonarQubeConfig,
TAPDConfig,
TestmoConfig,
diff --git a/config-ui/src/plugins/register/slack/assets/icon.svg
b/config-ui/src/plugins/register/slack/assets/icon.svg
new file mode 100644
index 000000000..8f153ca73
--- /dev/null
+++ b/config-ui/src/plugins/register/slack/assets/icon.svg
@@ -0,0 +1,22 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<svg width="100" height="100" viewBox="0 0 127 127" fill="#7497F7"
xmlns="http://www.w3.org/2000/svg";>
+ <path d="M27.2 80c0 7.3-5.9 13.2-13.2 13.2C6.7 93.2.8 87.3.8 80c0-7.3
5.9-13.2 13.2-13.2h13.2V80zm6.6 0c0-7.3 5.9-13.2 13.2-13.2 7.3 0 13.2 5.9 13.2
13.2v33c0 7.3-5.9 13.2-13.2 13.2-7.3 0-13.2-5.9-13.2-13.2V80z"/>
+ <path d="M47 27c-7.3 0-13.2-5.9-13.2-13.2C33.8 6.5 39.7.6 47 .6c7.3 0 13.2
5.9 13.2 13.2V27H47zm0 6.7c7.3 0 13.2 5.9 13.2 13.2 0 7.3-5.9 13.2-13.2
13.2H13.9C6.6 60.1.7 54.2.7 46.9c0-7.3 5.9-13.2 13.2-13.2H47z"/>
+ <path d="M99.9 46.9c0-7.3 5.9-13.2 13.2-13.2 7.3 0 13.2 5.9 13.2 13.2 0
7.3-5.9 13.2-13.2 13.2H99.9V46.9zm-6.6 0c0 7.3-5.9 13.2-13.2 13.2-7.3
0-13.2-5.9-13.2-13.2V13.8C66.9 6.5 72.8.6 80.1.6c7.3 0 13.2 5.9 13.2
13.2v33.1z"/>
+ <path d="M80.1 99.8c7.3 0 13.2 5.9 13.2 13.2 0 7.3-5.9 13.2-13.2 13.2-7.3
0-13.2-5.9-13.2-13.2V99.8h13.2zm0-6.6c-7.3 0-13.2-5.9-13.2-13.2 0-7.3 5.9-13.2
13.2-13.2h33.1c7.3 0 13.2 5.9 13.2 13.2 0 7.3-5.9 13.2-13.2 13.2H80.1z"/>
+</svg>
diff --git a/config-ui/src/plugins/register/slack/config.tsx
b/config-ui/src/plugins/register/slack/config.tsx
new file mode 100644
index 000000000..566cad726
--- /dev/null
+++ b/config-ui/src/plugins/register/slack/config.tsx
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { DOC_URL } from '@/release';
+import { IPluginConfig } from '@/types';
+
+import Icon from './assets/icon.svg?react';
+
+export const SlackConfig: IPluginConfig = {
+ plugin: 'slack',
+ name: 'Slack',
+ icon: ({ color }) => <Icon fill={color} />,
+ sort: 12,
+ connection: {
+ docLink: DOC_URL.PLUGIN.SLACK?.BASIS,
+ initialValues: {
+ endpoint: 'https://slack.com/api/',
+ },
+ fields: [
+ 'name',
+ {
+ key: 'endpoint',
+ multipleVersions: {
+ cloud: 'https://slack.com/api/',
+ server: '',
+ },
+ },
+ {
+ key: 'token',
+ label: 'Slack Bot Token',
+ subLabel:
+ 'Create a Slack App with the necessary permissions and use the Bot
User OAuth Token (starts with xoxb-).',
+ },
+ 'proxy',
+ {
+ key: 'rateLimitPerHour',
+ subLabel:
+ 'By default, DevLake uses 3,000 requests/hour for data collection
for Slack. You can adjust the collection speed by setting a custom rate limit.',
+ learnMore: DOC_URL.PLUGIN.SLACK?.RATE_LIMIT,
+ externalInfo: 'Slack’s rate limits vary by method and workspace plan.',
+ defaultValue: 3000,
+ },
+ ],
+ },
+ dataScope: {
+ title: 'Channels',
+ },
+};
diff --git a/config-ui/src/plugins/register/slack/index.ts
b/config-ui/src/plugins/register/slack/index.ts
new file mode 100644
index 000000000..de415db39
--- /dev/null
+++ b/config-ui/src/plugins/register/slack/index.ts
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+export * from './config';
diff --git a/config-ui/src/release/stable.ts b/config-ui/src/release/stable.ts
index ee5d44845..346f309c1 100644
--- a/config-ui/src/release/stable.ts
+++ b/config-ui/src/release/stable.ts
@@ -96,6 +96,10 @@ const URLS = {
BASIS: 'https://devlake.apache.org/docs/Configuration/PagerDuty',
RATE_LIMIT:
'https://devlake.apache.org/docs/Configuration/PagerDuty/#custom-rate-limit-optional',
},
+ SLACK: {
+ BASIS: 'https://devlake.apache.org/docs/Configuration/Slack',
+ RATE_LIMIT:
'https://devlake.apache.org/docs/Configuration/Slack#custom-rate-limit-optional',
+ },
SONARQUBE: {
BASIS: 'https://devlake.apache.org/docs/Configuration/SonarQube',
RATE_LIMIT:
'https://devlake.apache.org/docs/Configuration/SonarQube/#custom-rate-limit-optional',
