This is an automated email from the ASF dual-hosted git repository. vavila pushed a commit to branch chore/cache-slack-channels-list in repository https://gitbox.apache.org/repos/asf/superset.git
commit fdbfb193f05cafb161c18f84e9cfcf404c3b6a55 Author: Vitor Avila <[email protected]> AuthorDate: Thu Mar 6 12:11:43 2025 -0300 chore: Caching the Slack channels list --- .../alerts/components/NotificationMethod.tsx | 10 ++ superset/reports/api.py | 6 +- superset/utils/slack.py | 104 +++++++++++++-------- 3 files changed, 79 insertions(+), 41 deletions(-) diff --git a/superset-frontend/src/features/alerts/components/NotificationMethod.tsx b/superset-frontend/src/features/alerts/components/NotificationMethod.tsx index 60797929d1..aa6e075eef 100644 --- a/superset-frontend/src/features/alerts/components/NotificationMethod.tsx +++ b/superset-frontend/src/features/alerts/components/NotificationMethod.tsx @@ -36,6 +36,7 @@ import { } from '@superset-ui/core'; import { Select } from 'src/components'; import Icons from 'src/components/Icons'; +import { useToasts } from 'src/components/MessageToasts/withToasts'; import RefreshLabel from 'src/components/RefreshLabel'; import { NotificationMethodOption, @@ -225,6 +226,7 @@ export const NotificationMethod: FunctionComponent<NotificationMethodProps> = ({ ]); const [useSlackV1, setUseSlackV1] = useState<boolean>(false); + const { addInfoToast, addSuccessToast } = useToasts(); const onMethodChange = (selected: { label: string; @@ -274,6 +276,11 @@ export const NotificationMethod: FunctionComponent<NotificationMethodProps> = ({ }: { force?: boolean | undefined; } = {}) => { + if (force) { + addInfoToast( + t('Fetching Slack channels. This operation may take a while.'), + ); + } fetchSlackChannels({ types: ['public_channel', 'private_channel'], force }) .then(({ json }) => { const { result } = json; @@ -311,6 +318,9 @@ export const NotificationMethod: FunctionComponent<NotificationMethodProps> = ({ }) .finally(() => { setMethodOptionsLoading(false); + if (force) { + addSuccessToast(t('List of Slack channels updated')); + } }); }; diff --git a/superset/reports/api.py b/superset/reports/api.py index 320eb97c05..a0ebabb202 100644 --- a/superset/reports/api.py +++ b/superset/reports/api.py @@ -577,8 +577,12 @@ class ReportScheduleRestApi(BaseSupersetModelRestApi): search_string = params.get("search_string") types = params.get("types", []) exact_match = params.get("exact_match", False) + force = params.get("force", False) channels = get_channels_with_search( - search_string=search_string, types=types, exact_match=exact_match + search_string=search_string, + types=types, + exact_match=exact_match, + force=force, ) return self.response(200, result=channels) except SupersetException as ex: diff --git a/superset/utils/slack.py b/superset/utils/slack.py index 6d51f2765e..8125a3ac40 100644 --- a/superset/utils/slack.py +++ b/superset/utils/slack.py @@ -17,7 +17,7 @@ import logging -from typing import Optional +from typing import Any, Optional from flask import current_app from slack_sdk import WebClient @@ -26,7 +26,9 @@ from slack_sdk.http_retry.builtin_handlers import RateLimitErrorRetryHandler from superset import feature_flag_manager from superset.exceptions import SupersetException +from superset.extensions import cache_manager from superset.reports.schemas import SlackChannelSchema +from superset.utils import cache as cache_util from superset.utils.backports import StrEnum from superset.utils.core import recipients_string_to_list @@ -54,60 +56,82 @@ def get_slack_client() -> WebClient: return client +@cache_util.memoized_func( + key="slack_conversations_list", + cache=cache_manager.cache, +) +def get_channels(limit: int, extra_params: dict[str, Any]) -> list[SlackChannelSchema]: + """ + Retrieves a list of all conversations accessible by the bot + from the Slack API, and caches results (to avoid rate limits). + + The Slack API does not provide search so to apply a search use + get_channels_with_search instead. + """ + client = get_slack_client() + channel_schema = SlackChannelSchema() + channels: list[SlackChannelSchema] = [] + cursor = None + + while True: + response = client.conversations_list( + limit=limit, cursor=cursor, exclude_archived=True, **extra_params + ) + channels.extend( + channel_schema.load(channel) for channel in response.data["channels"] + ) + cursor = response.data.get("response_metadata", {}).get("next_cursor") + if not cursor: + break + + return channels + + def get_channels_with_search( search_string: str = "", limit: int = 999, types: Optional[list[SlackChannelTypes]] = None, exact_match: bool = False, + force: bool = False, ) -> list[SlackChannelSchema]: """ The slack api is paginated but does not include search, so we need to fetch all channels and filter them ourselves This will search by slack name or id """ - + extra_params = {} + extra_params["types"] = ",".join(types) if types else None try: - client = get_slack_client() - channel_schema = SlackChannelSchema() - channels: list[SlackChannelSchema] = [] - cursor = None - extra_params = {} - extra_params["types"] = ",".join(types) if types else None - - while True: - response = client.conversations_list( - limit=limit, cursor=cursor, exclude_archived=True, **extra_params - ) - channels.extend( - channel_schema.load(channel) for channel in response.data["channels"] - ) - cursor = response.data.get("response_metadata", {}).get("next_cursor") - if not cursor: - break - - # The search string can be multiple channels separated by commas - if search_string: - search_array = recipients_string_to_list(search_string) - channels = [ - channel - for channel in channels - if any( - ( - search.lower() == channel["name"].lower() - or search.lower() == channel["id"].lower() - if exact_match - else ( - search.lower() in channel["name"].lower() - or search.lower() in channel["id"].lower() - ) - ) - for search in search_array - ) - ] - return channels + channels = get_channels( + limit=limit, + extra_params=extra_params, + force=force, + cache_timeout=86400, + ) except (SlackClientError, SlackApiError) as ex: raise SupersetException(f"Failed to list channels: {ex}") from ex + # The search string can be multiple channels separated by commas + if search_string: + search_array = recipients_string_to_list(search_string) + channels = [ + channel + for channel in channels + if any( + ( + search.lower() == channel["name"].lower() + or search.lower() == channel["id"].lower() + if exact_match + else ( + search.lower() in channel["name"].lower() + or search.lower() in channel["id"].lower() + ) + ) + for search in search_array + ) + ] + return channels + def should_use_v2_api() -> bool: if not feature_flag_manager.is_feature_enabled("ALERT_REPORT_SLACK_V2"):
