This is an automated email from the ASF dual-hosted git repository. enzomartellucci pushed a commit to branch enxdev/feat/datasource-editor in repository https://gitbox.apache.org/repos/asf/superset.git
commit 11c6d7151f988c6f3483cc330ad1fd079521216d Author: Enzo Martellucci <[email protected]> AuthorDate: Thu Dec 18 13:56:24 2025 +0100 feat(datasource-connector): add schema editor for AI-analyzed tables and columns Add a new Datasource Editor panel that allows users to review and edit AI-generated descriptions for database tables and columns. This editor appears after the schema analysis completes in the datasource connector wizard flow. Frontend changes: - Add DatasourceEditorPanel with two-column layout (tree view + detail panel) - Add SchemaTreeView component using antd Tree for schema navigation - Add SchemaDetailPanel for viewing/editing table and column details - Add EditableDescription reusable component for inline editing - Add useSchemaReport hook for fetching schema reports - Add useSchemaEditorMutations hook for updating descriptions - Add centralized config.ts for USE_MOCK_DATA flag - Update types.ts with AnalyzedTable, AnalyzedColumn, SchemaSelection types - Update wizard index.tsx to integrate the editor panel - Export AIInfoBanner from superset-ui-core components Backend changes: - Add PUT /api/v1/datasource_analyzer/table/<id> endpoint - Add PUT /api/v1/datasource_analyzer/column/<id> endpoint - Add POST /api/v1/datasource_analyzer/generate endpoint - Add is_primary_key and is_foreign_key columns to AnalyzedColumn model - Update analyze.py to populate PK/FK flags - Remove redirect logic from datasource_connector.py view --- .../superset-ui-core/src/components/index.ts | 2 + .../components/ConnectorLayout.tsx | 29 ++- .../components/DatasourceEditorPanel.tsx | 222 ++++++++++++++++++ .../components/EditableDescription.tsx | 150 ++++++++++++ .../components/ReviewSchemaPanel.tsx | 6 +- .../components/SchemaDetailPanel.tsx | 244 ++++++++++++++++++++ .../components/SchemaTreeView.tsx | 230 ++++++++++++++++++ .../DatasourceConnector/{types.ts => config.ts} | 30 +-- .../hooks/useSchemaEditorMutations.ts | 162 +++++++++++++ .../DatasourceConnector/hooks/useSchemaReport.ts | 223 ++++++++++++++++++ .../src/pages/DatasourceConnector/index.tsx | 50 +++- .../src/pages/DatasourceConnector/types.ts | 58 ++++- superset/commands/database_analyzer/analyze.py | 38 +-- superset/databases/analyzer_api.py | 256 ++++++++++++++++++++- superset/models/database_analyzer.py | 2 + superset/views/datasource_connector.py | 26 --- 16 files changed, 1646 insertions(+), 82 deletions(-) diff --git a/superset-frontend/packages/superset-ui-core/src/components/index.ts b/superset-frontend/packages/superset-ui-core/src/components/index.ts index 758efa5947..19cc01a8a8 100644 --- a/superset-frontend/packages/superset-ui-core/src/components/index.ts +++ b/superset-frontend/packages/superset-ui-core/src/components/index.ts @@ -149,6 +149,8 @@ export { Progress, type ProgressProps } from './Progress'; export { Skeleton, type SkeletonProps } from './Skeleton'; +export { Spin } from './Spin'; + export { Switch, type SwitchProps } from './Switch'; export { TreeSelect, type TreeSelectProps } from './TreeSelect'; diff --git a/superset-frontend/src/pages/DatasourceConnector/components/ConnectorLayout.tsx b/superset-frontend/src/pages/DatasourceConnector/components/ConnectorLayout.tsx index 59b963d984..7509c09633 100644 --- a/superset-frontend/src/pages/DatasourceConnector/components/ConnectorLayout.tsx +++ b/superset-frontend/src/pages/DatasourceConnector/components/ConnectorLayout.tsx @@ -19,7 +19,12 @@ import { ReactNode } from 'react'; import { t } from '@superset-ui/core'; import { styled, css } from '@apache-superset/core/ui'; -import { AIInfoBanner, Flex, Icons, Typography } from '@superset-ui/core/components'; +import { + AIInfoBanner, + Flex, + Icons, + Typography, +} from '@superset-ui/core/components'; import { ConnectorStep } from '../types'; interface ConnectorLayoutProps { @@ -134,11 +139,29 @@ const stepsConfig: StepConfig[] = [ }, ]; +// Map internal steps to visual step index +// EDIT_SCHEMA is visually part of "Review Schema" step +function getVisualStepIndex(step: ConnectorStep): number { + switch (step) { + case ConnectorStep.CONNECT_DATA_SOURCE: + return 0; + case ConnectorStep.REVIEW_SCHEMA: + case ConnectorStep.EDIT_SCHEMA: + return 1; + case ConnectorStep.GENERATE_DASHBOARD: + return 2; + default: + return 0; + } +} + export default function ConnectorLayout({ currentStep, children, templateName, }: ConnectorLayoutProps) { + const visualStep = getVisualStepIndex(currentStep); + return ( <PageContainer> <PageHeader> @@ -149,8 +172,8 @@ export default function ConnectorLayout({ <StepsContainer> <Flex align="center" gap={0}> {stepsConfig.map((step, index) => { - const isActive = index === currentStep; - const isCompleted = index < currentStep; + const isActive = index === visualStep; + const isCompleted = index < visualStep; return ( <Flex key={step.title} align="center" gap={8}> diff --git a/superset-frontend/src/pages/DatasourceConnector/components/DatasourceEditorPanel.tsx b/superset-frontend/src/pages/DatasourceConnector/components/DatasourceEditorPanel.tsx new file mode 100644 index 0000000000..f23239c32b --- /dev/null +++ b/superset-frontend/src/pages/DatasourceConnector/components/DatasourceEditorPanel.tsx @@ -0,0 +1,222 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { useState, useCallback } from 'react'; +import { t } from '@superset-ui/core'; +import { styled } from '@apache-superset/core/ui'; +import { + AIInfoBanner, + Button, + Flex, + Icons, + Spin, + Typography, +} from '@superset-ui/core/components'; +import useSchemaReport from '../hooks/useSchemaReport'; +import useSchemaEditorMutations from '../hooks/useSchemaEditorMutations'; +import SchemaTreeView from './SchemaTreeView'; +import SchemaDetailPanel from './SchemaDetailPanel'; +import type { SchemaSelection } from '../types'; + +interface DatasourceEditorPanelProps { + reportId: number; + dashboardId: number | null; + onBack: () => void; + onConfirm: (runId: string) => void; +} + +const EditorContainer = styled.div` + ${({ theme }) => ` + width: 100%; + max-width: 1200px; + display: flex; + flex-direction: column; + gap: ${theme.marginLG}px; + `} +`; + +const ContentGrid = styled.div` + ${({ theme }) => ` + display: grid; + grid-template-columns: 1fr 1fr; + gap: ${theme.marginLG}px; + + @media (max-width: 900px) { + grid-template-columns: 1fr; + } + `} +`; + +const FooterActions = styled(Flex)` + ${({ theme }) => ` + padding-top: ${theme.paddingMD}px; + border-top: 1px solid ${theme.colorBorder}; + `} +`; + +const LoadingContainer = styled(Flex)` + ${({ theme }) => ` + padding: ${theme.paddingXL}px; + min-height: 400px; + `} +`; + +const ErrorContainer = styled(Flex)` + ${({ theme }) => ` + padding: ${theme.paddingXL}px; + background-color: ${theme.colorErrorBg}; + border: 1px solid ${theme.colorError}; + border-radius: ${theme.borderRadius}px; + `} +`; + +const AIBannerWrapper = styled.div` + ${({ theme }) => ` + width: 100%; + margin-bottom: ${theme.marginMD}px; + `} +`; + +export default function DatasourceEditorPanel({ + reportId, + dashboardId, + onBack, + onConfirm, +}: DatasourceEditorPanelProps) { + const { report, loading, error, refetch } = useSchemaReport(reportId); + const { + updateTableDescription, + updateColumnDescription, + generateDashboard, + mutationState, + } = useSchemaEditorMutations(); + + const [selection, setSelection] = useState<SchemaSelection>(null); + const [isGenerating, setIsGenerating] = useState(false); + + const handleSelectionChange = useCallback((newSelection: SchemaSelection) => { + setSelection(newSelection); + }, []); + + const handleUpdateTableDescription = useCallback( + async (tableId: number, description: string | null): Promise<boolean> => { + const success = await updateTableDescription(tableId, description); + if (success) { + refetch(); + } + return success; + }, + [updateTableDescription, refetch], + ); + + const handleUpdateColumnDescription = useCallback( + async (columnId: number, description: string | null): Promise<boolean> => { + const success = await updateColumnDescription(columnId, description); + if (success) { + refetch(); + } + return success; + }, + [updateColumnDescription, refetch], + ); + + const handleConfirmAndGenerate = useCallback(async () => { + if (!dashboardId) { + return; + } + + setIsGenerating(true); + const runId = await generateDashboard(reportId, dashboardId); + setIsGenerating(false); + + if (runId) { + onConfirm(runId); + } + }, [reportId, dashboardId, generateDashboard, onConfirm]); + + if (loading) { + return ( + <EditorContainer> + <LoadingContainer vertical align="center" justify="center"> + <Spin size="large" /> + <Typography.Text type="secondary" css={{ marginTop: 16 }}> + {t('Loading schema report...')} + </Typography.Text> + </LoadingContainer> + </EditorContainer> + ); + } + + if (error || !report) { + return ( + <EditorContainer> + <ErrorContainer vertical align="center" gap={16}> + <Icons.ExclamationCircleOutlined iconSize="xl" iconColor="error" /> + <Typography.Text type="danger"> + {error || t('Failed to load schema report')} + </Typography.Text> + <Button onClick={refetch}>{t('Retry')}</Button> + </ErrorContainer> + <FooterActions justify="flex-start"> + <Button onClick={onBack}>{t('Back')}</Button> + </FooterActions> + </EditorContainer> + ); + } + + return ( + <EditorContainer> + <AIBannerWrapper> + <AIInfoBanner + text={t( + 'Review and edit AI-generated descriptions for your tables and columns. These descriptions help improve data understanding and dashboard generation accuracy.', + )} + data-test="schema-editor-ai-hint" + /> + </AIBannerWrapper> + <ContentGrid> + <SchemaTreeView + tables={report.tables} + selection={selection} + onSelectionChange={handleSelectionChange} + schemaName={report.schema_name} + /> + <SchemaDetailPanel + selection={selection} + onUpdateTableDescription={handleUpdateTableDescription} + onUpdateColumnDescription={handleUpdateColumnDescription} + isUpdating={mutationState.loading} + /> + </ContentGrid> + + <FooterActions justify="space-between" align="center"> + <Button onClick={onBack} disabled={isGenerating}> + {t('Back')} + </Button> + <Button + buttonStyle="primary" + onClick={handleConfirmAndGenerate} + loading={isGenerating} + disabled={!dashboardId} + > + {t('Confirm Schema & Generate Dashboard')} + </Button> + </FooterActions> + </EditorContainer> + ); +} diff --git a/superset-frontend/src/pages/DatasourceConnector/components/EditableDescription.tsx b/superset-frontend/src/pages/DatasourceConnector/components/EditableDescription.tsx new file mode 100644 index 0000000000..da1f24fe9c --- /dev/null +++ b/superset-frontend/src/pages/DatasourceConnector/components/EditableDescription.tsx @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { useState, useCallback, useEffect } from 'react'; +import { t } from '@superset-ui/core'; +import { styled } from '@apache-superset/core/ui'; +import { + Button, + Flex, + Icons, + Input, + Typography, +} from '@superset-ui/core/components'; + +const { TextArea } = Input; + +interface EditableDescriptionProps { + description: string | null; + placeholder?: string; + onSave: (description: string | null) => Promise<boolean>; + isUpdating: boolean; +} + +const SectionTitle = styled(Flex)` + ${({ theme }) => ` + margin-bottom: ${theme.marginSM}px; + `} +`; + +const DescriptionBox = styled.div` + ${({ theme }) => ` + background-color: ${theme.colorBgLayout}; + border: 1px solid ${theme.colorBorderSecondary}; + border-radius: ${theme.borderRadiusSM}px; + padding: ${theme.paddingSM}px; + min-height: 80px; + `} +`; + +const EditActions = styled(Flex)` + ${({ theme }) => ` + margin-top: ${theme.marginSM}px; + `} +`; + +export default function EditableDescription({ + description, + placeholder, + onSave, + isUpdating, +}: EditableDescriptionProps) { + const [isEditing, setIsEditing] = useState(false); + const [editedDescription, setEditedDescription] = useState(''); + + // Reset editing state when description changes externally + useEffect(() => { + if (!isEditing) { + setEditedDescription(description || ''); + } + }, [description, isEditing]); + + const handleStartEdit = useCallback(() => { + setEditedDescription(description || ''); + setIsEditing(true); + }, [description]); + + const handleCancelEdit = useCallback(() => { + setIsEditing(false); + setEditedDescription(description || ''); + }, [description]); + + const handleSaveDescription = useCallback(async () => { + const success = await onSave(editedDescription || null); + if (success) { + setIsEditing(false); + } + }, [editedDescription, onSave]); + + return ( + <> + <SectionTitle align="center" justify="space-between"> + <Typography.Text strong>{t('AI-Generated Description')}</Typography.Text> + {!isEditing && ( + <Button + buttonSize="small" + buttonStyle="link" + onClick={handleStartEdit} + icon={<Icons.EditOutlined />} + > + {t('Edit')} + </Button> + )} + </SectionTitle> + + {isEditing ? ( + <> + <TextArea + value={editedDescription} + onChange={e => setEditedDescription(e.target.value)} + rows={4} + placeholder={placeholder || t('Enter a description...')} + /> + <EditActions gap={8} justify="flex-end"> + <Button + buttonSize="small" + buttonStyle="tertiary" + onClick={handleCancelEdit} + disabled={isUpdating} + > + {t('Cancel')} + </Button> + <Button + buttonSize="small" + buttonStyle="primary" + onClick={handleSaveDescription} + loading={isUpdating} + > + {t('Save')} + </Button> + </EditActions> + </> + ) : ( + <DescriptionBox> + <Typography.Text> + {description || ( + <Typography.Text type="secondary" italic> + {t('No description available')} + </Typography.Text> + )} + </Typography.Text> + </DescriptionBox> + )} + </> + ); +} diff --git a/superset-frontend/src/pages/DatasourceConnector/components/ReviewSchemaPanel.tsx b/superset-frontend/src/pages/DatasourceConnector/components/ReviewSchemaPanel.tsx index 421f2a2d56..3423e7cc4e 100644 --- a/superset-frontend/src/pages/DatasourceConnector/components/ReviewSchemaPanel.tsx +++ b/superset-frontend/src/pages/DatasourceConnector/components/ReviewSchemaPanel.tsx @@ -24,7 +24,7 @@ import { Flex, Icons, Typography } from '@superset-ui/core/components'; interface ReviewSchemaPanelProps { databaseName: string | null; schemaName: string | null; - onAnalysisComplete: () => void; + onAnalysisComplete: (reportId: number) => void; } enum AnalysisStep { @@ -241,8 +241,10 @@ export default function ReviewSchemaPanel({ advanceStep(nextStep); } else { // Analysis complete - trigger callback after a short delay + // TODO: In real implementation, get the reportId from the analysis API + // For now, use a placeholder reportId of 1 completionTimeoutId = setTimeout(() => { - onAnalysisComplete(); + onAnalysisComplete(1); }, 1000); } }, stepDurations[step]); diff --git a/superset-frontend/src/pages/DatasourceConnector/components/SchemaDetailPanel.tsx b/superset-frontend/src/pages/DatasourceConnector/components/SchemaDetailPanel.tsx new file mode 100644 index 0000000000..f41c948169 --- /dev/null +++ b/superset-frontend/src/pages/DatasourceConnector/components/SchemaDetailPanel.tsx @@ -0,0 +1,244 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { useCallback } from 'react'; +import { t } from '@superset-ui/core'; +import { styled, useTheme } from '@apache-superset/core/ui'; +import { Flex, Icons, Tag, Typography } from '@superset-ui/core/components'; +import type { SchemaSelection } from '../types'; +import EditableDescription from './EditableDescription'; + +interface SchemaDetailPanelProps { + selection: SchemaSelection; + onUpdateTableDescription: ( + tableId: number, + description: string | null, + ) => Promise<boolean>; + onUpdateColumnDescription: ( + columnId: number, + description: string | null, + ) => Promise<boolean>; + isUpdating: boolean; +} + +const PanelContainer = styled.div` + ${({ theme }) => ` + width: 100%; + background-color: ${theme.colorBgContainer}; + border: 1px solid ${theme.colorBorder}; + border-radius: ${theme.borderRadius}px; + overflow: hidden; + `} +`; + +const HeaderContainer = styled(Flex)` + ${({ theme }) => ` + padding: ${theme.paddingSM}px ${theme.paddingMD}px; + border-bottom: 1px solid ${theme.colorBorder}; + background-color: ${theme.colorBgLayout}; + `} +`; + +const ContentSection = styled.div` + ${({ theme }) => ` + padding: ${theme.paddingMD}px; + `} +`; + +const ItemHeader = styled(Flex)` + ${({ theme }) => ` + padding: ${theme.paddingMD}px; + border-bottom: 1px solid ${theme.colorBorderSecondary}; + `} +`; + +const ConfidenceIndicator = styled(Flex)` + ${({ theme }) => ` + margin-top: ${theme.marginMD}px; + padding: ${theme.paddingSM}px; + background-color: ${theme.colorSuccessBg}; + border-radius: ${theme.borderRadiusSM}px; + `} +`; + +const EmptyState = styled(Flex)` + ${({ theme }) => ` + padding: ${theme.paddingXL}px; + color: ${theme.colorTextSecondary}; + `} +`; + +const TypeLabel = styled(Typography.Text)` + ${({ theme }) => ` + color: ${theme.colorPrimary}; + font-weight: ${theme.fontWeightStrong}; + `} +`; + +export default function SchemaDetailPanel({ + selection, + onUpdateTableDescription, + onUpdateColumnDescription, + isUpdating, +}: SchemaDetailPanelProps) { + const theme = useTheme(); + + const handleSaveTableDescription = useCallback( + async (description: string | null): Promise<boolean> => { + if (selection?.type !== 'table') return false; + return onUpdateTableDescription(selection.table.id, description); + }, + [selection, onUpdateTableDescription], + ); + + const handleSaveColumnDescription = useCallback( + async (description: string | null): Promise<boolean> => { + if (selection?.type !== 'column') return false; + return onUpdateColumnDescription(selection.column.id, description); + }, + [selection, onUpdateColumnDescription], + ); + + // Empty state + if (!selection) { + return ( + <PanelContainer> + <HeaderContainer align="center" gap={8}> + <Icons.InfoCircleOutlined + iconSize="s" + iconColor={theme.colorPrimary} + /> + <Typography.Text strong> + {t('AI Schema Understanding')} + </Typography.Text> + </HeaderContainer> + <EmptyState vertical align="center" justify="center"> + <Icons.TableOutlined + iconSize="xl" + iconColor={theme.colorTextSecondary} + /> + <Typography.Text type="secondary" css={{ marginTop: theme.marginMD }}> + {t('Select a table or column from the schema tree to view details')} + </Typography.Text> + </EmptyState> + </PanelContainer> + ); + } + + // Column detail view + if (selection.type === 'column') { + const { column } = selection; + + return ( + <PanelContainer> + <HeaderContainer align="center" gap={8}> + <Icons.InfoCircleOutlined + iconSize="s" + iconColor={theme.colorPrimary} + /> + <Typography.Text strong> + {t('AI Schema Understanding')} + </Typography.Text> + </HeaderContainer> + + <ItemHeader vertical gap={8}> + <Flex align="center" gap={8}> + <Typography.Title level={5} css={{ margin: 0 }}> + {column.name} + </Typography.Title> + {column.is_primary_key && ( + <Tag color="warning">{t('PRIMARY KEY')}</Tag> + )} + {column.is_foreign_key && ( + <Tag color="processing">{t('FOREIGN KEY')}</Tag> + )} + </Flex> + <Flex align="center" gap={4}> + <Typography.Text type="secondary">{t('Type:')}</Typography.Text> + <TypeLabel>{column.type}</TypeLabel> + </Flex> + </ItemHeader> + + <ContentSection> + <EditableDescription + key={`column-${column.id}`} + description={column.description} + placeholder={t('Enter a description for this column...')} + onSave={handleSaveColumnDescription} + isUpdating={isUpdating} + /> + + <ConfidenceIndicator align="center" gap={8}> + <Icons.CheckCircleOutlined + iconSize="s" + iconColor={theme.colorSuccess} + /> + <Typography.Text css={{ color: theme.colorSuccess }}> + {t('AI Confidence: High')} + </Typography.Text> + </ConfidenceIndicator> + </ContentSection> + </PanelContainer> + ); + } + + // Table detail view + const { table } = selection; + + return ( + <PanelContainer> + <HeaderContainer align="center" gap={8}> + <Icons.InfoCircleOutlined iconSize="s" iconColor={theme.colorPrimary} /> + <Typography.Text strong>{t('AI Schema Understanding')}</Typography.Text> + </HeaderContainer> + + <ItemHeader align="center" gap={12}> + <Icons.CheckSquareOutlined + iconSize="m" + iconColor={theme.colorSuccess} + /> + <Flex vertical gap={2}> + <Typography.Title level={5} css={{ margin: 0 }}> + {table.name} + </Typography.Title> + <Tag color="default">{table.type}</Tag> + </Flex> + </ItemHeader> + + <ContentSection> + <EditableDescription + key={`table-${table.id}`} + description={table.description} + placeholder={t('Enter a description for this table...')} + onSave={handleSaveTableDescription} + isUpdating={isUpdating} + /> + + <ConfidenceIndicator align="center" gap={8}> + <Icons.CheckCircleOutlined + iconSize="s" + iconColor={theme.colorSuccess} + /> + <Typography.Text css={{ color: theme.colorSuccess }}> + {t('AI Confidence: High')} + </Typography.Text> + </ConfidenceIndicator> + </ContentSection> + </PanelContainer> + ); +} diff --git a/superset-frontend/src/pages/DatasourceConnector/components/SchemaTreeView.tsx b/superset-frontend/src/pages/DatasourceConnector/components/SchemaTreeView.tsx new file mode 100644 index 0000000000..7e0dc3cf39 --- /dev/null +++ b/superset-frontend/src/pages/DatasourceConnector/components/SchemaTreeView.tsx @@ -0,0 +1,230 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { useMemo } from 'react'; +import { t } from '@superset-ui/core'; +import { styled, useTheme } from '@apache-superset/core/ui'; +import { Tree } from 'antd'; +import type { DataNode as TreeDataNode } from 'antd/es/tree'; +import { Flex, Icons, Typography } from '@superset-ui/core/components'; +import type { AnalyzedTable, SchemaSelection } from '../types'; + +interface SchemaTreeViewProps { + tables: AnalyzedTable[]; + selection: SchemaSelection; + onSelectionChange: (selection: SchemaSelection) => void; + schemaName: string | null; +} + +const TreeContainer = styled.div` + ${({ theme }) => ` + width: 100%; + background-color: ${theme.colorBgContainer}; + border: 1px solid ${theme.colorBorder}; + border-radius: ${theme.borderRadius}px; + overflow: hidden; + + .ant-tree { + background: transparent; + padding: ${theme.paddingSM}px; + } + + .ant-tree-treenode { + padding: ${theme.paddingXXS}px 0; + } + + .ant-tree-node-content-wrapper { + display: flex; + align-items: center; + width: 100%; + } + + .ant-tree-title { + display: flex; + align-items: center; + gap: ${theme.marginXS}px; + width: 100%; + } + `} +`; + +const HeaderContainer = styled(Flex)` + ${({ theme }) => ` + padding: ${theme.paddingSM}px ${theme.paddingMD}px; + border-bottom: 1px solid ${theme.colorBorder}; + background-color: ${theme.colorBgLayout}; + `} +`; + +const TableIcon = styled.span` + ${({ theme }) => ` + display: flex; + align-items: center; + color: ${theme.colorPrimary}; + `} +`; + +const KeyIcon = styled.span` + ${({ theme }) => ` + display: flex; + align-items: center; + color: ${theme.colorError}; + `} +`; + +const ColumnIcon = styled.span` + ${({ theme }) => ` + display: flex; + align-items: center; + color: ${theme.colorTextSecondary}; + `} +`; + +const ColumnType = styled(Typography.Text)` + ${({ theme }) => ` + margin-left: auto; + padding-left: ${theme.paddingMD}px; + font-family: monospace; + font-size: ${theme.fontSizeSM}px; + `} +`; + +const TreeNodeTitle = styled(Flex)` + width: 100%; +`; + +export default function SchemaTreeView({ + tables, + selection, + onSelectionChange, + schemaName, +}: SchemaTreeViewProps) { + const theme = useTheme(); + + const treeData: TreeDataNode[] = useMemo( + () => + tables.map(table => ({ + key: `table-${table.id}`, + title: ( + <TreeNodeTitle align="center" gap={4}> + <TableIcon> + <Icons.TableOutlined iconSize="s" /> + </TableIcon> + <Typography.Text strong>{table.name}</Typography.Text> + </TreeNodeTitle> + ), + children: table.columns.map(column => ({ + key: `column-${table.id}-${column.id}`, + title: ( + <TreeNodeTitle align="center" justify="space-between"> + <Flex align="center" gap={4}> + {column.is_primary_key || column.is_foreign_key ? ( + <KeyIcon> + <Icons.KeyOutlined iconSize="s" /> + </KeyIcon> + ) : ( + <ColumnIcon> + <Icons.FieldNumberOutlined iconSize="s" /> + </ColumnIcon> + )} + <Typography.Text>{column.name}</Typography.Text> + </Flex> + <ColumnType type="secondary">({column.type})</ColumnType> + </TreeNodeTitle> + ), + isLeaf: true, + })), + })), + [tables], + ); + + const handleSelect = ( + _selectedKeys: React.Key[], + info: { node: TreeDataNode }, + ) => { + const key = String(info.node.key); + + if (key.startsWith('table-')) { + const tableId = parseInt(key.replace('table-', ''), 10); + const table = tables.find(t => t.id === tableId); + if (table) { + onSelectionChange({ type: 'table', table }); + } + } else if (key.startsWith('column-')) { + // Format: column-{tableId}-{columnId} + const parts = key.split('-'); + const tableId = parseInt(parts[1], 10); + const columnId = parseInt(parts[2], 10); + const table = tables.find(t => t.id === tableId); + const column = table?.columns.find(c => c.id === columnId); + if (table && column) { + onSelectionChange({ type: 'column', column, table }); + } + } + }; + + // Compute selected key from selection + const selectedKeys = useMemo(() => { + if (!selection) return []; + if (selection.type === 'table') { + return [`table-${selection.table.id}`]; + } + return [`column-${selection.table.id}-${selection.column.id}`]; + }, [selection]); + + const defaultExpandedKeys = tables.map(table => `table-${table.id}`); + + return ( + <TreeContainer> + <HeaderContainer align="center" gap={8}> + <Icons.DatabaseOutlined iconSize="s" iconColor={theme.colorPrimary} /> + <Typography.Text strong>{t('Database Schema')}</Typography.Text> + </HeaderContainer> + {schemaName && ( + <Flex + align="center" + gap={4} + css={{ + padding: `${theme.paddingXS}px ${theme.paddingMD}px`, + borderBottom: `1px solid ${theme.colorBorderSecondary}`, + }} + > + <Typography.Text type="secondary"> + {t('Connected to:')} + </Typography.Text> + <Typography.Text + css={{ + color: theme.colorPrimary, + fontWeight: theme.fontWeightStrong, + }} + > + {schemaName} + </Typography.Text> + </Flex> + )} + <Tree + treeData={treeData} + selectedKeys={selectedKeys} + defaultExpandedKeys={defaultExpandedKeys} + onSelect={handleSelect} + showIcon={false} + blockNode + /> + </TreeContainer> + ); +} diff --git a/superset-frontend/src/pages/DatasourceConnector/types.ts b/superset-frontend/src/pages/DatasourceConnector/config.ts similarity index 61% copy from superset-frontend/src/pages/DatasourceConnector/types.ts copy to superset-frontend/src/pages/DatasourceConnector/config.ts index e354da0643..ac38ad4ae9 100644 --- a/superset-frontend/src/pages/DatasourceConnector/types.ts +++ b/superset-frontend/src/pages/DatasourceConnector/config.ts @@ -17,28 +17,8 @@ * under the License. */ -export interface DatasourceConnectorState { - databaseId: number | null; - databaseName: string | null; - catalogName: string | null; - schemaName: string | null; - isSubmitting: boolean; -} - -export interface DatasourceAnalyzerPostPayload { - database_id: number; - schema_name: string; - catalog_name?: string | null; -} - -export interface DatasourceAnalyzerResponse { - result: { - run_id: string; - }; -} - -export enum ConnectorStep { - CONNECT_DATA_SOURCE = 0, - REVIEW_SCHEMA = 1, - GENERATE_DASHBOARD = 2, -} +/** + * Set to true to use mock data for UI testing without backend. + * Set to false to use real API endpoints. + */ +export const USE_MOCK_DATA = true; diff --git a/superset-frontend/src/pages/DatasourceConnector/hooks/useSchemaEditorMutations.ts b/superset-frontend/src/pages/DatasourceConnector/hooks/useSchemaEditorMutations.ts new file mode 100644 index 0000000000..61e34e8bfc --- /dev/null +++ b/superset-frontend/src/pages/DatasourceConnector/hooks/useSchemaEditorMutations.ts @@ -0,0 +1,162 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { useCallback, useState } from 'react'; +import { SupersetClient, logging } from '@superset-ui/core'; +import type { GenerateDashboardResponse } from '../types'; +import { USE_MOCK_DATA } from '../config'; + +interface MutationState { + loading: boolean; + error: string | null; +} + +interface UseSchemaEditorMutationsReturn { + updateTableDescription: ( + tableId: number, + description: string | null, + ) => Promise<boolean>; + updateColumnDescription: ( + columnId: number, + description: string | null, + ) => Promise<boolean>; + generateDashboard: ( + reportId: number, + dashboardId: number, + ) => Promise<string | null>; + mutationState: MutationState; +} + +export default function useSchemaEditorMutations(): UseSchemaEditorMutationsReturn { + const [mutationState, setMutationState] = useState<MutationState>({ + loading: false, + error: null, + }); + + const updateTableDescription = useCallback( + async (tableId: number, description: string | null): Promise<boolean> => { + setMutationState({ loading: true, error: null }); + + // Use mock for testing + if (USE_MOCK_DATA) { + await new Promise(resolve => setTimeout(resolve, 300)); + logging.info( + `Mock: Updated table ${tableId} description to:`, + description, + ); + setMutationState({ loading: false, error: null }); + return true; + } + + try { + await SupersetClient.put({ + endpoint: `/api/v1/datasource_analyzer/table/${tableId}`, + jsonPayload: { description }, + }); + setMutationState({ loading: false, error: null }); + return true; + } catch (err) { + logging.error('Error updating table description:', err); + const errorMessage = + err instanceof Error + ? err.message + : 'Failed to update table description'; + setMutationState({ loading: false, error: errorMessage }); + return false; + } + }, + [], + ); + + const updateColumnDescription = useCallback( + async (columnId: number, description: string | null): Promise<boolean> => { + setMutationState({ loading: true, error: null }); + + // Use mock for testing + if (USE_MOCK_DATA) { + await new Promise(resolve => setTimeout(resolve, 300)); + logging.info( + `Mock: Updated column ${columnId} description to:`, + description, + ); + setMutationState({ loading: false, error: null }); + return true; + } + + try { + await SupersetClient.put({ + endpoint: `/api/v1/datasource_analyzer/column/${columnId}`, + jsonPayload: { description }, + }); + setMutationState({ loading: false, error: null }); + return true; + } catch (err) { + logging.error('Error updating column description:', err); + const errorMessage = + err instanceof Error + ? err.message + : 'Failed to update column description'; + setMutationState({ loading: false, error: errorMessage }); + return false; + } + }, + [], + ); + + const generateDashboard = useCallback( + async (reportId: number, dashboardId: number): Promise<string | null> => { + setMutationState({ loading: true, error: null }); + + // Use mock for testing + if (USE_MOCK_DATA) { + await new Promise(resolve => setTimeout(resolve, 500)); + const mockRunId = `mock-run-${Date.now()}`; + logging.info(`Mock: Generated dashboard with run_id: ${mockRunId}`); + setMutationState({ loading: false, error: null }); + return mockRunId; + } + + try { + const response = await SupersetClient.post({ + endpoint: '/api/v1/datasource_analyzer/generate', + jsonPayload: { + report_id: reportId, + dashboard_id: dashboardId, + }, + }); + const data = response.json as GenerateDashboardResponse; + setMutationState({ loading: false, error: null }); + return data.result.run_id; + } catch (err) { + logging.error('Error generating dashboard:', err); + const errorMessage = + err instanceof Error ? err.message : 'Failed to generate dashboard'; + setMutationState({ loading: false, error: errorMessage }); + return null; + } + }, + [], + ); + + return { + updateTableDescription, + updateColumnDescription, + generateDashboard, + mutationState, + }; +} diff --git a/superset-frontend/src/pages/DatasourceConnector/hooks/useSchemaReport.ts b/superset-frontend/src/pages/DatasourceConnector/hooks/useSchemaReport.ts new file mode 100644 index 0000000000..049ac2e351 --- /dev/null +++ b/superset-frontend/src/pages/DatasourceConnector/hooks/useSchemaReport.ts @@ -0,0 +1,223 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { useState, useEffect, useCallback } from 'react'; +import { SupersetClient, logging } from '@superset-ui/core'; +import type { SchemaReportResponse, DatabaseSchemaReport } from '../types'; +import { USE_MOCK_DATA } from '../config'; + +// Mock data for testing UI without backend +const MOCK_REPORT: DatabaseSchemaReport = { + id: 1, + database_id: 1, + schema_name: 'postgres-prod', + status: 'completed', + created_at: new Date().toISOString(), + tables: [ + { + id: 1, + name: 'orders', + type: 'table', + description: + 'This table stores all customer orders including order details, timestamps, and status information.', + columns: [ + { + id: 1, + name: 'order_id', + type: 'INTEGER', + position: 1, + description: 'Unique identifier for each order', + is_primary_key: true, + }, + { + id: 2, + name: 'customer_id', + type: 'INTEGER', + position: 2, + description: 'Reference to the customer who placed the order', + is_foreign_key: true, + }, + { + id: 3, + name: 'order_date', + type: 'TIMESTAMP', + position: 3, + description: null, + }, + { + id: 4, + name: 'total_amount', + type: 'DECIMAL', + position: 4, + description: null, + }, + { + id: 5, + name: 'status', + type: 'VARCHAR', + position: 5, + description: null, + }, + ], + }, + { + id: 2, + name: 'customers', + type: 'table', + description: + 'Customer master data including contact information and account details.', + columns: [ + { + id: 6, + name: 'customer_id', + type: 'INTEGER', + position: 1, + description: 'Unique identifier for each customer', + is_primary_key: true, + }, + { + id: 7, + name: 'email', + type: 'VARCHAR', + position: 2, + description: null, + }, + { + id: 8, + name: 'created_at', + type: 'TIMESTAMP', + position: 3, + description: null, + }, + { + id: 9, + name: 'country', + type: 'VARCHAR', + position: 4, + description: null, + }, + ], + }, + { + id: 3, + name: 'products', + type: 'table', + description: 'Product catalog containing all available items for sale.', + columns: [ + { + id: 10, + name: 'product_id', + type: 'INTEGER', + position: 1, + description: 'Unique identifier for each product', + is_primary_key: true, + }, + { + id: 11, + name: 'name', + type: 'VARCHAR', + position: 2, + description: null, + }, + { + id: 12, + name: 'price', + type: 'DECIMAL', + position: 3, + description: null, + }, + { + id: 13, + name: 'category', + type: 'VARCHAR', + position: 4, + description: null, + }, + ], + }, + ], +}; + +interface UseSchemaReportReturn { + report: DatabaseSchemaReport | null; + loading: boolean; + error: string | null; + refetch: () => void; +} + +export default function useSchemaReport( + reportId: number | null, +): UseSchemaReportReturn { + const [report, setReport] = useState<DatabaseSchemaReport | null>(null); + const [loading, setLoading] = useState<boolean>(false); + const [error, setError] = useState<string | null>(null); + + const fetchReport = useCallback(async () => { + if (!reportId) { + setReport(null); + return; + } + + // Use mock data for testing + if (USE_MOCK_DATA) { + setLoading(true); + // Simulate network delay + await new Promise(resolve => setTimeout(resolve, 500)); + setReport(MOCK_REPORT); + setLoading(false); + return; + } + + setLoading(true); + setError(null); + + try { + const response = await SupersetClient.get({ + endpoint: `/api/v1/datasource_analyzer/report/${reportId}`, + }); + + const data = response.json as SchemaReportResponse; + setReport({ + id: data.id, + database_id: data.database_id, + schema_name: data.schema_name, + status: data.status, + created_at: data.created_at, + tables: data.tables, + }); + } catch (err) { + logging.error('Error fetching schema report:', err); + setError( + err instanceof Error ? err.message : 'Failed to fetch schema report', + ); + } finally { + setLoading(false); + } + }, [reportId]); + + useEffect(() => { + fetchReport(); + }, [fetchReport]); + + return { + report, + loading, + error, + refetch: fetchReport, + }; +} diff --git a/superset-frontend/src/pages/DatasourceConnector/index.tsx b/superset-frontend/src/pages/DatasourceConnector/index.tsx index dad949690f..638e7615e8 100644 --- a/superset-frontend/src/pages/DatasourceConnector/index.tsx +++ b/superset-frontend/src/pages/DatasourceConnector/index.tsx @@ -25,6 +25,7 @@ import type { DatabaseObject } from 'src/components/DatabaseSelector/types'; import DatabaseModal from 'src/features/databases/DatabaseModal'; import ConnectorLayout from './components/ConnectorLayout'; import DataSourcePanel from './components/DataSourcePanel'; +import DatasourceEditorPanel from './components/DatasourceEditorPanel'; import ReviewSchemaPanel from './components/ReviewSchemaPanel'; import useDatabaseListRefresh from './hooks/useDatabaseListRefresh'; import { ConnectorStep, DatasourceConnectorState } from './types'; @@ -57,6 +58,7 @@ export default function DatasourceConnector() { ); const [templateInfo, setTemplateInfo] = useState<TemplateDashboardInfo | null>(null); + const [databaseReportId, setDatabaseReportId] = useState<number | null>(null); // Get dashboard_id from query params const dashboardId = useMemo(() => { @@ -154,7 +156,7 @@ export default function DatasourceConnector() { ); const handleCancel = useCallback(() => { - history.push('/dashboard/templates/'); + history.goBack(); }, [history]); const handleContinueToReview = useCallback(async () => { @@ -176,13 +178,30 @@ export default function DatasourceConnector() { }, [state, addDangerToast, addSuccessToast]); //const handleBackToConnect = useCallback(() => { - // setCurrentStep(ConnectorStep.CONNECT_DATA_SOURCE); + // setCurrentStep(ConnectorStep.CONNECT_DATA_SOURCE); //}, []); - const handleContinueToGenerate = useCallback(() => { - addSuccessToast(t('Moving to Generate Dashboard step')); - setCurrentStep(ConnectorStep.GENERATE_DASHBOARD); - }, [addSuccessToast]); + const handleAnalysisComplete = useCallback( + (reportId: number) => { + setDatabaseReportId(reportId); + addSuccessToast(t('Analysis complete! Review and edit your schema.')); + setCurrentStep(ConnectorStep.EDIT_SCHEMA); + }, + [addSuccessToast], + ); + + const handleBackToReview = useCallback(() => { + setCurrentStep(ConnectorStep.REVIEW_SCHEMA); + }, []); + + const handleConfirmAndGenerate = useCallback( + (runId: string) => { + addSuccessToast(t('Dashboard generation started')); + // Navigate to the loading screen with the run_id + history.push(`/datasource-connector/loading/${runId}`); + }, + [addSuccessToast, history], + ); const renderCurrentStep = () => { switch (currentStep) { @@ -208,12 +227,27 @@ export default function DatasourceConnector() { <ReviewSchemaPanel databaseName={state.databaseName} schemaName={state.schemaName} - onAnalysisComplete={handleContinueToGenerate} + onAnalysisComplete={handleAnalysisComplete} + /> + ); + case ConnectorStep.EDIT_SCHEMA: + return databaseReportId ? ( + <DatasourceEditorPanel + reportId={databaseReportId} + dashboardId={dashboardId} + onBack={handleBackToReview} + onConfirm={handleConfirmAndGenerate} /> + ) : ( + <Flex vertical align="center" css={{ padding: 40 }}> + <Typography.Text type="danger"> + {t('No report ID available. Please restart the process.')} + </Typography.Text> + </Flex> ); case ConnectorStep.GENERATE_DASHBOARD: return ( - <Flex vertical align="center" style={{ padding: '40px' }}> + <Flex vertical align="center" css={{ padding: 40 }}> <Typography.Title level={3}> {t('Generate Dashboard')} </Typography.Title> diff --git a/superset-frontend/src/pages/DatasourceConnector/types.ts b/superset-frontend/src/pages/DatasourceConnector/types.ts index e354da0643..fcaa7cdd26 100644 --- a/superset-frontend/src/pages/DatasourceConnector/types.ts +++ b/superset-frontend/src/pages/DatasourceConnector/types.ts @@ -40,5 +40,61 @@ export interface DatasourceAnalyzerResponse { export enum ConnectorStep { CONNECT_DATA_SOURCE = 0, REVIEW_SCHEMA = 1, - GENERATE_DASHBOARD = 2, + EDIT_SCHEMA = 2, + GENERATE_DASHBOARD = 3, +} + +// Schema Editor Types +export interface AnalyzedColumn { + id: number; + name: string; + type: string; + position: number; + description: string | null; + is_primary_key?: boolean; + is_foreign_key?: boolean; +} + +export interface AnalyzedTable { + id: number; + name: string; + type: 'table' | 'view' | 'materialized_view'; + description: string | null; + columns: AnalyzedColumn[]; +} + +// Selection types for the detail panel +export type SchemaSelection = + | { type: 'table'; table: AnalyzedTable } + | { type: 'column'; column: AnalyzedColumn; table: AnalyzedTable } + | null; + +export interface DatabaseSchemaReport { + id: number; + database_id: number; + schema_name: string; + status: string; + created_at: string | null; + tables: AnalyzedTable[]; +} + +export interface SchemaReportResponse { + id: number; + database_id: number; + schema_name: string; + status: string; + created_at: string | null; + tables: AnalyzedTable[]; + joins: unknown[]; +} + +export interface GenerateDashboardPayload { + report_id: number; + dashboard_id: number; +} + +export interface GenerateDashboardResponse { + result: { + run_id: string; + }; } diff --git a/superset/commands/database_analyzer/analyze.py b/superset/commands/database_analyzer/analyze.py index 67f99eed84..1af5e02d76 100644 --- a/superset/commands/database_analyzer/analyze.py +++ b/superset/commands/database_analyzer/analyze.py @@ -20,7 +20,7 @@ import logging from concurrent.futures import as_completed, ThreadPoolExecutor from typing import Any -from flask import current_app +from flask import current_app, Flask from sqlalchemy import inspect, MetaData, text from superset import db @@ -30,10 +30,8 @@ from superset.models.core import Database from superset.models.database_analyzer import ( AnalyzedColumn, AnalyzedTable, - Cardinality, DatabaseSchemaReport, InferredJoin, - JoinType, TableType, ) from superset.utils import json @@ -178,17 +176,25 @@ class AnalyzeDatabaseSchemaCommand(BaseCommand): result = engine.execute(text(sample_sql)) for row in result: sample_rows.append(dict(row)) - logger.debug("Fetched %d sample rows from %s", len(sample_rows), table_name) - except Exception as e: + logger.debug( + "Fetched %d sample rows from %s", len(sample_rows), table_name + ) + except Exception: # Fallback to regular LIMIT if RANDOM() not supported try: fallback_sql = f'SELECT * FROM "{schema}"."{table_name}" LIMIT 3' # noqa: S608, E501 result = engine.execute(text(fallback_sql)) for row in result: sample_rows.append(dict(row)) - logger.debug("Fetched %d sample rows from %s (fallback)", len(sample_rows), table_name) + logger.debug( + "Fetched %d sample rows from %s (fallback)", + len(sample_rows), + table_name, + ) except Exception as e2: - logger.warning("Could not fetch sample data for %s: %s", table_name, str(e2)) + logger.warning( + "Could not fetch sample data for %s: %s", table_name, str(e2) + ) # Get row count (try reltuples first, fallback to actual count) row_count = None @@ -201,7 +207,7 @@ class AnalyzeDatabaseSchemaCommand(BaseCommand): result = engine.execute(text(count_sql)) row = result.fetchone() row_count = row[0] if row and row[0] >= 0 else None - + # If reltuples is -1 or None, get actual count for small tables if row_count is None or row_count < 0: actual_count_sql = f'SELECT COUNT(*) FROM "{schema}"."{table_name}"' # noqa: S608 @@ -266,12 +272,12 @@ class AnalyzeDatabaseSchemaCommand(BaseCommand): column_name=col_data["name"], data_type=col_data["type"], ordinal_position=col_data["position"], + is_primary_key=col_data["is_primary_key"], + is_foreign_key=col_data["is_foreign_key"], db_comment=col_data["comment"], extra_json=json.dumps( { "is_nullable": col_data["nullable"], - "is_primary_key": col_data["is_primary_key"], - "is_foreign_key": col_data["is_foreign_key"], } ), ) @@ -296,7 +302,7 @@ class AnalyzeDatabaseSchemaCommand(BaseCommand): return max_workers = min(10, len(tables)) - + # Capture the current Flask app context app = current_app._get_current_object() @@ -317,7 +323,7 @@ class AnalyzeDatabaseSchemaCommand(BaseCommand): str(e), ) - def _augment_table_with_ai_context(self, app, table: AnalyzedTable) -> None: + def _augment_table_with_ai_context(self, app: Flask, table: AnalyzedTable) -> None: """Wrapper to provide Flask context to the AI description thread""" with app.app_context(): self._augment_table_with_ai(table) @@ -458,10 +464,10 @@ class AnalyzeDatabaseSchemaCommand(BaseCommand): # Debug logging to see actual data being stored for i, join_data in enumerate(inferred_joins): logger.debug( - "Join %d data: join_type=%s, cardinality=%s", - i, - join_data.get("join_type"), - join_data.get("cardinality") + "Join %d data: join_type=%s, cardinality=%s", + i, + join_data.get("join_type"), + join_data.get("cardinality"), ) for join_data in inferred_joins: diff --git a/superset/databases/analyzer_api.py b/superset/databases/analyzer_api.py index 75e04d2fd2..8e3ee799ae 100644 --- a/superset/databases/analyzer_api.py +++ b/superset/databases/analyzer_api.py @@ -25,7 +25,11 @@ from marshmallow import fields, Schema, ValidationError from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP from superset.extensions import db, event_logger -from superset.models.database_analyzer import DatabaseSchemaReport +from superset.models.database_analyzer import ( + AnalyzedColumn, + AnalyzedTable, + DatabaseSchemaReport, +) from superset.tasks.database_analyzer import ( check_analysis_status, kickstart_analysis, @@ -78,6 +82,48 @@ class CheckStatusResponseSchema(Schema): joins_count = fields.Integer(allow_none=True) +class TableDescriptionPutSchema(Schema): + """Schema for updating table description""" + + description = fields.String( + required=True, + allow_none=True, + metadata={"description": "The AI-generated description for the table"}, + ) + + +class ColumnDescriptionPutSchema(Schema): + """Schema for updating column description""" + + description = fields.String( + required=True, + allow_none=True, + metadata={"description": "The AI-generated description for the column"}, + ) + + +class GenerateDashboardPostSchema(Schema): + """Schema for triggering dashboard generation""" + + report_id = fields.Integer( + required=True, + metadata={"description": "The database schema report ID"}, + ) + dashboard_id = fields.Integer( + required=True, + metadata={"description": "The dashboard template ID to use for generation"}, + ) + + +class GenerateDashboardResponseSchema(Schema): + """Schema for dashboard generation response""" + + run_id = fields.String( + required=True, + metadata={"description": "The unique identifier for this generation run"}, + ) + + class DatasourceAnalyzerRestApi(BaseSupersetApi): """API endpoints for database schema analyzer""" @@ -279,6 +325,8 @@ class DatasourceAnalyzerRestApi(BaseSupersetApi): "type": column.data_type, "position": column.ordinal_position, "description": column.ai_description or column.db_comment, + "is_primary_key": column.is_primary_key, + "is_foreign_key": column.is_foreign_key, } ) @@ -304,3 +352,209 @@ class DatasourceAnalyzerRestApi(BaseSupersetApi): except Exception as e: logger.exception("Error retrieving report") return self.response_500(message=str(e)) + + @expose("/table/<int:table_id>", methods=("PUT",)) + @protect() + @safe + @statsd_metrics + @requires_json + @event_logger.log_this_with_context( + action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.update_table", + log_to_statsd=True, + ) + def update_table(self, table_id: int) -> Response: + """ + Update table description. + --- + put: + summary: Update table AI description + description: >- + Updates the AI-generated description for an analyzed table + parameters: + - in: path + name: table_id + required: true + schema: + type: integer + description: The table ID + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/TableDescriptionPutSchema' + responses: + 200: + description: Table description updated successfully + 400: + $ref: '#/components/responses/400' + 404: + description: Table not found + 500: + description: Internal server error + """ + try: + schema = TableDescriptionPutSchema() + data = schema.load(request.json) + + table = db.session.query(AnalyzedTable).get(table_id) + if not table: + return self.response_404(message="Table not found") + + table.ai_description = data["description"] + db.session.commit() # pylint: disable=consider-using-transaction + + return self.response( + 200, + id=table.id, + name=table.table_name, + description=table.ai_description, + ) + + except ValidationError as error: + return self.response_400(message=str(error.messages)) + except Exception as e: + db.session.rollback() # pylint: disable=consider-using-transaction + logger.exception("Error updating table description") + return self.response_500(message=str(e)) + + @expose("/column/<int:column_id>", methods=("PUT",)) + @protect() + @safe + @statsd_metrics + @requires_json + @event_logger.log_this_with_context( + action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.update_column", + log_to_statsd=True, + ) + def update_column(self, column_id: int) -> Response: + """ + Update column description. + --- + put: + summary: Update column AI description + description: >- + Updates the AI-generated description for an analyzed column + parameters: + - in: path + name: column_id + required: true + schema: + type: integer + description: The column ID + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ColumnDescriptionPutSchema' + responses: + 200: + description: Column description updated successfully + 400: + $ref: '#/components/responses/400' + 404: + description: Column not found + 500: + description: Internal server error + """ + try: + schema = ColumnDescriptionPutSchema() + data = schema.load(request.json) + + column = db.session.query(AnalyzedColumn).get(column_id) + if not column: + return self.response_404(message="Column not found") + + column.ai_description = data["description"] + db.session.commit() # pylint: disable=consider-using-transaction + + return self.response( + 200, + id=column.id, + name=column.column_name, + description=column.ai_description, + ) + + except ValidationError as error: + return self.response_400(message=str(error.messages)) + except Exception as e: + db.session.rollback() # pylint: disable=consider-using-transaction + logger.exception("Error updating column description") + return self.response_500(message=str(e)) + + @expose("/generate", methods=("POST",)) + @protect() + @safe + @statsd_metrics + @requires_json + @event_logger.log_this_with_context( + action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.generate", + log_to_statsd=True, + ) + def generate_dashboard(self) -> Response: + """ + Trigger dashboard generation from schema report. + --- + post: + summary: Generate dashboard from analyzed schema + description: >- + Triggers the dashboard generation Celery job using the analyzed + schema report and a dashboard template. Returns a run_id for + tracking the generation progress. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/GenerateDashboardPostSchema' + responses: + 200: + description: Dashboard generation initiated successfully + content: + application/json: + schema: + type: object + properties: + result: + $ref: '#/components/schemas/GenerateDashboardResponseSchema' + 400: + $ref: '#/components/responses/400' + 404: + description: Report or dashboard not found + 500: + description: Internal server error + """ + try: + schema = GenerateDashboardPostSchema() + data = schema.load(request.json) + + report_id = data["report_id"] + dashboard_id = data["dashboard_id"] + + # Verify report exists + report = db.session.query(DatabaseSchemaReport).get(report_id) + if not report: + return self.response_404(message="Report not found") + + # TODO: Integrate with Dashboard Generation Celery Job + # For now, return a placeholder run_id + # The actual implementation will call the Celery task and return + # its task ID + import uuid + + placeholder_run_id = str(uuid.uuid4()) + + logger.info( + "Dashboard generation requested for report_id=%s, dashboard_id=%s", + report_id, + dashboard_id, + ) + + return self.response(200, result={"run_id": placeholder_run_id}) + + except ValidationError as error: + return self.response_400(message=str(error.messages)) + except Exception as e: + logger.exception("Error initiating dashboard generation") + return self.response_500(message=str(e)) diff --git a/superset/models/database_analyzer.py b/superset/models/database_analyzer.py index a3a1df6724..54fe3f6e03 100644 --- a/superset/models/database_analyzer.py +++ b/superset/models/database_analyzer.py @@ -148,6 +148,8 @@ class AnalyzedColumn(Model, AuditMixinNullable, UUIDMixin): column_name = sa.Column(sa.String(256), nullable=False) data_type = sa.Column(sa.String(256), nullable=False) ordinal_position = sa.Column(sa.Integer, nullable=False) + is_primary_key = sa.Column(sa.Boolean, default=False, nullable=False) + is_foreign_key = sa.Column(sa.Boolean, default=False, nullable=False) db_comment = sa.Column(sa.Text, nullable=True) ai_description = sa.Column(sa.Text, nullable=True) extra_json = sa.Column(sa.Text, nullable=True) diff --git a/superset/views/datasource_connector.py b/superset/views/datasource_connector.py index 4956d817ff..d51fc54f1e 100644 --- a/superset/views/datasource_connector.py +++ b/superset/views/datasource_connector.py @@ -16,12 +16,9 @@ # under the License. """Views for the Datasource Connector feature""" -from flask import redirect, request from flask_appbuilder import expose from flask_appbuilder.security.decorators import has_access, permission_name -from superset import db -from superset.models.dashboard import Dashboard from superset.superset_typing import FlaskResponse from superset.views.base import BaseSupersetView @@ -30,33 +27,10 @@ class DatasourceConnectorView(BaseSupersetView): route_base = "/datasource-connector" class_permission_name = "Dashboard" - def _is_valid_template_dashboard(self, dashboard_id: str | None) -> bool: - """Check if the dashboard_id is valid and refers to a template dashboard.""" - if not dashboard_id: - return False - - try: - dashboard_id_int = int(dashboard_id) - except (ValueError, TypeError): - return False - - dashboard = db.session.query(Dashboard).filter_by(id=dashboard_id_int).first() - if not dashboard: - return False - - # Check if the dashboard is a template (is_template in json_metadata) - metadata = dashboard.params_dict - return metadata.get("is_template", False) - @expose("/") @has_access @permission_name("read") def root(self) -> FlaskResponse: - dashboard_id = request.args.get("dashboard_id") - - if not self._is_valid_template_dashboard(dashboard_id): - return redirect("/dashboard/templates/") - return super().render_app_template() @expose("/loading/<run_id>")
