diff --git a/superset-frontend/src/components/Datasource/components/DatasourceEditor/DatasourceEditor.tsx b/superset-frontend/src/components/Datasource/components/DatasourceEditor/DatasourceEditor.tsx index 8bc2b6cf8f39..0bf47f57c783 100644 --- a/superset-frontend/src/components/Datasource/components/DatasourceEditor/DatasourceEditor.tsx +++ b/superset-frontend/src/components/Datasource/components/DatasourceEditor/DatasourceEditor.tsx @@ -89,6 +89,8 @@ import { import { DatabaseSelector } from '../../../DatabaseSelector'; import CollectionTable from '../CollectionTable'; import Fieldset from '../Fieldset'; +import { useDatasetLineage } from 'src/hooks/apiResources'; +import { LineageView } from 'src/features/lineage'; import Field from '../Field'; import { fetchSyncedColumns, updateColumns } from '../../utils'; import DatasetUsageTab from './components/DatasetUsageTab'; @@ -425,6 +427,16 @@ const StyledTableTabWrapper = styled.div` } `; +// Functional wrapper for the lineage tab, since hooks can't be used directly in +// the DatasourceEditor class component. +function DatasetLineageTab({ datasourceId }: { datasourceId?: number }) { + const lineageResource = useDatasetLineage(datasourceId ?? 0); + if (!datasourceId) { + return ; + } + return ; +} + const DefaultColumnSettingsContainer = styled.div` ${({ theme }) => css` margin-bottom: ${theme.sizeUnit * 4}px; @@ -477,6 +489,7 @@ const TABS_KEYS = { COLUMNS: 'COLUMNS', CALCULATED_COLUMNS: 'CALCULATED_COLUMNS', USAGE: 'USAGE', + LINEAGE: 'LINEAGE', FOLDERS: 'FOLDERS', SETTINGS: 'SETTINGS', SPATIAL: 'SPATIAL', @@ -2515,6 +2528,15 @@ class DatasourceEditor extends PureComponent< ), }, + { + key: TABS_KEYS.LINEAGE, + label: t('Lineage'), + children: ( + + + + ), + }, ...(isFeatureEnabled(FeatureFlag.DatasetFolders) ? [ { diff --git a/superset-frontend/src/dashboard/components/Header/useHeaderActionsDropdownMenu.tsx b/superset-frontend/src/dashboard/components/Header/useHeaderActionsDropdownMenu.tsx index 102a0a705816..ba2d80d298ce 100644 --- a/superset-frontend/src/dashboard/components/Header/useHeaderActionsDropdownMenu.tsx +++ b/superset-frontend/src/dashboard/components/Header/useHeaderActionsDropdownMenu.tsx @@ -37,6 +37,7 @@ import { getUrlParam } from 'src/utils/urlUtils'; import { MenuKeys, RootState } from 'src/dashboard/types'; import { HeaderDropdownProps } from 'src/dashboard/components/Header/types'; import { usePermissions } from 'src/hooks/usePermissions'; +import { LineageModal } from 'src/features/lineage'; export const useHeaderActionsMenu = ({ customCss, @@ -234,6 +235,23 @@ export const useHeaderActionsMenu = ({ }); } + // View lineage (available in both view and edit mode; lineage is + // read-only information about the dashboard's upstream assets) + if (dashboardId) { + menuItems.push( + createModalMenuItem( + MenuKeys.ViewLineage, + {t('View lineage')} + } + />, + ), + ); + } + // Edit properties if (editMode) { menuItems.push({ diff --git a/superset-frontend/src/dashboard/types.ts b/superset-frontend/src/dashboard/types.ts index 2a7056bebab3..fd0ea611f905 100644 --- a/superset-frontend/src/dashboard/types.ts +++ b/superset-frontend/src/dashboard/types.ts @@ -393,4 +393,5 @@ export enum MenuKeys { ManageEmailReports = 'manage_email_reports', ExportPivotXlsx = 'export_pivot_xlsx', EmbedCode = 'embed_code', + ViewLineage = 'view_lineage', } diff --git a/superset-frontend/src/explore/components/useExploreAdditionalActionsMenu/index.tsx b/superset-frontend/src/explore/components/useExploreAdditionalActionsMenu/index.tsx index 6b9e7b048de4..9f441c0d6de0 100644 --- a/superset-frontend/src/explore/components/useExploreAdditionalActionsMenu/index.tsx +++ b/superset-frontend/src/explore/components/useExploreAdditionalActionsMenu/index.tsx @@ -72,6 +72,7 @@ import { ReportObject } from 'src/features/reports/types'; import ViewQueryModal from '../controls/ViewQueryModal'; import EmbedCodeContent from '../EmbedCodeContent'; import { useDashboardsMenuItems } from './DashboardsSubMenu'; +import { LineageModal } from 'src/features/lineage'; export const SEARCH_THRESHOLD = 10; @@ -102,6 +103,7 @@ const MENU_KEYS = { EDIT_REPORT: 'edit_report', DELETE_REPORT: 'delete_report', VIEW_QUERY: 'view_query', + VIEW_LINEAGE: 'view_lineage', RUN_IN_SQL_LAB: 'run_in_sql_lab', }; @@ -1028,6 +1030,23 @@ export const useExploreAdditionalActionsMenu = ( onClick: () => setIsDropdownVisible(false), }); + // View lineage + if (slice?.slice_id) { + menuItems.push({ + key: MENU_KEYS.VIEW_LINEAGE, + label: ( + {t('View lineage')} + } + /> + ), + onClick: () => setIsDropdownVisible(false), + }); + } + // Run in SQL Lab if (datasource) { menuItems.push({ diff --git a/superset-frontend/src/features/charts/ChartCard.tsx b/superset-frontend/src/features/charts/ChartCard.tsx index 47562c84450f..2ef82e5df189 100644 --- a/superset-frontend/src/features/charts/ChartCard.tsx +++ b/superset-frontend/src/features/charts/ChartCard.tsx @@ -36,6 +36,7 @@ import { handleChartDelete, CardStyles } from 'src/views/CRUD/utils'; import { assetUrl } from 'src/utils/assetUrl'; import type { ListViewFetchDataConfig as FetchDataConfig } from 'src/components'; import { TableTab } from 'src/views/CRUD/types'; +import { LineageModal } from 'src/features/lineage'; interface ChartCardProps { chart: Chart; @@ -76,6 +77,7 @@ export default function ChartCard({ const canEdit = hasPerm('can_write'); const canDelete = hasPerm('can_write'); const canExport = hasPerm('can_export'); + const canRead = hasPerm('can_read'); const menuItems: MenuItem[] = []; if (canEdit) { @@ -100,6 +102,29 @@ export default function ChartCard({ }); } + if (canRead) { + menuItems.push({ + key: 'lineage', + label: ( + + {' '} + {t('View Lineage')} + + } + /> + ), + }); + } + if (canExport) { menuItems.push({ key: 'export', @@ -167,54 +192,59 @@ export default function ChartCard({ } return ( - { - if (!bulkSelectEnabled && chart.url) { - history.push(chart.url); - } - }} - > - - ) : null - } - url={bulkSelectEnabled ? undefined : chart.url} - imgURL={chart.thumbnail_url || ''} - imgFallbackURL={assetUrl( - '/static/assets/images/chart-card-fallback.svg', - )} - description={t('Modified %s', chart.changed_on_delta_humanized)} - coverLeft={} - coverRight={} - linkComponent={Link} - actions={ - { - e.stopPropagation(); - e.preventDefault(); - }} - > - {userId && ( - - )} - - - - - } - /> - + <> + { + if (!bulkSelectEnabled && chart.url) { + history.push(chart.url); + } + }} + > + + ) : null + } + url={bulkSelectEnabled ? undefined : chart.url} + imgURL={chart.thumbnail_url || ''} + imgFallbackURL={assetUrl( + '/static/assets/images/chart-card-fallback.svg', + )} + description={t('Modified %s', chart.changed_on_delta_humanized)} + coverLeft={} + coverRight={} + linkComponent={Link} + actions={ + { + e.stopPropagation(); + e.preventDefault(); + }} + > + {userId && ( + + )} + + + + + } + /> + + ); } diff --git a/superset-frontend/src/features/dashboards/DashboardCard.tsx b/superset-frontend/src/features/dashboards/DashboardCard.tsx index 5793e0e1bc0b..6b57e3019832 100644 --- a/superset-frontend/src/features/dashboards/DashboardCard.tsx +++ b/superset-frontend/src/features/dashboards/DashboardCard.tsx @@ -37,6 +37,7 @@ import { Icons } from '@superset-ui/core/components/Icons'; import { Dashboard } from 'src/views/CRUD/types'; import { assetUrl } from 'src/utils/assetUrl'; import { FacePile } from 'src/components'; +import { LineageModal } from 'src/features/lineage'; interface DashboardCardProps { isChart?: boolean; @@ -69,6 +70,7 @@ function DashboardCard({ const canEdit = hasPerm('can_write'); const canDelete = hasPerm('can_write'); const canExport = hasPerm('can_export'); + const canRead = hasPerm('can_read'); const [thumbnailUrl, setThumbnailUrl] = useState(null); const [fetchingThumbnail, setFetchingThumbnail] = useState(false); @@ -99,6 +101,23 @@ function DashboardCard({ const menuItems: MenuItem[] = []; + if (canRead) { + menuItems.push({ + key: 'lineage', + label: ( + + {t('View Lineage')} + + } + /> + ), + }); + } + if (canEdit && openDashboardEditModal) { menuItems.push({ key: 'edit', @@ -151,55 +170,60 @@ function DashboardCard({ } return ( - { - if (!bulkSelectEnabled) { - history.push(dashboard.url); - } - }} - > - } - cover={ - !isFeatureEnabled(FeatureFlag.Thumbnails) || !showThumbnails ? ( - <> - ) : null - } - url={bulkSelectEnabled ? undefined : dashboard.url} - linkComponent={Link} - imgURL={thumbnailUrl} - imgFallbackURL={assetUrl( - '/static/assets/images/dashboard-card-fallback.svg', - )} - description={t('Modified %s', dashboard.changed_on_delta_humanized)} - coverLeft={} - actions={ - { - e.stopPropagation(); - e.preventDefault(); - }} - > - {userId && ( - - )} - - - - - } - /> - + <> + { + if (!bulkSelectEnabled) { + history.push(dashboard.url); + } + }} + > + } + cover={ + !isFeatureEnabled(FeatureFlag.Thumbnails) || !showThumbnails ? ( + <> + ) : null + } + url={bulkSelectEnabled ? undefined : dashboard.url} + linkComponent={Link} + imgURL={thumbnailUrl} + imgFallbackURL={assetUrl( + '/static/assets/images/dashboard-card-fallback.svg', + )} + description={t('Modified %s', dashboard.changed_on_delta_humanized)} + coverLeft={} + actions={ + { + e.stopPropagation(); + e.preventDefault(); + }} + > + {userId && ( + + )} + + + + + } + /> + + ); } diff --git a/superset-frontend/src/features/datasets/AddDataset/EditDataset/index.tsx b/superset-frontend/src/features/datasets/AddDataset/EditDataset/index.tsx index e0438cee0bd6..d816b2b1450d 100644 --- a/superset-frontend/src/features/datasets/AddDataset/EditDataset/index.tsx +++ b/superset-frontend/src/features/datasets/AddDataset/EditDataset/index.tsx @@ -16,11 +16,14 @@ * specific language governing permissions and limitations * under the License. */ +import { useState } from 'react'; import { t } from '@apache-superset/core/translation'; import { styled } from '@apache-superset/core/theme'; import useGetDatasetRelatedCounts from 'src/features/datasets/hooks/useGetDatasetRelatedCounts'; import { Badge } from '@superset-ui/core/components'; import Tabs from '@superset-ui/core/components/Tabs'; +import { useDatasetLineage } from 'src/hooks/apiResources'; +import { LineageView } from 'src/features/lineage'; const StyledTabs = styled(Tabs)` ${({ theme }) => ` @@ -51,16 +54,25 @@ const TRANSLATIONS = { USAGE_TEXT: t('Usage'), COLUMNS_TEXT: t('Columns'), METRICS_TEXT: t('Metrics'), + LINEAGE_TEXT: t('Lineage'), }; const TABS_KEYS = { COLUMNS: 'COLUMNS', METRICS: 'METRICS', USAGE: 'USAGE', + LINEAGE: 'LINEAGE', }; const EditPage = ({ id }: EditPageProps) => { const { usageCount } = useGetDatasetRelatedCounts(id); + const [activeKey, setActiveKey] = useState(TABS_KEYS.COLUMNS); + // Only fetch lineage once the user opens the Lineage tab to avoid + // unnecessary requests/backend load on page load. + const lineageResource = useDatasetLineage( + id, + activeKey !== TABS_KEYS.LINEAGE, + ); const usageTab = ( @@ -85,9 +97,23 @@ const EditPage = ({ id }: EditPageProps) => { label: usageTab, children: null, }, + { + key: TABS_KEYS.LINEAGE, + label: TRANSLATIONS.LINEAGE_TEXT, + children: ( + + ), + }, ]; - return ; + return ( + + ); }; export default EditPage; diff --git a/superset-frontend/src/features/lineage/LineageModal.tsx b/superset-frontend/src/features/lineage/LineageModal.tsx new file mode 100644 index 000000000000..77f820523e12 --- /dev/null +++ b/superset-frontend/src/features/lineage/LineageModal.tsx @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { FC, ReactNode } from 'react'; +import { t } from '@apache-superset/core/translation'; +import { ModalTrigger } from '@superset-ui/core/components'; +import { + useChartLineage, + useDashboardLineage, + useDatasetLineage, +} from 'src/hooks/apiResources'; +import LineageView from './LineageView'; + +export interface LineageModalProps { + entityType: 'dataset' | 'chart' | 'dashboard'; + entityId: string | number; + triggerNode: ReactNode; +} + +const LineageModal: FC = ({ + entityType, + entityId, + triggerNode, +}) => { + const datasetLineage = useDatasetLineage( + entityType === 'dataset' ? entityId : '', + ); + const chartLineage = useChartLineage(entityType === 'chart' ? entityId : ''); + const dashboardLineage = useDashboardLineage( + entityType === 'dashboard' ? entityId : '', + ); + + const lineageResource = + entityType === 'dataset' + ? datasetLineage + : entityType === 'chart' + ? chartLineage + : dashboardLineage; + + const title = + entityType === 'dataset' + ? t('Dataset Lineage') + : entityType === 'chart' + ? t('Chart Lineage') + : t('Dashboard Lineage'); + + return ( + + } + width="850px" + responsive + destroyOnHidden + /> + ); +}; + +export default LineageModal; diff --git a/superset-frontend/src/features/lineage/LineageView.tsx b/superset-frontend/src/features/lineage/LineageView.tsx new file mode 100644 index 000000000000..90b785d7b277 --- /dev/null +++ b/superset-frontend/src/features/lineage/LineageView.tsx @@ -0,0 +1,728 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { FC, useMemo, useState, useCallback } from 'react'; +import { t } from '@apache-superset/core/translation'; +import { styled, useTheme } from '@apache-superset/core/theme'; +import { Empty, Loading } from '@superset-ui/core/components'; +import { Button } from '@superset-ui/core/components'; +import { ResourceStatus } from 'src/hooks/apiResources/apiResources'; +import type { Resource } from 'src/hooks/apiResources/apiResources'; +import type { + DatasetLineage, + ChartLineage, + DashboardLineage, + ChartEntity, + DashboardEntity, + DatasetEntity, + DatabaseEntity, +} from 'src/hooks/apiResources/lineage'; +import Echart from '../../../plugins/plugin-chart-echarts/src/components/Echart'; +import type { EChartsCoreOption } from 'echarts/core'; + +const LineageContainer = styled.div` + display: flex; + flex-direction: column; + width: 100%; + height: 100%; +`; + +const Legend = styled.div` + ${({ theme }) => ` + display: flex; + justify-content: center; + align-items: center; + gap: ${theme.sizeUnit * 4}px; + padding: ${theme.sizeUnit * 3}px; + background-color: ${theme.colorBgLayout}; + border-bottom: 1px solid ${theme.colorBorder}; + `} +`; + +const LegendItem = styled.div<{ color: string }>` + ${({ theme, color }) => ` + display: flex; + align-items: center; + gap: ${theme.sizeUnit * 2}px; + font-size: ${theme.fontSizeSM}px; + color: ${theme.colorText}; + + &::before { + content: ''; + width: 12px; + height: 12px; + border-radius: 2px; + background-color: ${color}; + } + `} +`; + +const DetailsPanel = styled.div` + ${({ theme }) => ` + padding: ${theme.sizeUnit * 4}px; + background-color: ${theme.colorBgLayout}; + border-top: 1px solid ${theme.colorBorder}; + min-height: 120px; + `} +`; + +const DetailsPanelHeader = styled.div` + ${({ theme }) => ` + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: ${theme.sizeUnit * 3}px; + `} +`; + +const DetailsPanelActions = styled.div` + ${({ theme }) => ` + display: flex; + gap: ${theme.sizeUnit * 2}px; + `} +`; + +const DetailsPanelTitle = styled.h4` + ${({ theme }) => ` + margin: 0; + font-size: ${theme.fontSizeLG}px; + font-weight: ${theme.fontWeightStrong}; + color: ${theme.colorText}; + `} +`; + +const DetailsPanelContent = styled.div` + ${({ theme }) => ` + display: flex; + flex-direction: column; + gap: ${theme.sizeUnit * 2}px; + `} +`; + +const DetailRow = styled.div` + ${({ theme }) => ` + display: flex; + gap: ${theme.sizeUnit * 2}px; + font-size: ${theme.fontSizeSM}px; + color: ${theme.colorText}; + `} +`; + +const DetailLabel = styled.span` + ${({ theme }) => ` + font-weight: ${theme.fontWeightStrong}; + min-width: 100px; + `} +`; + +const DetailValue = styled.span` + ${({ theme }) => ` + color: ${theme.colorTextSecondary}; + `} +`; + +type NodeType = 'database' | 'dataset' | 'chart' | 'dashboard'; + +type NodeDetails = { + name: string; + type: NodeType; + id?: number; + additionalInfo?: Record; +}; + +// Build a stable, unique graph identity for a node so that entities sharing the +// same display name (e.g. two charts with identical titles) never collapse into +// a single Sankey node. The human-readable name is kept separately as the label. +const nodeKey = (type: NodeType, id?: number, name?: string): string => + id != null ? `${type}:${id}` : `${type}:${name ?? ''}`; + +type LineageViewProps = { + lineageResource: + | Resource + | Resource + | Resource; + entityType: 'dataset' | 'chart' | 'dashboard'; +}; + +const LineageView: FC = ({ lineageResource, entityType }) => { + const theme = useTheme(); + const [selectedNode, setSelectedNode] = useState(null); + + // Create a mapping of node names to their details + const nodeDetailsMap = useMemo(() => { + if ( + lineageResource.status !== ResourceStatus.Complete || + !lineageResource.result + ) { + return new Map(); + } + + const data = lineageResource.result; + const map = new Map(); + + if (entityType === 'dataset' && 'dataset' in data) { + const { dataset, upstream, downstream } = data as DatasetLineage; + + // Add current dataset + map.set(nodeKey('dataset', dataset.id, dataset.name), { + name: dataset.name, + type: 'dataset', + id: dataset.id, + additionalInfo: { + schema: dataset.schema, + table_name: dataset.table_name, + database_name: dataset.database_name, + }, + }); + + // Add upstream database + if (upstream?.database) { + map.set( + nodeKey( + 'database', + upstream.database.id, + upstream.database.database_name, + ), + { + name: upstream.database.database_name, + type: 'database', + id: upstream.database.id, + }, + ); + } + + // Add downstream charts + if (downstream?.charts?.result) { + downstream.charts.result.forEach((chart: ChartEntity) => { + map.set(nodeKey('chart', chart.id, chart.slice_name), { + name: chart.slice_name, + type: 'chart', + id: chart.id, + additionalInfo: { + viz_type: chart.viz_type, + }, + }); + }); + } + + // Add downstream dashboards + if (downstream?.dashboards?.result) { + downstream.dashboards.result.forEach((dashboard: DashboardEntity) => { + map.set(nodeKey('dashboard', dashboard.id, dashboard.title), { + name: dashboard.title, + type: 'dashboard', + id: dashboard.id, + additionalInfo: { + slug: dashboard.slug, + }, + }); + }); + } + } else if (entityType === 'chart' && 'chart' in data) { + const { chart, upstream, downstream } = data as ChartLineage; + + // Add current chart + map.set(nodeKey('chart', chart.id, chart.slice_name), { + name: chart.slice_name, + type: 'chart', + id: chart.id, + additionalInfo: { + viz_type: chart.viz_type, + }, + }); + + // Add upstream dataset + if (upstream?.dataset) { + map.set( + nodeKey('dataset', upstream.dataset.id, upstream.dataset.name), + { + name: upstream.dataset.name, + type: 'dataset', + id: upstream.dataset.id, + additionalInfo: { + schema: upstream.dataset.schema, + table_name: upstream.dataset.table_name, + }, + }, + ); + } + + // Add upstream database + if (upstream?.database) { + map.set( + nodeKey( + 'database', + upstream.database.id, + upstream.database.database_name, + ), + { + name: upstream.database.database_name, + type: 'database', + id: upstream.database.id, + }, + ); + } + + // Add downstream dashboards + if (downstream?.dashboards?.result) { + downstream.dashboards.result.forEach((dashboard: DashboardEntity) => { + map.set(nodeKey('dashboard', dashboard.id, dashboard.title), { + name: dashboard.title, + type: 'dashboard', + id: dashboard.id, + additionalInfo: { + slug: dashboard.slug, + }, + }); + }); + } + } else if (entityType === 'dashboard' && 'dashboard' in data) { + const { dashboard, upstream } = data as DashboardLineage; + + // Add current dashboard + map.set(nodeKey('dashboard', dashboard.id, dashboard.title), { + name: dashboard.title, + type: 'dashboard', + id: dashboard.id, + additionalInfo: { + slug: dashboard.slug, + }, + }); + + // Add upstream charts + if (upstream?.charts?.result) { + upstream.charts.result.forEach((chart: ChartEntity) => { + map.set(nodeKey('chart', chart.id, chart.slice_name), { + name: chart.slice_name, + type: 'chart', + id: chart.id, + additionalInfo: { + viz_type: chart.viz_type, + }, + }); + }); + } + + // Add upstream datasets + if (upstream?.datasets?.result) { + upstream.datasets.result.forEach((dataset: DatasetEntity) => { + map.set(nodeKey('dataset', dataset.id, dataset.name), { + name: dataset.name, + type: 'dataset', + id: dataset.id, + additionalInfo: { + schema: dataset.schema, + table_name: dataset.table_name, + }, + }); + }); + } + + // Add upstream databases + if (upstream?.databases?.result) { + upstream.databases.result.forEach((database: DatabaseEntity) => { + map.set(nodeKey('database', database.id, database.database_name), { + name: database.database_name, + type: 'database', + id: database.id, + }); + }); + } + } + + return map; + }, [lineageResource, entityType]); + + // Handle node click + const handleNodeClick = useCallback( + (params: any) => { + if (params.dataType === 'node') { + const nodeName = params.name; + const nodeDetails = nodeDetailsMap.get(nodeName); + if (nodeDetails) { + setSelectedNode(nodeDetails); + } + } + // Always stop event propagation to prevent tooltip issues + if (params.event) { + params.event.stop(); + } + }, + [nodeDetailsMap], + ); + + const echartOptions: EChartsCoreOption | null = useMemo(() => { + if ( + lineageResource.status !== ResourceStatus.Complete || + !lineageResource.result + ) { + return null; + } + + const data = lineageResource.result; + const nodes: { + name: string; + label?: { position?: string; formatter?: string }; + itemStyle?: { color: string }; + }[] = []; + const links: { source: string; target: string; value: number }[] = []; + const nodeSet = new Set(); + + // Helper to add a node. `key` is the stable unique identity used for graph + // links and detail lookups; `label` is the human-readable text shown. + const addNode = ( + key: string, + label: string, + color: string, + labelPosition: 'left' | 'right' | 'inside', + ) => { + if (!nodeSet.has(key)) { + nodeSet.add(key); + nodes.push({ + name: key, + itemStyle: { color }, + label: { + position: labelPosition, + formatter: label, + }, + }); + } + }; + + // Helper to add a link between two node keys + const addLink = (source: string, target: string) => { + links.push({ source, target, value: 1 }); + }; + + // Build nodes and links based on entity type + if (entityType === 'dataset' && 'dataset' in data) { + const { dataset, upstream, downstream } = data as DatasetLineage; + + const datasetKey = nodeKey('dataset', dataset.id, dataset.name); + // Add current dataset node (center) - label inside + addNode(datasetKey, dataset.name, theme.colorPrimary, 'inside'); + + // Add upstream database - label on left + if (upstream?.database) { + const dbKey = nodeKey( + 'database', + upstream.database.id, + upstream.database.database_name, + ); + addNode( + dbKey, + upstream.database.database_name, + theme.colorInfo, + 'left', + ); + addLink(dbKey, datasetKey); + } + + // Add downstream charts - label on right + const chartKeys = new Map(); + if (downstream?.charts?.result) { + downstream.charts.result.forEach((chart: ChartEntity) => { + const chartKey = nodeKey('chart', chart.id, chart.slice_name); + chartKeys.set(chart.id, chartKey); + addNode(chartKey, chart.slice_name, theme.colorSuccess, 'right'); + addLink(datasetKey, chartKey); + }); + } + + // Add downstream dashboards - label on right + if (downstream?.dashboards?.result) { + downstream.dashboards.result.forEach((dashboard: DashboardEntity) => { + const dashKey = nodeKey('dashboard', dashboard.id, dashboard.title); + addNode(dashKey, dashboard.title, theme.colorWarning, 'right'); + + // Link from charts to dashboards using chart_ids + if (dashboard.chart_ids && dashboard.chart_ids.length > 0) { + dashboard.chart_ids.forEach(chartId => { + const chartKey = chartKeys.get(chartId); + if (chartKey) { + addLink(chartKey, dashKey); + } + }); + } + }); + } + } else if (entityType === 'chart' && 'chart' in data) { + const { chart, upstream, downstream } = data as ChartLineage; + + const chartKey = nodeKey('chart', chart.id, chart.slice_name); + // Add current chart node (center) - label inside + addNode(chartKey, chart.slice_name, theme.colorPrimary, 'inside'); + + // Add upstream dataset - label on left + if (upstream?.dataset) { + const datasetKey = nodeKey( + 'dataset', + upstream.dataset.id, + upstream.dataset.name, + ); + addNode(datasetKey, upstream.dataset.name, theme.colorInfo, 'left'); + addLink(datasetKey, chartKey); + + // Add upstream database - label on left + if (upstream.database) { + const dbKey = nodeKey( + 'database', + upstream.database.id, + upstream.database.database_name, + ); + addNode( + dbKey, + upstream.database.database_name, + theme.colorWarning, + 'left', + ); + addLink(dbKey, datasetKey); + } + } + + // Add downstream dashboards - label on right + if (downstream?.dashboards?.result) { + downstream.dashboards.result.forEach((dashboard: DashboardEntity) => { + const dashKey = nodeKey('dashboard', dashboard.id, dashboard.title); + addNode(dashKey, dashboard.title, theme.colorSuccess, 'right'); + addLink(chartKey, dashKey); + }); + } + } else if (entityType === 'dashboard' && 'dashboard' in data) { + const { dashboard, upstream } = data as DashboardLineage; + + const dashKey = nodeKey('dashboard', dashboard.id, dashboard.title); + // Add current dashboard node (right) - label inside + addNode(dashKey, dashboard.title, theme.colorPrimary, 'inside'); + + // Add upstream charts - label on left + const chartKeys = new Map(); + if (upstream?.charts?.result) { + upstream.charts.result.forEach((chart: ChartEntity) => { + const chartKey = nodeKey('chart', chart.id, chart.slice_name); + chartKeys.set(chart.id, chartKey); + addNode(chartKey, chart.slice_name, theme.colorInfo, 'left'); + addLink(chartKey, dashKey); + }); + } + + // Add upstream datasets - label on left + const datasetKeys = new Map(); + if (upstream?.datasets?.result) { + upstream.datasets.result.forEach(dataset => { + const datasetKey = nodeKey('dataset', dataset.id, dataset.name); + datasetKeys.set(dataset.id, datasetKey); + addNode(datasetKey, dataset.name, theme.colorSuccess, 'left'); + }); + } + + // Link charts to their specific datasets using dataset_id from each chart + if (upstream?.charts?.result) { + upstream.charts.result.forEach((chart: ChartEntity) => { + if (chart.dataset_id) { + const datasetKey = datasetKeys.get(chart.dataset_id); + const chartKey = chartKeys.get(chart.id); + if (datasetKey && chartKey) { + addLink(datasetKey, chartKey); + } + } + }); + } + + // Add upstream databases and link to their specific datasets + if (upstream?.databases?.result) { + upstream.databases.result.forEach(database => { + const dbKey = nodeKey( + 'database', + database.id, + database.database_name, + ); + addNode(dbKey, database.database_name, theme.colorWarning, 'left'); + + // Link databases to datasets that belong to them using database_id + if (upstream.datasets?.result) { + upstream.datasets.result.forEach(dataset => { + if (dataset.database_id === database.id) { + const datasetKey = datasetKeys.get(dataset.id); + if (datasetKey) { + addLink(dbKey, datasetKey); + } + } + }); + } + }); + } + } + + return { + series: { + animation: false, + data: nodes, + lineStyle: { + color: 'source', + }, + links, + type: 'sankey', + }, + tooltip: { + show: false, + }, + }; + }, [lineageResource, entityType, theme]); + + // Build legend data based on entity type + const legendItems: { label: string; color: string }[] = useMemo(() => { + if (entityType === 'dataset') { + return [ + { label: 'Database (Upstream)', color: theme.colorInfo }, + { label: 'Dataset (Current)', color: theme.colorPrimary }, + { label: 'Chart (Downstream)', color: theme.colorSuccess }, + { label: 'Dashboard (Downstream)', color: theme.colorWarning }, + ]; + } else if (entityType === 'chart') { + return [ + { label: 'Database (Upstream)', color: theme.colorWarning }, + { label: 'Dataset (Upstream)', color: theme.colorInfo }, + { label: 'Chart (Current)', color: theme.colorPrimary }, + { label: 'Dashboard (Downstream)', color: theme.colorSuccess }, + ]; + } else if (entityType === 'dashboard') { + return [ + { label: 'Database (Upstream)', color: theme.colorWarning }, + { label: 'Dataset (Upstream)', color: theme.colorSuccess }, + { label: 'Chart (Upstream)', color: theme.colorInfo }, + { label: 'Dashboard (Current)', color: theme.colorPrimary }, + ]; + } + return []; + }, [entityType, theme]); + + if (lineageResource.status === ResourceStatus.Loading) { + return ; + } + + if ( + lineageResource.status === ResourceStatus.Error || + !lineageResource.result + ) { + return ; + } + + if (!echartOptions) { + return ; + } + + // Helper function to get the URL for an entity + const getEntityUrl = (nodeDetails: NodeDetails): string => { + switch (nodeDetails.type) { + case 'dashboard': + return `/superset/dashboard/${nodeDetails.id}/`; + case 'chart': + return `/explore/?slice_id=${nodeDetails.id}`; + case 'dataset': + return `/dataset/${nodeDetails.id}`; + default: + return '#'; + } + }; + + return ( + + + {legendItems.map(item => ( + + {item.label} + + ))} + + + {selectedNode && ( + + + + {t( + '%s Details', + selectedNode.type.charAt(0).toUpperCase() + + selectedNode.type.slice(1), + )} + + + {(selectedNode.type === 'dashboard' || + selectedNode.type === 'chart') && ( + + )} + + + + + + {t('Name')}: + {selectedNode.name} + + {selectedNode.id && ( + + {t('ID')}: + {selectedNode.id} + + )} + {selectedNode.additionalInfo && + Object.entries(selectedNode.additionalInfo).map( + ([key, value]) => ( + + + {key.charAt(0).toUpperCase() + + key.slice(1).replace(/_/g, ' ')} + : + + {String(value)} + + ), + )} + + + )} + + ); +}; + +export default LineageView; diff --git a/superset-frontend/src/features/lineage/index.ts b/superset-frontend/src/features/lineage/index.ts new file mode 100644 index 000000000000..b197eca9258e --- /dev/null +++ b/superset-frontend/src/features/lineage/index.ts @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export { default as LineageView } from './LineageView'; +export { default as LineageModal } from './LineageModal'; diff --git a/superset-frontend/src/hooks/apiResources/apiResources.test.ts b/superset-frontend/src/hooks/apiResources/apiResources.test.ts index 9a66838638c3..e489d27330c9 100644 --- a/superset-frontend/src/hooks/apiResources/apiResources.test.ts +++ b/superset-frontend/src/hooks/apiResources/apiResources.test.ts @@ -97,6 +97,48 @@ describe('apiResource hooks', () => { error: fakeError, }); }); + + test('skips the fetch and stays loading when skip is true', async () => { + const fetchMock = jest.fn().mockResolvedValue(fakeApiResult); + (makeApi as any).mockReturnValue(fetchMock); + const { result } = renderHook(() => + useApiResourceFullBody('/test/endpoint', true), + ); + await act(async () => { + jest.runAllTimers(); + }); + expect(fetchMock).not.toHaveBeenCalled(); + expect(result.current).toEqual({ + status: ResourceStatus.Loading, + result: null, + error: null, + }); + }); + + test('re-enables the fetch when skip toggles from true to false', async () => { + const fetchMock = jest.fn().mockResolvedValue(fakeApiResult); + (makeApi as any).mockReturnValue(fetchMock); + const { result, rerender } = renderHook( + ({ skip }) => useApiResourceFullBody('/test/endpoint', skip), + { initialProps: { skip: true } }, + ); + await act(async () => { + jest.runAllTimers(); + }); + expect(fetchMock).not.toHaveBeenCalled(); + expect(result.current.status).toEqual(ResourceStatus.Loading); + + rerender({ skip: false }); + await act(async () => { + jest.runAllTimers(); + }); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(result.current).toEqual({ + status: ResourceStatus.Complete, + result: fakeApiResult, + error: null, + }); + }); }); // eslint-disable-next-line no-restricted-globals -- TODO: Migrate from describe blocks diff --git a/superset-frontend/src/hooks/apiResources/apiResources.ts b/superset-frontend/src/hooks/apiResources/apiResources.ts index 04166db1d32f..ec6894128d45 100644 --- a/superset-frontend/src/hooks/apiResources/apiResources.ts +++ b/superset-frontend/src/hooks/apiResources/apiResources.ts @@ -86,6 +86,7 @@ const initialState: LoadingState = { */ export function useApiResourceFullBody( endpoint: string, + skip = false, ): Resource { const [resource, setResource] = useState>(initialState); const cancelRef = useRef<() => void>(() => {}); @@ -98,6 +99,12 @@ export function useApiResourceFullBody( // when this effect runs, the endpoint has changed. // cancel any current calls so that state doesn't get messed up. cancelRef.current(); + + // Allow callers to opt out of fetching (e.g. when the identifier isn't + // known yet) so we don't fire requests against invalid endpoints. + if (skip) { + return undefined; + } let cancelled = false; cancelRef.current = () => { cancelled = true; @@ -132,7 +139,7 @@ export function useApiResourceFullBody( return () => { cancelled = true; }; - }, [endpoint]); + }, [endpoint, skip]); return resource; } @@ -181,9 +188,12 @@ const extractInnerResult = (responseBody: { result: T }) => * * @param endpoint The url where the resource is located. */ -export function useApiV1Resource(endpoint: string): Resource { +export function useApiV1Resource( + endpoint: string, + skip = false, +): Resource { return useTransformedResource( - useApiResourceFullBody<{ result: RESULT }>(endpoint), + useApiResourceFullBody<{ result: RESULT }>(endpoint, skip), extractInnerResult, ); } diff --git a/superset-frontend/src/hooks/apiResources/index.ts b/superset-frontend/src/hooks/apiResources/index.ts index 53aa7aa113cb..61d3628a61b8 100644 --- a/superset-frontend/src/hooks/apiResources/index.ts +++ b/superset-frontend/src/hooks/apiResources/index.ts @@ -29,6 +29,7 @@ export { export * from './catalogs'; export * from './charts'; export * from './dashboards'; +export * from './lineage'; export * from './tables'; export * from './schemas'; export * from './queryValidations'; diff --git a/superset-frontend/src/hooks/apiResources/lineage.test.ts b/superset-frontend/src/hooks/apiResources/lineage.test.ts new file mode 100644 index 000000000000..38072dcc8eae --- /dev/null +++ b/superset-frontend/src/hooks/apiResources/lineage.test.ts @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { renderHook, waitFor } from '@testing-library/react'; +import { makeApi } from '@superset-ui/core'; +import { + useChartLineage, + useDashboardLineage, + useDatasetLineage, +} from './lineage'; + +jest.mock('@superset-ui/core', () => ({ + ...jest.requireActual('@superset-ui/core'), + makeApi: jest.fn(), +})); + +const mockedMakeApi = jest.mocked(makeApi); + +// makeApi returns a function that issues the request; capture the endpoint it +// was configured with so we can assert the correct URL was built. +function mockApiSuccess(payload: unknown) { + const fetcher = jest.fn().mockResolvedValue({ result: payload }); + mockedMakeApi.mockReturnValue(fetcher as any); + return fetcher; +} + +function mockApiError(error: Error) { + const fetcher = jest.fn().mockRejectedValue(error); + mockedMakeApi.mockReturnValue(fetcher as any); + return fetcher; +} + +beforeEach(() => { + jest.clearAllMocks(); +}); + +test('useDatasetLineage fetches dataset lineage and unwraps result', async () => { + const payload = { + dataset: { id: 1, name: 'ds' }, + upstream: { database: { id: 2, database_name: 'db', backend: 'pg' } }, + downstream: { + charts: { count: 0, result: [] }, + dashboards: { count: 0, result: [] }, + }, + }; + mockApiSuccess(payload); + + const { result } = renderHook(() => useDatasetLineage(1)); + + expect(result.current.status).toBe('loading'); + await waitFor(() => expect(result.current.status).toBe('complete')); + + expect(mockedMakeApi).toHaveBeenCalledWith( + expect.objectContaining({ + method: 'GET', + endpoint: '/api/v1/dataset/1/lineage', + }), + ); + expect(result.current.result).toEqual(payload); + expect(result.current.error).toBeNull(); +}); + +test('useChartLineage builds the chart lineage endpoint', async () => { + mockApiSuccess({ chart: { id: 5, slice_name: 'c', viz_type: 'pie' } }); + + const { result } = renderHook(() => useChartLineage(5)); + + await waitFor(() => expect(result.current.status).toBe('complete')); + expect(mockedMakeApi).toHaveBeenCalledWith( + expect.objectContaining({ endpoint: '/api/v1/chart/5/lineage' }), + ); +}); + +test('useDashboardLineage builds the dashboard lineage endpoint', async () => { + mockApiSuccess({ dashboard: { id: 9, title: 'd', slug: 'd' } }); + + const { result } = renderHook(() => useDashboardLineage(9)); + + await waitFor(() => expect(result.current.status).toBe('complete')); + expect(mockedMakeApi).toHaveBeenCalledWith( + expect.objectContaining({ endpoint: '/api/v1/dashboard/9/lineage' }), + ); +}); + +test('lineage hooks surface network errors', async () => { + mockApiError(new Error('Network error')); + + const { result } = renderHook(() => useDatasetLineage(1)); + + await waitFor(() => expect(result.current.status).toBe('error')); + expect(result.current.result).toBeNull(); + expect(result.current.error).toBeInstanceOf(Error); +}); + +test('lineage hooks skip the request when the id is empty', async () => { + const fetcher = mockApiSuccess({}); + + const { result } = renderHook(() => useDatasetLineage('')); + + // Empty id resolves immediately without ever firing a request, so we never + // hit an invalid endpoint such as `/api/v1/dataset//lineage`. + expect(result.current.status).toBe('loading'); + expect(fetcher).not.toHaveBeenCalled(); +}); + +test('lineage hooks skip the request when skip is true', async () => { + const fetcher = mockApiSuccess({}); + + const { result } = renderHook(() => useChartLineage(5, true)); + + expect(result.current.status).toBe('loading'); + expect(fetcher).not.toHaveBeenCalled(); +}); diff --git a/superset-frontend/src/hooks/apiResources/lineage.ts b/superset-frontend/src/hooks/apiResources/lineage.ts new file mode 100644 index 000000000000..56e7a7a6a80c --- /dev/null +++ b/superset-frontend/src/hooks/apiResources/lineage.ts @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { useApiV1Resource } from './apiResources'; + +// Database entity type +export type DatabaseEntity = { + id: number; + database_name: string; + backend: string; +}; + +// Dataset entity type +export type DatasetEntity = { + id: number; + name: string; + schema: string | null; + table_name: string; + database_id: number; + database_name: string; + chart_ids?: number[]; +}; + +// Chart entity type +export type ChartEntity = { + id: number; + slice_name: string; + viz_type: string; + dashboard_ids?: number[]; + dataset_id?: number; +}; + +// Dashboard entity type +export type DashboardEntity = { + id: number; + title: string; + slug: string; + chart_ids?: number[]; +}; + +// Dataset lineage response type +export type DatasetLineage = { + dataset: DatasetEntity; + upstream: { + database: DatabaseEntity; + }; + downstream: { + charts: { + count: number; + result: ChartEntity[]; + }; + dashboards: { + count: number; + result: DashboardEntity[]; + }; + }; +}; + +// Chart lineage response type +export type ChartLineage = { + chart: ChartEntity & { + datasource_id: number; + datasource_type: string; + }; + upstream: { + dataset: DatasetEntity; + database: DatabaseEntity; + }; + downstream: { + dashboards: { + count: number; + result: DashboardEntity[]; + }; + }; +}; + +// Dashboard lineage response type +export type DashboardLineage = { + dashboard: DashboardEntity & { + published: boolean; + }; + upstream: { + charts: { + count: number; + result: ChartEntity[]; + }; + datasets: { + count: number; + result: DatasetEntity[]; + }; + databases: { + count: number; + result: DatabaseEntity[]; + }; + }; + downstream: null; +}; + +// A missing/empty identifier means we have nothing to fetch yet; skip the +// request so we never hit invalid endpoints like `/api/v1/chart//lineage`. +const isEmptyId = (idOrUuid: string | number): boolean => + idOrUuid === '' || idOrUuid == null; + +/** + * Hook to fetch lineage data for a dataset + * @param idOrUuid Dataset ID or UUID + * @param skip When true, defers the request (e.g. until the tab is active) + */ +export const useDatasetLineage = (idOrUuid: string | number, skip = false) => + useApiV1Resource( + `/api/v1/dataset/${idOrUuid}/lineage`, + skip || isEmptyId(idOrUuid), + ); + +/** + * Hook to fetch lineage data for a chart + * @param idOrUuid Chart ID or UUID + * @param skip When true, defers the request (e.g. until the tab is active) + */ +export const useChartLineage = (idOrUuid: string | number, skip = false) => + useApiV1Resource( + `/api/v1/chart/${idOrUuid}/lineage`, + skip || isEmptyId(idOrUuid), + ); + +/** + * Hook to fetch lineage data for a dashboard + * @param idOrSlug Dashboard ID or slug + * @param skip When true, defers the request (e.g. until the tab is active) + */ +export const useDashboardLineage = (idOrSlug: string | number, skip = false) => + useApiV1Resource( + `/api/v1/dashboard/${idOrSlug}/lineage`, + skip || isEmptyId(idOrSlug), + ); diff --git a/superset/charts/api.py b/superset/charts/api.py index 3534d6dfa527..cf2d875062af 100644 --- a/superset/charts/api.py +++ b/superset/charts/api.py @@ -131,6 +131,7 @@ def ensure_thumbnails_enabled(self) -> Optional[Response]: "screenshot", "cache_screenshot", "warm_up_cache", + "lineage", } class_permission_name = "Chart" method_permission_name = MODEL_API_RW_METHOD_PERMISSION_MAP @@ -313,6 +314,107 @@ def get(self, id_or_uuid: str) -> Response: except ChartNotFoundError: return self.response_404() + @expose("//lineage", methods=("GET",)) + @protect() + @safe + @statsd_metrics + @event_logger.log_this_with_context( + action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.lineage", + log_to_statsd=False, + ) + def lineage(self, id_or_uuid: str) -> Response: + """Get lineage information for a chart. + --- + get: + summary: Get lineage information for a chart + description: >- + Returns upstream (dataset, database) and downstream (dashboards) lineage + information for a chart + parameters: + - in: path + name: id_or_uuid + schema: + type: string + description: Either the id of the chart, or its uuid + responses: + 200: + description: Lineage information + content: + application/json: + schema: + $ref: "#/components/schemas/ChartLineageResponseSchema" + 401: + $ref: '#/components/responses/401' + 404: + $ref: '#/components/responses/404' + 500: + $ref: '#/components/responses/500' + """ + try: + chart = ChartDAO.get_by_id_or_uuid(id_or_uuid) + except ChartNotFoundError: + return self.response_404() + + chart_info = { + "id": chart.id, + "slice_name": chart.slice_name, + "viz_type": chart.viz_type, + } + + # Get upstream (dataset and database) information + upstream: dict[str, Any] = {} + if dataset := chart.datasource: + upstream["dataset"] = { + "id": dataset.id, + "name": dataset.name, + "database_id": dataset.database_id, + "database_name": dataset.database.database_name + if dataset.database + else None, + "schema": dataset.schema, + "table_name": dataset.table_name, + } + if dataset.database: + upstream["database"] = { + "id": dataset.database.id, + "database_name": dataset.database.database_name, + "backend": dataset.database.backend, + } + else: + upstream["database"] = None + else: + upstream["dataset"] = None + upstream["database"] = None + + # Get downstream (dashboards) information, filtered by the current + # user's permissions so lineage never exposes dashboards the user + # cannot access. + dashboards = [] + for dashboard in chart.dashboards: + if not security_manager.can_access_dashboard(dashboard): + continue + dashboards.append( + { + "id": dashboard.id, + "title": dashboard.dashboard_title, + "slug": dashboard.slug, + } + ) + + downstream = { + "dashboards": { + "count": len(dashboards), + "result": dashboards, + }, + } + + result = { + "chart": chart_info, + "upstream": upstream, + "downstream": downstream, + } + return self.response(200, result=result) + @expose("/", methods=("POST",)) @protect() @safe diff --git a/superset/charts/schemas.py b/superset/charts/schemas.py index 0a603cbb6ec5..2f976ee2de12 100644 --- a/superset/charts/schemas.py +++ b/superset/charts/schemas.py @@ -1755,6 +1755,53 @@ class ChartGetResponseSchema(Schema): datasource_uuid = fields.UUID(attribute="table.uuid") +class ChartLineageChartSchema(Schema): + id = fields.Integer() + slice_name = fields.String() + viz_type = fields.String() + + +class ChartLineageDatasetSchema(Schema): + id = fields.Integer() + name = fields.String() + database_id = fields.Integer() + database_name = fields.String() + schema = fields.String(allow_none=True) + table_name = fields.String() + + +class ChartLineageDatabaseSchema(Schema): + id = fields.Integer() + database_name = fields.String() + backend = fields.String() + + +class ChartLineageDashboardSchema(Schema): + id = fields.Integer() + title = fields.String() + slug = fields.String() + + +class ChartLineageUpstreamSchema(Schema): + dataset = fields.Nested(ChartLineageDatasetSchema, allow_none=True) + database = fields.Nested(ChartLineageDatabaseSchema, allow_none=True) + + +class ChartLineageDownstreamDashboardsSchema(Schema): + count = fields.Integer() + result = fields.List(fields.Nested(ChartLineageDashboardSchema)) + + +class ChartLineageDownstreamSchema(Schema): + dashboards = fields.Nested(ChartLineageDownstreamDashboardsSchema) + + +class ChartLineageResponseSchema(Schema): + chart = fields.Nested(ChartLineageChartSchema) + upstream = fields.Nested(ChartLineageUpstreamSchema) + downstream = fields.Nested(ChartLineageDownstreamSchema) + + CHART_SCHEMAS = ( ChartCacheWarmUpRequestSchema, ChartCacheWarmUpResponseSchema, @@ -1782,4 +1829,5 @@ class ChartGetResponseSchema(Schema): ChartGetResponseSchema, ChartCacheScreenshotResponseSchema, GetFavStarIdsSchema, + ChartLineageResponseSchema, ) diff --git a/superset/constants.py b/superset/constants.py index 863d0cf82ba1..c5d7fd6a92d1 100644 --- a/superset/constants.py +++ b/superset/constants.py @@ -179,6 +179,7 @@ class RouteMethod: # pylint: disable=too-few-public-methods "put_colors": "write", "sync_permissions": "write", "restore": "write", + "lineage": "read", } EXTRA_FORM_DATA_APPEND_KEYS = { diff --git a/superset/dashboards/api.py b/superset/dashboards/api.py index ff0c2a89c668..a2726c5e33c2 100644 --- a/superset/dashboards/api.py +++ b/superset/dashboards/api.py @@ -104,6 +104,7 @@ DashboardCopySchema, DashboardDatasetSchema, DashboardGetResponseSchema, + DashboardLineageResponseSchema, DashboardNativeFiltersConfigUpdateSchema, DashboardPostSchema, DashboardPutSchema, @@ -252,6 +253,7 @@ class DashboardRestApi(CustomTagsOptimizationMixin, BaseSupersetModelRestApi): "put_chart_customizations", "put_colors", "export_as_example", + "lineage", } resource_name = "dashboard" allow_browser_login = True @@ -429,6 +431,7 @@ def get_list(self, **kwargs: Any) -> Response: DashboardCacheScreenshotResponseSchema, DashboardCopySchema, DashboardGetResponseSchema, + DashboardLineageResponseSchema, DashboardDatasetSchema, TabsPayloadSchema, GetFavStarIdsSchema, @@ -524,6 +527,126 @@ def get( ) return self.response(200, result=result) + @expose("//lineage", methods=("GET",)) + @protect() + @safe + @statsd_metrics + @with_dashboard + @event_logger.log_this_with_context( + action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.lineage", + log_to_statsd=False, + ) + # pylint: disable=arguments-differ,arguments-renamed + def lineage(self, dash: Dashboard) -> Response: + """Get lineage information for a dashboard. + --- + get: + summary: Get lineage information for a dashboard + description: >- + Returns upstream (charts, datasets, databases) lineage information + for a dashboard + parameters: + - in: path + name: id_or_slug + schema: + type: string + description: Either the id of the dashboard, or its slug + responses: + 200: + description: Lineage information + content: + application/json: + schema: + $ref: "#/components/schemas/DashboardLineageResponseSchema" + 401: + $ref: '#/components/responses/401' + 404: + $ref: '#/components/responses/404' + 500: + $ref: '#/components/responses/500' + """ + dashboard_info = { + "id": dash.id, + "title": dash.dashboard_title, + "slug": dash.slug, + "published": dash.published, + } + + # Get upstream (charts, datasets, databases) information + charts = [] + dataset_map = {} + database_map = {} + + for chart in dash.slices: + charts.append( + { + "id": chart.id, + "slice_name": chart.slice_name, + "viz_type": chart.viz_type, + "dataset_id": chart.datasource_id, + } + ) + + # Collect dataset information. Schema/table/database details are + # only exposed to users who can access the underlying datasource; + # otherwise they are redacted so lineage never leaks datasource + # internals (the dataset id/name are kept so the graph still + # renders). + dataset = chart.datasource + if dataset and dataset.id not in dataset_map: + can_access = security_manager.can_access_datasource(dataset) + dataset_map[dataset.id] = { + "id": dataset.id, + "name": dataset.name, + "database_id": dataset.database_id if can_access else None, + "database_name": ( + dataset.database.database_name + if can_access and dataset.database + else None + ), + "schema": dataset.schema if can_access else None, + "table_name": dataset.table_name if can_access else None, + "chart_ids": [], + } + + if dataset and dataset.id in dataset_map: + dataset_map[dataset.id]["chart_ids"].append(chart.id) + + # Collect database information, only for accessible datasources + if ( + dataset + and security_manager.can_access_datasource(dataset) + and dataset.database + and dataset.database.id not in database_map + ): + database_map[dataset.database.id] = { + "id": dataset.database.id, + "database_name": dataset.database.database_name, + "backend": dataset.database.backend, + } + + upstream = { + "charts": { + "count": len(charts), + "result": charts, + }, + "datasets": { + "count": len(dataset_map), + "result": list(dataset_map.values()), + }, + "databases": { + "count": len(database_map), + "result": list(database_map.values()), + }, + } + + result = { + "dashboard": dashboard_info, + "upstream": upstream, + "downstream": None, + } + return self.response(200, result=result) + @expose("//datasets", methods=("GET",)) @protect() @handle_api_exception diff --git a/superset/dashboards/schemas.py b/superset/dashboards/schemas.py index a71b90a1afd3..b06c018e5032 100644 --- a/superset/dashboards/schemas.py +++ b/superset/dashboards/schemas.py @@ -558,3 +558,60 @@ class CacheScreenshotSchema(Schema): fields.List(fields.Str(), validate=lambda x: len(x) == 2), required=False ) permalinkKey = fields.Str(required=False) # noqa: N815 + + +class DashboardLineageDashboardSchema(Schema): + id = fields.Integer() + title = fields.String() + slug = fields.String() + published = fields.Boolean() + + +class DashboardLineageChartSchema(Schema): + id = fields.Integer() + slice_name = fields.String() + viz_type = fields.String() + dataset_id = fields.Integer() + + +class DashboardLineageDatasetSchema(Schema): + id = fields.Integer() + name = fields.String() + database_id = fields.Integer() + database_name = fields.String() + schema = fields.String(allow_none=True) + table_name = fields.String() + chart_ids = fields.List(fields.Integer()) + + +class DashboardLineageDatabaseSchema(Schema): + id = fields.Integer() + database_name = fields.String() + backend = fields.String() + + +class DashboardLineageUpstreamChartsSchema(Schema): + count = fields.Integer() + result = fields.List(fields.Nested(DashboardLineageChartSchema)) + + +class DashboardLineageUpstreamDatasetsSchema(Schema): + count = fields.Integer() + result = fields.List(fields.Nested(DashboardLineageDatasetSchema)) + + +class DashboardLineageUpstreamDatabasesSchema(Schema): + count = fields.Integer() + result = fields.List(fields.Nested(DashboardLineageDatabaseSchema)) + + +class DashboardLineageUpstreamSchema(Schema): + charts = fields.Nested(DashboardLineageUpstreamChartsSchema) + datasets = fields.Nested(DashboardLineageUpstreamDatasetsSchema) + databases = fields.Nested(DashboardLineageUpstreamDatabasesSchema) + + +class DashboardLineageResponseSchema(Schema): + dashboard = fields.Nested(DashboardLineageDashboardSchema) + upstream = fields.Nested(DashboardLineageUpstreamSchema) + downstream = fields.Field(allow_none=True) diff --git a/superset/datasets/api.py b/superset/datasets/api.py index cee61509067f..13256fe8ba63 100644 --- a/superset/datasets/api.py +++ b/superset/datasets/api.py @@ -64,6 +64,7 @@ DatasetCacheWarmUpResponseSchema, DatasetDrillInfoSchema, DatasetDuplicateSchema, + DatasetLineageResponseSchema, DatasetPostSchema, DatasetPutSchema, DatasetRelatedObjectsResponse, @@ -111,6 +112,7 @@ class DatasetRestApi(BaseSupersetModelRestApi): "get_or_create_dataset", "warm_up_cache", "get_drill_info", + "lineage", } list_columns = [ "id", @@ -299,6 +301,7 @@ class DatasetRestApi(BaseSupersetModelRestApi): DatasetRelatedObjectsResponse, DatasetDuplicateSchema, GetOrCreateDatasetSchema, + DatasetLineageResponseSchema, ) openapi_spec_methods = openapi_spec_methods_override @@ -846,6 +849,129 @@ def related_objects(self, id_or_uuid: str) -> Response: dashboards={"count": len(dashboards), "result": dashboards}, ) + @expose("//lineage", methods=("GET",)) + @protect() + @safe + @statsd_metrics + @event_logger.log_this_with_context( + action=lambda self, *args, **kwargs: f"{self.__class__.__name__}.lineage", + log_to_statsd=False, + ) + def lineage(self, id_or_uuid: str) -> Response: + """Get lineage information for a dataset. + --- + get: + summary: Get lineage information for a dataset + description: >- + Returns upstream (database) and downstream (charts, dashboards) lineage + information for a dataset + parameters: + - in: path + name: id_or_uuid + schema: + type: string + description: Either the id of the dataset, or its uuid + responses: + 200: + description: Lineage information + content: + application/json: + schema: + $ref: "#/components/schemas/DatasetLineageResponseSchema" + 401: + $ref: '#/components/responses/401' + 404: + $ref: '#/components/responses/404' + 500: + $ref: '#/components/responses/500' + """ + dataset = DatasetDAO.find_by_id_or_uuid(id_or_uuid) + if not dataset: + return self.response_404() + + dataset_info = { + "id": dataset.id, + "name": dataset.name, + "database_id": dataset.database_id, + "database_name": ( + dataset.database.database_name if dataset.database else None + ), + "schema": dataset.schema, + "table_name": dataset.table_name, + } + + # Get upstream (database) information + upstream: dict[str, Any] = {} + if dataset.database: + upstream["database"] = { + "id": dataset.database.id, + "database_name": dataset.database.database_name, + "backend": dataset.database.backend, + } + else: + upstream["database"] = None + + # Get downstream (charts and dashboards) information + related_data = DatasetDAO.get_related_objects(dataset.id) + + # Build chart information with dashboard IDs, filtering both the charts + # and their linked dashboards by the current user's permissions so + # lineage never exposes assets the user cannot access. + charts = [] + for chart in related_data["charts"]: + if not security_manager.can_access_chart(chart): + continue + dashboard_ids = [ + d.id + for d in chart.dashboards + if security_manager.can_access_dashboard(d) + ] + charts.append( + { + "id": chart.id, + "slice_name": chart.slice_name, + "viz_type": chart.viz_type, + "dashboard_ids": dashboard_ids, + } + ) + + # Build dashboard information with chart IDs + dashboards = [] + for dashboard in related_data["dashboards"]: + if not security_manager.can_access_dashboard(dashboard): + continue + chart_ids = [ + chart.id + for chart in dashboard.slices + if chart.datasource_id == dataset.id + ] + dashboards.append( + { + "id": dashboard.id, + "title": dashboard.dashboard_title, + "slug": dashboard.slug, + "chart_ids": chart_ids, + } + ) + + downstream = { + "charts": { + "count": len(charts), + "result": charts, + }, + "dashboards": { + "count": len(dashboards), + "result": dashboards, + }, + } + + result = { + "dataset": dataset_info, + "upstream": upstream, + "downstream": downstream, + } + return self.response(200, result=result) + @expose("/", methods=("DELETE",)) @protect() @safe diff --git a/superset/datasets/schemas.py b/superset/datasets/schemas.py index 386686d57003..8209f67cbfb3 100644 --- a/superset/datasets/schemas.py +++ b/superset/datasets/schemas.py @@ -250,6 +250,60 @@ class DatasetRelatedObjectsResponse(Schema): dashboards = fields.Nested(DatasetRelatedDashboards) +class DatasetLineageDatasetSchema(Schema): + id = fields.Integer() + name = fields.String() + database_id = fields.Integer() + database_name = fields.String() + schema = fields.String(allow_none=True) + table_name = fields.String() + + +class DatasetLineageDatabaseSchema(Schema): + id = fields.Integer() + database_name = fields.String() + backend = fields.String() + + +class DatasetLineageChartSchema(Schema): + id = fields.Integer() + slice_name = fields.String() + viz_type = fields.String() + dashboard_ids = fields.List(fields.Integer()) + + +class DatasetLineageDashboardSchema(Schema): + id = fields.Integer() + title = fields.String() + slug = fields.String() + chart_ids = fields.List(fields.Integer()) + + +class DatasetLineageUpstreamSchema(Schema): + database = fields.Nested(DatasetLineageDatabaseSchema, allow_none=True) + + +class DatasetLineageDownstreamChartsSchema(Schema): + count = fields.Integer() + result = fields.List(fields.Nested(DatasetLineageChartSchema)) + + +class DatasetLineageDownstreamDashboardsSchema(Schema): + count = fields.Integer() + result = fields.List(fields.Nested(DatasetLineageDashboardSchema)) + + +class DatasetLineageDownstreamSchema(Schema): + charts = fields.Nested(DatasetLineageDownstreamChartsSchema) + dashboards = fields.Nested(DatasetLineageDownstreamDashboardsSchema) + + +class DatasetLineageResponseSchema(Schema): + dataset = fields.Nested(DatasetLineageDatasetSchema) + upstream = fields.Nested(DatasetLineageUpstreamSchema) + downstream = fields.Nested(DatasetLineageDownstreamSchema) + + class ImportV1ColumnSchema(Schema): # pylint: disable=unused-argument @pre_load diff --git a/tests/integration_tests/charts/api_tests.py b/tests/integration_tests/charts/api_tests.py index 512b6f682c05..d41d13bd27fb 100644 --- a/tests/integration_tests/charts/api_tests.py +++ b/tests/integration_tests/charts/api_tests.py @@ -50,6 +50,7 @@ load_birth_names_dashboard_with_slices, # noqa: F401 load_birth_names_data, # noqa: F401 ) +from tests.integration_tests.fixtures.client import client # noqa: F401 from tests.integration_tests.fixtures.energy_dashboard import ( load_energy_table_data, # noqa: F401 load_energy_table_with_slice, # noqa: F401 @@ -59,6 +60,10 @@ database_config, dataset_config, ) +from tests.integration_tests.fixtures.lineage import ( + inject_expected_chart_lineage, # noqa: F401 + lineage_test_data, # noqa: F401 +) from tests.integration_tests.fixtures.tags import ( create_custom_tags, # noqa: F401 get_filter_params, @@ -2444,3 +2449,30 @@ def test_related_owners_allowed_for_write_user(self): self.login(ADMIN_USERNAME) rv = self.client.get("api/v1/chart/related/owners") assert rv.status_code == 200 + + @pytest.mark.usefixtures("inject_expected_chart_lineage") + def test_get_chart_lineage(self): + """ + Chart API: Test get chart lineage + """ + self.login(ADMIN_USERNAME) + chart_id = self.chart_lineage["chart_id"] + expected = self.chart_lineage["expected"] + + uri = f"api/v1/chart/{chart_id}/lineage" + rv = self.get_assert_metric(uri, "lineage") + assert rv.status_code == 200 + + data = json.loads(rv.data.decode("utf-8")) + + # The lineage payload is wrapped under "result" + assert data["result"] == expected + + def test_get_chart_lineage_not_found(self): + """ + Chart API: Test get chart lineage with non-existent chart + """ + self.login(ADMIN_USERNAME) + uri = "api/v1/chart/99999/lineage" + rv = self.client.get(uri) + assert rv.status_code == 404 diff --git a/tests/integration_tests/dashboards/api_tests.py b/tests/integration_tests/dashboards/api_tests.py index bdc42ed9287c..a635378c0bea 100644 --- a/tests/integration_tests/dashboards/api_tests.py +++ b/tests/integration_tests/dashboards/api_tests.py @@ -66,6 +66,11 @@ load_birth_names_dashboard_with_slices, # noqa: F401 load_birth_names_data, # noqa: F401 ) +from tests.integration_tests.fixtures.client import client # noqa: F401 +from tests.integration_tests.fixtures.lineage import ( + inject_expected_dashboard_lineage, # noqa: F401 + lineage_test_data, # noqa: F401 +) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, # noqa: F401 load_world_bank_data, # noqa: F401 @@ -4049,6 +4054,33 @@ def test_put_dashboard_colors_not_authorized(self): db.session.delete(dashboard) db.session.commit() + @pytest.mark.usefixtures("inject_expected_dashboard_lineage") + def test_get_dashboard_lineage(self): + """ + Dashboard API: Test get dashboard lineage + """ + self.login(ADMIN_USERNAME) + dashboard_id = self.dashboard_lineage["dashboard_id"] + expected = self.dashboard_lineage["expected"] + + uri = f"api/v1/dashboard/{dashboard_id}/lineage" + rv = self.get_assert_metric(uri, "lineage") + assert rv.status_code == 200 + + data = json.loads(rv.data.decode("utf-8")) + + # The lineage payload is wrapped under "result" + assert data["result"] == expected + + def test_get_dashboard_lineage_not_found(self): + """ + Dashboard API: Test get dashboard lineage with non-existent dashboard + """ + self.login(ADMIN_USERNAME) + uri = "api/v1/dashboard/99999/lineage" + rv = self.client.get(uri) + assert rv.status_code == 404 + class TestDashboardCustomTagsFiltering(SupersetTestCase): """Test dashboard list API tags field behavior. diff --git a/tests/integration_tests/datasets/api_tests.py b/tests/integration_tests/datasets/api_tests.py index ba253573e2e9..a5c652d55f51 100644 --- a/tests/integration_tests/datasets/api_tests.py +++ b/tests/integration_tests/datasets/api_tests.py @@ -55,6 +55,7 @@ load_birth_names_dashboard_with_slices, # noqa: F401 load_birth_names_data, # noqa: F401 ) +from tests.integration_tests.fixtures.client import client # noqa: F401 from tests.integration_tests.fixtures.energy_dashboard import ( load_energy_table_data, # noqa: F401 load_energy_table_with_slice, # noqa: F401 @@ -64,6 +65,10 @@ dataset_config, dataset_ui_export, ) +from tests.integration_tests.fixtures.lineage import ( + inject_expected_dataset_lineage, # noqa: F401 + lineage_test_data, # noqa: F401 +) class TestDatasetApi(SupersetTestCase): @@ -3555,3 +3560,30 @@ def test_get_drill_info_dashboard_rbac_no_dashboard_id(self): assert rv.status_code == 404 self.items_to_delete = [dashboard, chart, dataset] + + @pytest.mark.usefixtures("inject_expected_dataset_lineage") + def test_get_dataset_lineage(self): + """ + Dataset API: Test get dataset lineage + """ + self.login(ADMIN_USERNAME) + dataset_id = self.dataset_lineage["dataset_id"] + expected = self.dataset_lineage["expected"] + + uri = f"api/v1/dataset/{dataset_id}/lineage" + rv = self.get_assert_metric(uri, "lineage") + assert rv.status_code == 200 + + data = json.loads(rv.data.decode("utf-8")) + + # The lineage payload is wrapped under "result" + assert data["result"] == expected + + def test_get_dataset_lineage_not_found(self): + """ + Dataset API: Test get dataset lineage with non-existent dataset + """ + self.login(ADMIN_USERNAME) + uri = "api/v1/dataset/99999/lineage" + rv = self.client.get(uri) + assert rv.status_code == 404 diff --git a/tests/integration_tests/fixtures/lineage.py b/tests/integration_tests/fixtures/lineage.py new file mode 100644 index 000000000000..e4d606b3cb27 --- /dev/null +++ b/tests/integration_tests/fixtures/lineage.py @@ -0,0 +1,266 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import pytest + +from superset import db +from superset.models.dashboard import Dashboard +from superset.models.slice import Slice +from superset.utils.database import get_example_database +from tests.integration_tests.dashboard_utils import create_table_metadata + + +@pytest.fixture +def lineage_test_data(app_context, load_birth_names_data): + """ + Base fixture that creates a simple lineage structure and returns + the created entities (database, dataset, charts, dashboard). + """ + database = get_example_database() + + # Create dataset + dataset = create_table_metadata( + table_name="lineage_test_dataset", + database=database, + ) + db.session.add(dataset) + db.session.flush() + + # Create charts + chart1 = Slice( + slice_name="Lineage Test Chart 1", + viz_type="table", + datasource_id=dataset.id, + datasource_type="table", + params="{}", + ) + chart2 = Slice( + slice_name="Lineage Test Chart 2", + viz_type="pie", + datasource_id=dataset.id, + datasource_type="table", + params="{}", + ) + db.session.add(chart1) + db.session.add(chart2) + db.session.flush() + + # Create dashboard with charts + dashboard = Dashboard( + dashboard_title="Lineage Test Dashboard", + slug="lineage-test-dashboard", + slices=[chart1, chart2], + published=True, + ) + db.session.add(dashboard) + db.session.commit() + + # Return the created entities + result = { + "database": database, + "dataset": dataset, + "charts": [chart1, chart2], + "dashboard": dashboard, + } + + yield result + + # Cleanup + db.session.delete(dashboard) + db.session.delete(chart1) + db.session.delete(chart2) + for col in dataset.columns + dataset.metrics: + db.session.delete(col) + db.session.delete(dataset) + db.session.commit() + + +@pytest.fixture(autouse=False) +def inject_expected_dataset_lineage(request, lineage_test_data): + """ + Injects dataset lineage data into test class instance. + """ + dataset = lineage_test_data["dataset"] + database = lineage_test_data["database"] + charts = lineage_test_data["charts"] + dashboard = lineage_test_data["dashboard"] + + request.instance.dataset_lineage = { + "dataset_id": dataset.id, + "expected": { + "dataset": { + "id": dataset.id, + "name": dataset.name, + "schema": dataset.schema, + "table_name": dataset.table_name, + "database_id": database.id, + "database_name": database.database_name, + }, + "upstream": { + "database": { + "id": database.id, + "database_name": database.database_name, + "backend": database.backend, + } + }, + "downstream": { + "charts": { + "count": 2, + "result": [ + { + "id": charts[0].id, + "slice_name": charts[0].slice_name, + "viz_type": charts[0].viz_type, + "dashboard_ids": [dashboard.id], + }, + { + "id": charts[1].id, + "slice_name": charts[1].slice_name, + "viz_type": charts[1].viz_type, + "dashboard_ids": [dashboard.id], + }, + ], + }, + "dashboards": { + "count": 1, + "result": [ + { + "id": dashboard.id, + "title": dashboard.dashboard_title, + "slug": dashboard.slug, + "chart_ids": sorted([charts[0].id, charts[1].id]), + } + ], + }, + }, + }, + } + + +@pytest.fixture(autouse=False) +def inject_expected_chart_lineage(request, lineage_test_data): + """ + Injects chart lineage data into test class instance. + """ + dataset = lineage_test_data["dataset"] + database = lineage_test_data["database"] + chart = lineage_test_data["charts"][0] # Use first chart + dashboard = lineage_test_data["dashboard"] + + request.instance.chart_lineage = { + "chart_id": chart.id, + "expected": { + "chart": { + "id": chart.id, + "slice_name": chart.slice_name, + "viz_type": chart.viz_type, + }, + "upstream": { + "dataset": { + "id": dataset.id, + "name": dataset.name, + "schema": dataset.schema, + "table_name": dataset.table_name, + "database_id": database.id, + "database_name": database.database_name, + }, + "database": { + "id": database.id, + "database_name": database.database_name, + "backend": database.backend, + }, + }, + "downstream": { + "dashboards": { + "count": 1, + "result": [ + { + "id": dashboard.id, + "title": dashboard.dashboard_title, + "slug": dashboard.slug, + } + ], + } + }, + }, + } + + +@pytest.fixture(autouse=False) +def inject_expected_dashboard_lineage(request, lineage_test_data): + """ + Injects dashboard lineage data into test class instance. + """ + dataset = lineage_test_data["dataset"] + database = lineage_test_data["database"] + charts = lineage_test_data["charts"] + dashboard = lineage_test_data["dashboard"] + + request.instance.dashboard_lineage = { + "dashboard_id": dashboard.id, + "expected": { + "dashboard": { + "id": dashboard.id, + "title": dashboard.dashboard_title, + "slug": dashboard.slug, + "published": dashboard.published, + }, + "upstream": { + "charts": { + "count": 2, + "result": [ + { + "id": charts[0].id, + "slice_name": charts[0].slice_name, + "viz_type": charts[0].viz_type, + "dataset_id": dataset.id, + }, + { + "id": charts[1].id, + "slice_name": charts[1].slice_name, + "viz_type": charts[1].viz_type, + "dataset_id": dataset.id, + }, + ], + }, + "datasets": { + "count": 1, + "result": [ + { + "id": dataset.id, + "name": dataset.name, + "schema": dataset.schema, + "table_name": dataset.table_name, + "database_id": database.id, + "database_name": database.database_name, + "chart_ids": sorted([charts[0].id, charts[1].id]), + } + ], + }, + "databases": { + "count": 1, + "result": [ + { + "id": database.id, + "database_name": database.database_name, + "backend": database.backend, + } + ], + }, + }, + "downstream": None, + }, + }