diff --git a/src/preset_cli/cli/superset/sync/native/command.py b/src/preset_cli/cli/superset/sync/native/command.py index 25e9700a..122c14ed 100644 --- a/src/preset_cli/cli/superset/sync/native/command.py +++ b/src/preset_cli/cli/superset/sync/native/command.py @@ -11,7 +11,7 @@ from io import BytesIO from pathlib import Path from types import ModuleType -from typing import Any, Dict, Iterator, Tuple +from typing import Any, Dict, Iterator, Set, Tuple from zipfile import ZipFile import backoff @@ -272,7 +272,7 @@ def import_resources_individually( ("databases", lambda config: []), ("datasets", lambda config: [config["database_uuid"]]), ("charts", lambda config: [config["dataset_uuid"]]), - ("dashboards", get_charts_uuids), + ("dashboards", get_dashboard_related_uuids), ] related_configs: Dict[str, Dict[Path, AssetConfig]] = {} for resource_name, get_related_uuids in imports: @@ -297,6 +297,16 @@ def import_resources_individually( os.unlink(checkpoint_path) +def get_dashboard_related_uuids(config: AssetConfig) -> Iterator[str]: + """ + Extract dataset and chart UUID from a dashboard config. + """ + for uuid in get_charts_uuids(config): + yield uuid + for uuid in get_dataset_filter_uuids(config): + yield uuid + + def get_charts_uuids(config: AssetConfig) -> Iterator[str]: """ Extract chart UUID from a dashboard config. @@ -310,6 +320,19 @@ def get_charts_uuids(config: AssetConfig) -> Iterator[str]: yield child["meta"]["uuid"] +def get_dataset_filter_uuids(config: AssetConfig) -> Set[str]: + """ + Extract dataset UUID for datasets that are used in dashboard filters. + """ + dataset_uuids = set() + for filter_config in config["metadata"].get("native_filter_configuration", []): + for target in filter_config["targets"]: + if uuid := target.get("datasetUuid"): + if uuid not in dataset_uuids: + dataset_uuids.add(uuid) + return dataset_uuids + + def verify_db_connectivity(config: Dict[str, Any]) -> None: """ Test if we can connect to a given database. diff --git a/tests/cli/superset/sync/native/command_test.py b/tests/cli/superset/sync/native/command_test.py index 6596b985..9ababc63 100644 --- a/tests/cli/superset/sync/native/command_test.py +++ b/tests/cli/superset/sync/native/command_test.py @@ -732,7 +732,10 @@ def test_verify_db_connectivity_error(mocker: MockerFixture) -> None: ) -def test_native_split(mocker: MockerFixture, fs: FakeFilesystem) -> None: +def test_native_split( # pylint: disable=too-many-locals + mocker: MockerFixture, + fs: FakeFilesystem, +) -> None: """ Test the ``native`` command with split imports. """ @@ -768,8 +771,84 @@ def test_native_split(mocker: MockerFixture, fs: FakeFilesystem) -> None: }, }, }, + "metadata": {}, "uuid": "4", } + dataset_filter_config = { + "table_name": "filter_test", + "is_managed_externally": False, + "database_uuid": "1", + "uuid": "5", + } + dashboard_with_filter_config = { + "dashboard_title": "Some dashboard", + "is_managed_externally": False, + "position": {}, + "metadata": { + "native_filter_configuration": [ + { + "type": "NATIVE_FILTER", + "targets": [ + { + "column": "some_column", + "datasetUuid": "5", + }, + ], + }, + { + "type": "NATIVE_FILTER", + "targets": [ + { + "column": "other_column", + "datasetUuid": "5", + }, + ], + }, + { + "type": "NATIVE_FILTER", + "targets": [ + { + "column": "blah", + "datasetUuid": "2", + }, + ], + }, + ], + }, + "uuid": "6", + } + dashboard_with_temporal_filter = { + "dashboard_title": "Some dashboard", + "is_managed_externally": False, + "position": {}, + "metadata": { + "native_filter_configuration": [ + { + "type": "NATIVE_FILTER", + "targets": [], + }, + ], + }, + "uuid": "6", + } + dashboard_deleted_dataset = { + "dashboard_title": "Some dashboard", + "is_managed_externally": False, + "position": {}, + "metadata": { + "native_filter_configuration": [ + { + "type": "NATIVE_FILTER", + "targets": [ + { + "column": "some_column", + }, + ], + }, + ], + }, + "uuid": "6", + } fs.create_file( root / "databases/gsheets.yaml", contents=yaml.dump(database_config), @@ -786,6 +865,22 @@ def test_native_split(mocker: MockerFixture, fs: FakeFilesystem) -> None: root / "dashboards/dashboard.yaml", contents=yaml.dump(dashboard_config), ) + fs.create_file( + root / "datasets/gsheets/filter_test.yaml", + contents=yaml.dump(dataset_filter_config), + ) + fs.create_file( + root / "dashboards/dashboard_with_filter_config.yaml", + contents=yaml.dump(dashboard_with_filter_config), + ) + fs.create_file( + root / "dashboards/dashboard_with_temporal_filter.yaml", + contents=yaml.dump(dashboard_with_temporal_filter), + ) + fs.create_file( + root / "dashboards/dashboard_deleted_dataset.yaml", + contents=yaml.dump(dashboard_deleted_dataset), + ) SupersetClient = mocker.patch( "preset_cli.cli.superset.sync.native.command.SupersetClient", @@ -812,6 +907,16 @@ def test_native_split(mocker: MockerFixture, fs: FakeFilesystem) -> None: client, False, ), + mock.call( + { + "bundle/datasets/gsheets/filter_test.yaml": yaml.dump( + dataset_filter_config, + ), + "bundle/databases/gsheets.yaml": yaml.dump(database_config), + }, + client, + False, + ), mock.call( { "bundle/datasets/gsheets/test.yaml": yaml.dump(dataset_config), @@ -829,6 +934,38 @@ def test_native_split(mocker: MockerFixture, fs: FakeFilesystem) -> None: client, False, ), + mock.call( + { + "bundle/dashboards/dashboard_deleted_dataset.yaml": yaml.dump( + dashboard_deleted_dataset, + ), + }, + client, + False, + ), + mock.call( + { + "bundle/dashboards/dashboard_with_temporal_filter.yaml": yaml.dump( + dashboard_with_temporal_filter, + ), + }, + client, + False, + ), + mock.call( + { + "bundle/dashboards/dashboard_with_filter_config.yaml": yaml.dump( + dashboard_with_filter_config, + ), + "bundle/datasets/gsheets/filter_test.yaml": yaml.dump( + dataset_filter_config, + ), + "bundle/datasets/gsheets/test.yaml": yaml.dump(dataset_config), + "bundle/databases/gsheets.yaml": yaml.dump(database_config), + }, + client, + False, + ), mock.call( { "bundle/dashboards/dashboard.yaml": yaml.dump(dashboard_config),