Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Adding config option for filtering models #204

Merged
merged 4 commits into from
May 11, 2023
Merged

Adding config option for filtering models #204

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4949484
1st version
hughhhh May 5, 2023
57a0be8
linting
hughhhh May 5, 2023
a15b991
allow none
hughhhh May 9, 2023
9845f47
Merge branch 'main' into add-config-filter
betodealmeida May 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/preset_cli/api/clients/dbt.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -535,6 +535,7 @@ class ModelSchema(PostelSchema):
unique_id = fields.String(data_key="uniqueId")
tags = fields.List(fields.String())
columns = fields.Raw()
config = fields.Dict(fields.String(), fields.Raw(allow_none=True))


class FilterSchema(PostelSchema):
Expand Down
8 changes: 8 additions & 0 deletions src/preset_cli/cli/superset/sync/dbt/lib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,14 @@ def filter_models(models: List[ModelSchema], condition: str) -> List[ModelSchema
tag = condition.split(":", 1)[1]
return [model for model in models if tag in model["tags"]]

if condition.startswith("config"):
filtered_models = []
config_key, config_value = re.split(r"[.:]", condition)[1:]
for model in models:
if model.get("config", {}).get(config_key) == config_value:
filtered_models.append(model)
return filtered_models

# simple match by name
model_names = {model["name"]: model for model in models}
if condition in model_names:
Expand Down
11 changes: 11 additions & 0 deletions tests/cli/superset/sync/dbt/lib_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -386,6 +386,12 @@ def test_filter_models() -> None:
"unique_id": "model.three",
"depends_on": ["source.zero"],
"children": ["model.two"],
"config": {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like the tests are failing because you need to add config to the other models, even if it's an empty dict.

"materialized": "view",
"persist_docs": {},
"quoting": {},
"column_types": {},
},
}
models: List[ModelSchema] = [one, two, three] # type: ignore

Expand All @@ -406,6 +412,11 @@ def test_filter_models() -> None:
"three",
}

# testing config filtering
assert {
model["name"] for model in filter_models(models, "config.materialized:view")
} == {"three"}

with pytest.raises(NotImplementedError) as excinfo:
filter_models(models, "invalid")
assert str(excinfo.value) == (
Expand Down