Skip to content

Commit 35c9420

Browse files
committed
[dagster-dbt][refactor] Clean up multi-asset creation
1 parent 9ce5277 commit 35c9420

File tree

9 files changed

+261
-347
lines changed

9 files changed

+261
-347
lines changed

python_modules/dagster/dagster/_core/definitions/asset_key.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ def to_python_identifier(self, suffix: Optional[str] = None) -> str:
101101
if suffix is not None:
102102
path.append(suffix)
103103

104-
return "__".join(path).replace("-", "_")
104+
return "__".join(path).replace("-", "_").replace(".", "_")
105105

106106
@staticmethod
107107
def from_user_string(asset_key_string: str) -> "AssetKey":

python_modules/libraries/dagster-dbt/dagster_dbt/asset_decorator.py

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from dagster_dbt.asset_utils import (
1515
DAGSTER_DBT_EXCLUDE_METADATA_KEY,
1616
DAGSTER_DBT_SELECT_METADATA_KEY,
17-
build_dbt_multi_asset_args,
17+
build_dbt_specs,
1818
)
1919
from dagster_dbt.dagster_dbt_translator import DagsterDbtTranslator, validate_translator
2020
from dagster_dbt.dbt_manifest import DbtManifestParam, validate_manifest
@@ -302,14 +302,9 @@ def partitionshop_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource
302302
dagster_dbt_translator = validate_translator(dagster_dbt_translator or DagsterDbtTranslator())
303303
manifest = validate_manifest(manifest)
304304

305-
(
306-
deps,
307-
outs,
308-
internal_asset_deps,
309-
check_specs,
310-
) = build_dbt_multi_asset_args(
305+
specs, check_specs = build_dbt_specs(
306+
translator=dagster_dbt_translator,
311307
manifest=manifest,
312-
dagster_dbt_translator=dagster_dbt_translator,
313308
select=select,
314309
exclude=exclude or "",
315310
io_manager_key=io_manager_key,
@@ -342,15 +337,13 @@ def partitionshop_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource
342337
backfill_policy = BackfillPolicy.single_run()
343338

344339
return multi_asset(
345-
outs=outs,
346340
name=name,
347-
internal_asset_deps=internal_asset_deps,
348-
deps=deps,
341+
specs=specs,
342+
check_specs=check_specs,
343+
can_subset=True,
349344
required_resource_keys=required_resource_keys,
350345
partitions_def=partitions_def,
351-
can_subset=True,
352346
op_tags=resolved_op_tags,
353-
check_specs=check_specs,
354347
backfill_policy=backfill_policy,
355348
retry_policy=retry_policy,
356349
)

python_modules/libraries/dagster-dbt/dagster_dbt/asset_specs.py

Lines changed: 7 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
11
from typing import Optional, Sequence
22

3-
import dagster._check as check
4-
from dagster import AssetDep, AssetKey, AssetSpec
3+
from dagster import AssetSpec
54
from dagster._annotations import experimental
6-
from dagster._core.storage.tags import KIND_PREFIX
75

8-
from dagster_dbt.asset_utils import build_dbt_multi_asset_args
6+
from dagster_dbt.asset_utils import build_dbt_specs
97
from dagster_dbt.dagster_dbt_translator import DagsterDbtTranslator, validate_translator
108
from dagster_dbt.dbt_manifest import DbtManifestParam, validate_manifest
119
from dagster_dbt.dbt_project import DbtProject
@@ -43,33 +41,17 @@ def build_dbt_asset_specs(
4341
manifest = validate_manifest(manifest)
4442
dagster_dbt_translator = validate_translator(dagster_dbt_translator or DagsterDbtTranslator())
4543

46-
(
47-
_,
48-
outs,
49-
internal_asset_deps,
50-
_,
51-
) = build_dbt_multi_asset_args(
44+
specs, _ = build_dbt_specs(
5245
manifest=manifest,
53-
dagster_dbt_translator=dagster_dbt_translator,
46+
translator=dagster_dbt_translator,
5447
select=select,
5548
exclude=exclude or "",
5649
io_manager_key=None,
5750
project=project,
5851
)
5952

60-
specs = [
61-
asset_out.to_spec(
62-
key=check.inst(asset_out.key, AssetKey),
63-
deps=[AssetDep(asset=dep) for dep in internal_asset_deps.get(output_name, set())],
64-
additional_tags={f"{KIND_PREFIX}dbt": ""},
65-
partitions_def=None,
66-
)
53+
return [
6754
# Allow specs to be represented as external assets by adhering to external asset invariants.
68-
._replace(
69-
skippable=False,
70-
code_version=None,
71-
)
72-
for output_name, asset_out in outs.items()
55+
spec.replace_attributes(skippable=False, code_version=None)
56+
for spec in specs
7357
]
74-
75-
return specs

0 commit comments

Comments
 (0)