Skip to content

Commit

Permalink
[components] Make dg generate code-location --use-editable-dagster in…
Browse files Browse the repository at this point in the history
…clude all dagster libs (#26612)

## Summary & Motivation

Make it so that the editable sources included in `pyproject.toml` when
generating a code location are constructed from a crawl of the dagster
repo rather than hardcoded. Preemptively provides editable sources for
integrations we may add components for.

## How I Tested These Changes

Modified unit tests.
  • Loading branch information
smackesey authored Dec 20, 2024
1 parent 3645506 commit 2cea1b3
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 14 deletions.
37 changes: 23 additions & 14 deletions python_modules/libraries/dagster-dg/dagster_dg/generate.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import json
import os
import textwrap
from pathlib import Path
from typing import Any, Mapping, Optional
from typing import Any, Mapping, Optional, Sequence

import click

Expand Down Expand Up @@ -74,17 +73,27 @@ def get_pyproject_toml_dev_dependencies(use_editable_dagster: bool) -> str:
)


def get_pyproject_toml_uv_sources(editable_dagster_root: str) -> str:
return textwrap.dedent(f"""
[tool.uv.sources]
dagster = {{ path = "{editable_dagster_root}/python_modules/dagster", editable = true }}
dagster-graphql = {{ path = "{editable_dagster_root}/python_modules/dagster-graphql", editable = true }}
dagster-pipes = {{ path = "{editable_dagster_root}/python_modules/dagster-pipes", editable = true }}
dagster-webserver = {{ path = "{editable_dagster_root}/python_modules/dagster-webserver", editable = true }}
dagster-components = {{ path = "{editable_dagster_root}/python_modules/libraries/dagster-components", editable = true }}
dagster-embedded-elt = {{ path = "{editable_dagster_root}/python_modules/libraries/dagster-embedded-elt", editable = true }}
dagster-dbt = {{ path = "{editable_dagster_root}/python_modules/libraries/dagster-dbt", editable = true }}
""")
def get_pyproject_toml_uv_sources(editable_dagster_root: Path) -> str:
lib_lines = [
f'{path.name} = {{ path = "{path}", editable = true }}'
for path in _gather_dagster_packages(editable_dagster_root)
]
return "\n".join(
[
"[tool.uv.sources]",
*lib_lines,
]
)


def _gather_dagster_packages(editable_dagster_root: Path) -> Sequence[Path]:
return [
p.parent
for p in (
*editable_dagster_root.glob("python_modules/dagster*/setup.py"),
*editable_dagster_root.glob("python_modules/libraries/dagster*/setup.py"),
)
]


def generate_code_location(
Expand All @@ -100,7 +109,7 @@ def generate_code_location(
use_editable_dagster=bool(editable_dagster_root)
)
uv_sources = (
get_pyproject_toml_uv_sources(editable_dagster_root) if editable_dagster_root else ""
get_pyproject_toml_uv_sources(Path(editable_dagster_root)) if editable_dagster_root else ""
)

generate_subtree(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,12 @@ def test_generate_code_location_editable_dagster_success(mode: str, monkeypatch)
"path": f"{dagster_git_repo_dir}/python_modules/libraries/dagster-components",
"editable": True,
}
# Check for presence of one random package with no component to ensure we are
# preemptively adding all packages
assert toml["tool"]["uv"]["sources"]["dagstermill"] == {
"path": f"{dagster_git_repo_dir}/python_modules/libraries/dagstermill",
"editable": True,
}


def test_generate_code_location_skip_venv_success() -> None:
Expand Down

0 comments on commit 2cea1b3

Please sign in to comment.