diff --git a/python_modules/libraries/dagster-dg/dagster_dg/cli/__init__.py b/python_modules/libraries/dagster-dg/dagster_dg/cli/__init__.py index 4d32561d772b3..8302e5e659877 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/cli/__init__.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/cli/__init__.py @@ -4,9 +4,10 @@ import click from dagster_dg.cache import DgCache -from dagster_dg.cli.generate import generate_cli -from dagster_dg.cli.info import info_cli -from dagster_dg.cli.list import list_cli +from dagster_dg.cli.code_location import code_location_group +from dagster_dg.cli.component import component_group +from dagster_dg.cli.component_type import component_type_group +from dagster_dg.cli.deployment import deployment_group from dagster_dg.config import DgConfig, set_config_on_cli_context from dagster_dg.context import ( DgContext, @@ -22,9 +23,10 @@ def create_dg_cli(): commands = { - "generate": generate_cli, - "info": info_cli, - "list": list_cli, + "code-location": code_location_group, + "deployment": deployment_group, + "component": component_group, + "component-type": component_type_group, } # Defaults are defined on the DgConfig object. diff --git a/python_modules/libraries/dagster-dg/dagster_dg/cli/code_location.py b/python_modules/libraries/dagster-dg/dagster_dg/cli/code_location.py new file mode 100644 index 0000000000000..bed03bd9f804c --- /dev/null +++ b/python_modules/libraries/dagster-dg/dagster_dg/cli/code_location.py @@ -0,0 +1,116 @@ +import os +import sys +from pathlib import Path +from typing import Optional + +import click + +from dagster_dg.context import DeploymentDirectoryContext, DgContext, is_inside_deployment_directory +from dagster_dg.generate import generate_code_location +from dagster_dg.utils import DgClickCommand, DgClickGroup + + +@click.group(name="code-location", cls=DgClickGroup) +def code_location_group(): + """Commands for operating code location directories.""" + + +# ######################## +# ##### GENERATE +# ######################## + + +@code_location_group.command(name="generate", cls=DgClickCommand) +@click.argument("name", type=str) +@click.option( + "--use-editable-dagster", + type=str, + flag_value="TRUE", + is_flag=False, + default=None, + help=( + "Install Dagster package dependencies from a local Dagster clone. Accepts a path to local Dagster clone root or" + " may be set as a flag (no value is passed). If set as a flag," + " the location of the local Dagster clone will be read from the `DAGSTER_GIT_REPO_DIR` environment variable." + ), +) +@click.option( + "--skip-venv", + is_flag=True, + default=False, + help="Do not create a virtual environment for the code location.", +) +@click.pass_context +def code_location_generate_command( + cli_context: click.Context, name: str, use_editable_dagster: Optional[str], skip_venv: bool +) -> None: + """Generate a Dagster code location file structure and a uv-managed virtual environment scoped + to the code location. + + This command can be run inside or outside of a deployment directory. If run inside a deployment, + the code location will be created within the deployment directory's code location directory. + + The code location file structure defines a Python package with some pre-existing internal + structure: + + ├── + │ ├── __init__.py + │ ├── components + │ ├── definitions.py + │ └── lib + │ └── __init__.py + ├── _tests + │ └── __init__.py + └── pyproject.toml + + The `.components` directory holds components (which can be created with `dg generate + component`). The `.lib` directory holds custom component types scoped to the code + location (which can be created with `dg component-type generate`). + """ + dg_context = DgContext.from_cli_context(cli_context) + if is_inside_deployment_directory(Path.cwd()): + context = DeploymentDirectoryContext.from_path(Path.cwd(), dg_context) + if context.has_code_location(name): + click.echo(click.style(f"A code location named {name} already exists.", fg="red")) + sys.exit(1) + code_location_path = context.code_location_root_path / name + else: + code_location_path = Path.cwd() / name + + if use_editable_dagster == "TRUE": + if not os.environ.get("DAGSTER_GIT_REPO_DIR"): + click.echo( + click.style( + "The `--use-editable-dagster` flag requires the `DAGSTER_GIT_REPO_DIR` environment variable to be set.", + fg="red", + ) + ) + sys.exit(1) + editable_dagster_root = os.environ["DAGSTER_GIT_REPO_DIR"] + elif use_editable_dagster: # a string value was passed + editable_dagster_root = use_editable_dagster + else: + editable_dagster_root = None + + generate_code_location(code_location_path, dg_context, editable_dagster_root, skip_venv) + + +# ######################## +# ##### LIST +# ######################## + + +@code_location_group.command(name="list", cls=DgClickCommand) +@click.pass_context +def code_location_list_command(cli_context: click.Context) -> None: + """List code locations in the current deployment.""" + dg_context = DgContext.from_cli_context(cli_context) + if not is_inside_deployment_directory(Path.cwd()): + click.echo( + click.style("This command must be run inside a Dagster deployment directory.", fg="red") + ) + sys.exit(1) + + context = DeploymentDirectoryContext.from_path(Path.cwd(), dg_context) + for code_location in context.get_code_location_names(): + click.echo(code_location) diff --git a/python_modules/libraries/dagster-dg/dagster_dg/cli/generate.py b/python_modules/libraries/dagster-dg/dagster_dg/cli/component.py similarity index 55% rename from python_modules/libraries/dagster-dg/dagster_dg/cli/generate.py rename to python_modules/libraries/dagster-dg/dagster_dg/cli/component.py index 49fb5f822f26e..a39592c51fa84 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/cli/generate.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/cli/component.py @@ -1,4 +1,3 @@ -import os import sys from pathlib import Path from typing import Any, Mapping, Optional @@ -9,17 +8,10 @@ from dagster_dg.component import RemoteComponentType from dagster_dg.context import ( CodeLocationDirectoryContext, - DeploymentDirectoryContext, DgContext, is_inside_code_location_directory, - is_inside_deployment_directory, -) -from dagster_dg.generate import ( - generate_code_location, - generate_component_instance, - generate_component_type, - generate_deployment, ) +from dagster_dg.generate import generate_component_instance from dagster_dg.utils import ( DgClickCommand, DgClickGroup, @@ -28,131 +20,17 @@ ) -@click.group(name="generate", cls=DgClickGroup) -def generate_cli() -> None: - """Commands for generating Dagster components and related entities.""" - - -@generate_cli.command(name="deployment", cls=DgClickCommand) -@click.argument("path", type=Path) -def generate_deployment_command(path: Path) -> None: - """Generate a Dagster deployment file structure. - - The deployment file structure includes a directory for code locations and configuration files - for deploying to Dagster Plus. - """ - dir_abspath = os.path.abspath(path) - if os.path.exists(dir_abspath): - click.echo( - click.style(f"A file or directory at {dir_abspath} already exists. ", fg="red") - + "\nPlease delete the contents of this path or choose another location." - ) - sys.exit(1) - generate_deployment(path) - - -@generate_cli.command(name="code-location", cls=DgClickCommand) -@click.argument("name", type=str) -@click.option( - "--use-editable-dagster", - type=str, - flag_value="TRUE", - is_flag=False, - default=None, - help=( - "Install Dagster package dependencies from a local Dagster clone. Accepts a path to local Dagster clone root or" - " may be set as a flag (no value is passed). If set as a flag," - " the location of the local Dagster clone will be read from the `DAGSTER_GIT_REPO_DIR` environment variable." - ), -) -@click.option( - "--skip-venv", - is_flag=True, - default=False, - help="Do not create a virtual environment for the code location.", -) -@click.pass_context -def generate_code_location_command( - cli_context: click.Context, name: str, use_editable_dagster: Optional[str], skip_venv: bool -) -> None: - """Generate a Dagster code location file structure and a uv-managed virtual environment scoped - to the code location. - - This command can be run inside or outside of a deployment directory. If run inside a deployment, - the code location will be created within the deployment directory's code location directory. - - The code location file structure defines a Python package with some pre-existing internal - structure: - - ├── - │ ├── __init__.py - │ ├── components - │ ├── definitions.py - │ └── lib - │ └── __init__.py - ├── _tests - │ └── __init__.py - └── pyproject.toml - - The `.components` directory holds components (which can be created with `dg generate - component`). The `.lib` directory holds custom component types scoped to the code - location (which can be created with `dg generate component-type`). - """ - dg_context = DgContext.from_cli_context(cli_context) - if is_inside_deployment_directory(Path.cwd()): - context = DeploymentDirectoryContext.from_path(Path.cwd(), dg_context) - if context.has_code_location(name): - click.echo(click.style(f"A code location named {name} already exists.", fg="red")) - sys.exit(1) - code_location_path = context.code_location_root_path / name - else: - code_location_path = Path.cwd() / name - - if use_editable_dagster == "TRUE": - if not os.environ.get("DAGSTER_GIT_REPO_DIR"): - click.echo( - click.style( - "The `--use-editable-dagster` flag requires the `DAGSTER_GIT_REPO_DIR` environment variable to be set.", - fg="red", - ) - ) - sys.exit(1) - editable_dagster_root = os.environ["DAGSTER_GIT_REPO_DIR"] - elif use_editable_dagster: # a string value was passed - editable_dagster_root = use_editable_dagster - else: - editable_dagster_root = None - - generate_code_location(code_location_path, dg_context, editable_dagster_root, skip_venv) - +@click.group(name="component", cls=DgClickGroup) +def component_group(): + """Commands for operating on components.""" -@generate_cli.command(name="component-type", cls=DgClickCommand) -@click.argument("name", type=str) -@click.pass_context -def generate_component_type_command(cli_context: click.Context, name: str) -> None: - """Generate a scaffold of a custom Dagster component type. - - This command must be run inside a Dagster code location directory. The component type scaffold - will be generated in submodule `.lib.`. - """ - dg_context = DgContext.from_cli_context(cli_context) - if not is_inside_code_location_directory(Path.cwd()): - click.echo( - click.style( - "This command must be run inside a Dagster code location directory.", fg="red" - ) - ) - sys.exit(1) - context = CodeLocationDirectoryContext.from_path(Path.cwd(), dg_context) - full_component_name = f"{context.name}.{name}" - if context.has_component_type(full_component_name): - click.echo(click.style(f"A component type named `{name}` already exists.", fg="red")) - sys.exit(1) - generate_component_type(context, name) +# ######################## +# ##### GENERATE +# ######################## -# The `dg generate component` command is special because its subcommands are dynamically generated +# The `dg component generate` command is special because its subcommands are dynamically generated # from the registered component types in the code location. Because the registered component types # depend on the built-in component library we are using, we cannot resolve them until we have the # built-in component library, which can be set via a global option, e.g.: @@ -160,7 +38,7 @@ def generate_component_type_command(cli_context: click.Context, name: str) -> No # dg --builtin-component-lib dagster_components.test ... # # To handle this, we define a custom click.Group subclass that loads the commands on demand. -class GenerateComponentGroup(DgClickGroup): +class ComponentGenerateGroup(DgClickGroup): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._commands_defined = False @@ -189,16 +67,16 @@ def _define_commands(self, cli_context: click.Context) -> None: context = CodeLocationDirectoryContext.from_path(Path.cwd(), app_context) for key, component_type in context.iter_component_types(): - command = _create_generate_component_subcommand(key, component_type) + command = _create_component_generate_subcommand(key, component_type) self.add_command(command) -@generate_cli.group(name="component", cls=GenerateComponentGroup) -def generate_component_group() -> None: +@component_group.group(name="generate", cls=ComponentGenerateGroup) +def component_generate_group() -> None: """Generate a scaffold of a Dagster component.""" -def _create_generate_component_subcommand( +def _create_component_generate_subcommand( component_key: str, component_type: RemoteComponentType ) -> DgClickCommand: @click.command(name=component_key, cls=DgClickCommand) @@ -226,11 +104,11 @@ def generate_component_command( (1) Passing a single --json-params option with a JSON string of parameters. For example: - dg generate component foo.bar my_component --json-params '{{"param1": "value", "param2": "value"}}'`. + dg component generate foo.bar my_component --json-params '{{"param1": "value", "param2": "value"}}'`. (2) Passing each parameter as an option. For example: - dg generate component foo.bar my_component --param1 value1 --param2 value2` + dg component generate foo.bar my_component --param1 value1 --param2 value2` It is an error to pass both --json-params and key-value pairs as options. """ @@ -299,3 +177,26 @@ def generate_component_command( generate_component_command.params.append(option) return generate_component_command + + +# ######################## +# ##### LIST +# ######################## + + +@component_group.command(name="list", cls=DgClickCommand) +@click.pass_context +def component_list_command(cli_context: click.Context) -> None: + """List Dagster component instances defined in the current code location.""" + dg_context = DgContext.from_cli_context(cli_context) + if not is_inside_code_location_directory(Path.cwd()): + click.echo( + click.style( + "This command must be run inside a Dagster code location directory.", fg="red" + ) + ) + sys.exit(1) + + context = CodeLocationDirectoryContext.from_path(Path.cwd(), dg_context) + for component_name in context.get_component_instance_names(): + click.echo(component_name) diff --git a/python_modules/libraries/dagster-dg/dagster_dg/cli/info.py b/python_modules/libraries/dagster-dg/dagster_dg/cli/component_type.py similarity index 57% rename from python_modules/libraries/dagster-dg/dagster_dg/cli/info.py rename to python_modules/libraries/dagster-dg/dagster_dg/cli/component_type.py index 4152567a74611..b222d3f921e78 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/cli/info.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/cli/component_type.py @@ -10,25 +10,58 @@ DgContext, is_inside_code_location_directory, ) +from dagster_dg.generate import generate_component_type from dagster_dg.utils import DgClickCommand, DgClickGroup -@click.group(name="info", cls=DgClickGroup) -def info_cli(): - """Commands for listing Dagster components and related entities.""" +@click.group(name="component-type", cls=DgClickGroup) +def component_type_group(): + """Commands for operating on components types.""" -def _serialize_json_schema(schema: Mapping[str, Any]) -> str: - return json.dumps(schema, indent=4) +# ######################## +# ##### GENERATE +# ######################## + + +@component_type_group.command(name="generate", cls=DgClickCommand) +@click.argument("name", type=str) +@click.pass_context +def component_type_generate_command(cli_context: click.Context, name: str) -> None: + """Generate a scaffold of a custom Dagster component type. + + This command must be run inside a Dagster code location directory. The component type scaffold + will be generated in submodule `.lib.`. + """ + dg_context = DgContext.from_cli_context(cli_context) + if not is_inside_code_location_directory(Path.cwd()): + click.echo( + click.style( + "This command must be run inside a Dagster code location directory.", fg="red" + ) + ) + sys.exit(1) + context = CodeLocationDirectoryContext.from_path(Path.cwd(), dg_context) + full_component_name = f"{context.name}.{name}" + if context.has_component_type(full_component_name): + click.echo(click.style(f"A component type named `{name}` already exists.", fg="red")) + sys.exit(1) + + generate_component_type(context, name) + + +# ######################## +# ##### INFO +# ######################## -@info_cli.command(name="component-type", cls=DgClickCommand) +@component_type_group.command(name="info", cls=DgClickCommand) @click.argument("component_type", type=str) @click.option("--description", is_flag=True, default=False) @click.option("--generate-params-schema", is_flag=True, default=False) @click.option("--component-params-schema", is_flag=True, default=False) @click.pass_context -def info_component_type_command( +def component_type_info_command( cli_context: click.Context, component_type: str, description: bool, @@ -91,3 +124,32 @@ def info_component_type_command( if component_type_metadata.component_params_schema: click.echo("\nComponent params schema:\n") click.echo(_serialize_json_schema(component_type_metadata.component_params_schema)) + + +def _serialize_json_schema(schema: Mapping[str, Any]) -> str: + return json.dumps(schema, indent=4) + + +# ######################## +# ##### LIST +# ######################## + + +@component_type_group.command(name="list", cls=DgClickCommand) +@click.pass_context +def component_type_list(cli_context: click.Context) -> None: + """List registered Dagster components in the current code location environment.""" + dg_context = DgContext.from_cli_context(cli_context) + if not is_inside_code_location_directory(Path.cwd()): + click.echo( + click.style( + "This command must be run inside a Dagster code location directory.", fg="red" + ) + ) + sys.exit(1) + + context = CodeLocationDirectoryContext.from_path(Path.cwd(), dg_context) + for key, component_type in context.iter_component_types(): + click.echo(key) + if component_type.summary: + click.echo(f" {component_type.summary}") diff --git a/python_modules/libraries/dagster-dg/dagster_dg/cli/deployment.py b/python_modules/libraries/dagster-dg/dagster_dg/cli/deployment.py new file mode 100644 index 0000000000000..c20e1f31a3cec --- /dev/null +++ b/python_modules/libraries/dagster-dg/dagster_dg/cli/deployment.py @@ -0,0 +1,36 @@ +import os +import sys +from pathlib import Path + +import click + +from dagster_dg.generate import generate_deployment +from dagster_dg.utils import DgClickCommand, DgClickGroup + + +@click.group(name="deployment", cls=DgClickGroup) +def deployment_group(): + """Commands for operating on deployment directories.""" + + +# ######################## +# ##### GENERATE +# ######################## + + +@deployment_group.command(name="generate", cls=DgClickCommand) +@click.argument("path", type=Path) +def deployment_generate_command(path: Path) -> None: + """Generate a Dagster deployment file structure. + + The deployment file structure includes a directory for code locations and configuration files + for deploying to Dagster Plus. + """ + dir_abspath = os.path.abspath(path) + if os.path.exists(dir_abspath): + click.echo( + click.style(f"A file or directory at {dir_abspath} already exists. ", fg="red") + + "\nPlease delete the contents of this path or choose another location." + ) + sys.exit(1) + generate_deployment(path) diff --git a/python_modules/libraries/dagster-dg/dagster_dg/cli/list.py b/python_modules/libraries/dagster-dg/dagster_dg/cli/list.py deleted file mode 100644 index fa8679f3dc728..0000000000000 --- a/python_modules/libraries/dagster-dg/dagster_dg/cli/list.py +++ /dev/null @@ -1,72 +0,0 @@ -import sys -from pathlib import Path - -import click - -from dagster_dg.context import ( - CodeLocationDirectoryContext, - DeploymentDirectoryContext, - DgContext, - is_inside_code_location_directory, - is_inside_deployment_directory, -) -from dagster_dg.utils import DgClickCommand, DgClickGroup - - -@click.group(name="list", cls=DgClickGroup) -def list_cli(): - """Commands for listing Dagster components and related entities.""" - - -@list_cli.command(name="code-locations", cls=DgClickCommand) -@click.pass_context -def list_code_locations_command(cli_context: click.Context) -> None: - """List code locations in the current deployment.""" - dg_context = DgContext.from_cli_context(cli_context) - if not is_inside_deployment_directory(Path.cwd()): - click.echo( - click.style("This command must be run inside a Dagster deployment directory.", fg="red") - ) - sys.exit(1) - - context = DeploymentDirectoryContext.from_path(Path.cwd(), dg_context) - for code_location in context.get_code_location_names(): - click.echo(code_location) - - -@list_cli.command(name="component-types", cls=DgClickCommand) -@click.pass_context -def list_component_types_command(cli_context: click.Context) -> None: - """List registered Dagster components in the current code location environment.""" - dg_context = DgContext.from_cli_context(cli_context) - if not is_inside_code_location_directory(Path.cwd()): - click.echo( - click.style( - "This command must be run inside a Dagster code location directory.", fg="red" - ) - ) - sys.exit(1) - - context = CodeLocationDirectoryContext.from_path(Path.cwd(), dg_context) - for key, component_type in context.iter_component_types(): - click.echo(key) - if component_type.summary: - click.echo(f" {component_type.summary}") - - -@list_cli.command(name="components", cls=DgClickCommand) -@click.pass_context -def list_components_command(cli_context: click.Context) -> None: - """List Dagster component instances defined in the current code location.""" - dg_context = DgContext.from_cli_context(cli_context) - if not is_inside_code_location_directory(Path.cwd()): - click.echo( - click.style( - "This command must be run inside a Dagster code location directory.", fg="red" - ) - ) - sys.exit(1) - - context = CodeLocationDirectoryContext.from_path(Path.cwd(), dg_context) - for component_name in context.get_component_instance_names(): - click.echo(component_name) diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_code_location_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_code_location_commands.py new file mode 100644 index 0000000000000..c144fc9357726 --- /dev/null +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_code_location_commands.py @@ -0,0 +1,169 @@ +import textwrap +from pathlib import Path + +import pytest +import tomli +from dagster_dg.utils import discover_git_root, ensure_dagster_dg_tests_import, pushd + +ensure_dagster_dg_tests_import() + +from dagster_dg_tests.utils import ( + ProxyRunner, + assert_runner_result, + isolated_example_deployment_foo, +) + +# ######################## +# ##### GENERATE +# ######################## + + +def test_code_location_generate_inside_deployment_success() -> None: + # Don't use the test component lib because it is not present in published dagster-components, + # which this test is currently accessing since we are not doing an editable install. + with ( + ProxyRunner.test(use_test_component_lib=False) as runner, + isolated_example_deployment_foo(runner), + ): + result = runner.invoke("code-location", "generate", "bar") + assert_runner_result(result) + assert Path("code_locations/bar").exists() + assert Path("code_locations/bar/bar").exists() + assert Path("code_locations/bar/bar/lib").exists() + assert Path("code_locations/bar/bar/components").exists() + assert Path("code_locations/bar/bar_tests").exists() + assert Path("code_locations/bar/pyproject.toml").exists() + + # Check venv created + assert Path("code_locations/bar/.venv").exists() + assert Path("code_locations/bar/uv.lock").exists() + + with open("code_locations/bar/pyproject.toml") as f: + toml = tomli.loads(f.read()) + + # No tool.uv.sources added without --use-editable-dagster + assert "uv" not in toml["tool"] + + # Check cache was populated + with pushd("code_locations/bar"): + result = runner.invoke("--verbose", "component-type", "list") + assert "CACHE [hit]" in result.output + + +def test_code_location_generate_outside_deployment_success() -> None: + # Don't use the test component lib because it is not present in published dagster-components, + # which this test is currently accessing since we are not doing an editable install. + with ProxyRunner.test(use_test_component_lib=False) as runner, runner.isolated_filesystem(): + result = runner.invoke("code-location", "generate", "bar") + assert_runner_result(result) + assert Path("bar").exists() + assert Path("bar/bar").exists() + assert Path("bar/bar/lib").exists() + assert Path("bar/bar/components").exists() + assert Path("bar/bar_tests").exists() + assert Path("bar/pyproject.toml").exists() + + # Check venv created + assert Path("bar/.venv").exists() + assert Path("bar/uv.lock").exists() + + +@pytest.mark.parametrize("mode", ["env_var", "arg"]) +def test_code_location_generate_editable_dagster_success(mode: str, monkeypatch) -> None: + dagster_git_repo_dir = discover_git_root(Path(__file__)) + if mode == "env_var": + monkeypatch.setenv("DAGSTER_GIT_REPO_DIR", str(dagster_git_repo_dir)) + editable_args = ["--use-editable-dagster", "--"] + else: + editable_args = ["--use-editable-dagster", str(dagster_git_repo_dir)] + with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner): + result = runner.invoke("code-location", "generate", *editable_args, "bar") + assert_runner_result(result) + assert Path("code_locations/bar").exists() + assert Path("code_locations/bar/pyproject.toml").exists() + with open("code_locations/bar/pyproject.toml") as f: + toml = tomli.loads(f.read()) + assert toml["tool"]["uv"]["sources"]["dagster"] == { + "path": f"{dagster_git_repo_dir}/python_modules/dagster", + "editable": True, + } + assert toml["tool"]["uv"]["sources"]["dagster-pipes"] == { + "path": f"{dagster_git_repo_dir}/python_modules/dagster-pipes", + "editable": True, + } + assert toml["tool"]["uv"]["sources"]["dagster-webserver"] == { + "path": f"{dagster_git_repo_dir}/python_modules/dagster-webserver", + "editable": True, + } + assert toml["tool"]["uv"]["sources"]["dagster-components"] == { + "path": f"{dagster_git_repo_dir}/python_modules/libraries/dagster-components", + "editable": True, + } + # Check for presence of one random package with no component to ensure we are + # preemptively adding all packages + assert toml["tool"]["uv"]["sources"]["dagstermill"] == { + "path": f"{dagster_git_repo_dir}/python_modules/libraries/dagstermill", + "editable": True, + } + + +def test_code_location_generate_skip_venv_success() -> None: + # Don't use the test component lib because it is not present in published dagster-components, + # which this test is currently accessing since we are not doing an editable install. + with ProxyRunner.test() as runner, runner.isolated_filesystem(): + result = runner.invoke("code-location", "generate", "--skip-venv", "bar") + assert_runner_result(result) + assert Path("bar").exists() + assert Path("bar/bar").exists() + assert Path("bar/bar/lib").exists() + assert Path("bar/bar/components").exists() + assert Path("bar/bar_tests").exists() + assert Path("bar/pyproject.toml").exists() + + # Check venv created + assert not Path("bar/.venv").exists() + assert not Path("bar/uv.lock").exists() + + +def test_code_location_generate_editable_dagster_no_env_var_no_value_fails(monkeypatch) -> None: + monkeypatch.setenv("DAGSTER_GIT_REPO_DIR", "") + with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner): + result = runner.invoke("code-location", "generate", "--use-editable-dagster", "--", "bar") + assert_runner_result(result, exit_0=False) + assert "requires the `DAGSTER_GIT_REPO_DIR`" in result.output + + +def test_code_location_generate_already_exists_fails() -> None: + with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner): + result = runner.invoke("code-location", "generate", "bar", "--skip-venv") + assert_runner_result(result) + result = runner.invoke("code-location", "generate", "bar", "--skip-venv") + assert_runner_result(result, exit_0=False) + assert "already exists" in result.output + + +# ######################## +# ##### LIST +# ######################## + + +def test_code_location_list_success(): + with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner): + runner.invoke("code-location", "generate", "foo") + runner.invoke("code-location", "generate", "bar") + result = runner.invoke("code-location", "list") + assert_runner_result(result) + assert ( + result.output.strip() + == textwrap.dedent(""" + bar + foo + """).strip() + ) + + +def test_code_location_list_outside_deployment_fails() -> None: + with ProxyRunner.test() as runner, runner.isolated_filesystem(): + result = runner.invoke("code-location", "list") + assert_runner_result(result, exit_0=False) + assert "must be run inside a Dagster deployment directory" in result.output diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_component_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_component_commands.py new file mode 100644 index 0000000000000..c8e2038d0e18c --- /dev/null +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_component_commands.py @@ -0,0 +1,240 @@ +import json +import subprocess +import textwrap +from pathlib import Path + +import pytest +from dagster_dg.utils import ensure_dagster_dg_tests_import + +ensure_dagster_dg_tests_import() + +from dagster_dg_tests.utils import ( + ProxyRunner, + assert_runner_result, + isolated_example_code_location_bar, + isolated_example_deployment_foo, +) + +# ######################## +# ##### GENERATE +# ######################## + + +def test_component_generate_dynamic_subcommand_generation() -> None: + with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner): + result = runner.invoke("component", "generate", "--help") + assert_runner_result(result) + assert ( + textwrap.dedent(""" + Commands: + dagster_components.test.all_metadata_empty_asset + dagster_components.test.simple_asset + dagster_components.test.simple_pipes_script_asset + """).strip() + in result.output + ) + + +@pytest.mark.parametrize("in_deployment", [True, False]) +def test_component_generate_no_params_success(in_deployment: bool) -> None: + with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner, in_deployment): + result = runner.invoke( + "component", + "generate", + "dagster_components.test.all_metadata_empty_asset", + "qux", + ) + assert_runner_result(result) + assert Path("bar/components/qux").exists() + component_yaml_path = Path("bar/components/qux/component.yaml") + assert component_yaml_path.exists() + assert ( + "type: dagster_components.test.all_metadata_empty_asset" + in component_yaml_path.read_text() + ) + + +@pytest.mark.parametrize("in_deployment", [True, False]) +def test_component_generate_json_params_success(in_deployment: bool) -> None: + with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner, in_deployment): + result = runner.invoke( + "component", + "generate", + "dagster_components.test.simple_pipes_script_asset", + "qux", + "--json-params", + '{"asset_key": "foo", "filename": "hello.py"}', + ) + assert_runner_result(result) + assert Path("bar/components/qux").exists() + assert Path("bar/components/qux/hello.py").exists() + component_yaml_path = Path("bar/components/qux/component.yaml") + assert component_yaml_path.exists() + assert ( + "type: dagster_components.test.simple_pipes_script_asset" + in component_yaml_path.read_text() + ) + + +@pytest.mark.parametrize("in_deployment", [True, False]) +def test_component_generate_key_value_params_success(in_deployment: bool) -> None: + with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner, in_deployment): + result = runner.invoke( + "component", + "generate", + "dagster_components.test.simple_pipes_script_asset", + "qux", + "--asset-key=foo", + "--filename=hello.py", + ) + assert_runner_result(result) + assert Path("bar/components/qux").exists() + assert Path("bar/components/qux/hello.py").exists() + component_yaml_path = Path("bar/components/qux/component.yaml") + assert component_yaml_path.exists() + assert ( + "type: dagster_components.test.simple_pipes_script_asset" + in component_yaml_path.read_text() + ) + + +def test_component_generate_json_params_and_key_value_params_fails() -> None: + with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner): + result = runner.invoke( + "component", + "generate", + "dagster_components.test.simple_pipes_script_asset", + "qux", + "--json-params", + '{"filename": "hello.py"}', + "--filename=hello.py", + ) + assert_runner_result(result, exit_0=False) + assert ( + "Detected params passed as both --json-params and individual options" in result.output + ) + + +def test_component_generate_outside_code_location_fails() -> None: + with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner): + result = runner.invoke("component", "generate", "bar.baz", "qux") + assert_runner_result(result, exit_0=False) + assert "must be run inside a Dagster code location directory" in result.output + + +@pytest.mark.parametrize("in_deployment", [True, False]) +def test_component_generate_already_exists_fails(in_deployment: bool) -> None: + with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner, in_deployment): + result = runner.invoke( + "component", + "generate", + "dagster_components.test.all_metadata_empty_asset", + "qux", + ) + assert_runner_result(result) + result = runner.invoke( + "component", + "generate", + "dagster_components.test.all_metadata_empty_asset", + "qux", + ) + assert_runner_result(result, exit_0=False) + assert "already exists" in result.output + + +# ######################## +# ##### REAL COMPONENTS +# ######################## + + +def test_generate_sling_replication_instance() -> None: + with ( + ProxyRunner.test(use_test_component_lib=False) as runner, + isolated_example_code_location_bar(runner), + ): + # We need to add dagster-embedded-elt also because we are using editable installs. Only + # direct dependencies will be resolved by uv.tool.sources. + subprocess.run( + ["uv", "add", "dagster-components[sling]", "dagster-embedded-elt"], check=True + ) + result = runner.invoke( + "component", "generate", "dagster_components.sling_replication", "file_ingest" + ) + assert_runner_result(result) + assert Path("bar/components/file_ingest").exists() + + component_yaml_path = Path("bar/components/file_ingest/component.yaml") + assert component_yaml_path.exists() + assert "type: dagster_components.sling_replication" in component_yaml_path.read_text() + + replication_path = Path("bar/components/file_ingest/replication.yaml") + assert replication_path.exists() + assert "source: " in replication_path.read_text() + + +dbt_project_path = "../stub_code_locations/dbt_project_location/components/jaffle_shop" + + +@pytest.mark.parametrize( + "params", + [ + ["--json-params", json.dumps({"project_path": str(dbt_project_path)})], + ["--project-path", dbt_project_path], + ], +) +def test_generate_dbt_project_instance(params) -> None: + with ( + ProxyRunner.test(use_test_component_lib=False) as runner, + isolated_example_code_location_bar(runner), + ): + # We need to add dagster-dbt also because we are using editable installs. Only + # direct dependencies will be resolved by uv.tool.sources. + subprocess.run(["uv", "add", "dagster-components[dbt]", "dagster-dbt"], check=True) + result = runner.invoke( + "component", + "generate", + "dagster_components.dbt_project", + "my_project", + *params, + ) + assert_runner_result(result) + assert Path("bar/components/my_project").exists() + + component_yaml_path = Path("bar/components/my_project/component.yaml") + assert component_yaml_path.exists() + assert "type: dagster_components.dbt_project" in component_yaml_path.read_text() + assert ( + "stub_code_locations/dbt_project_location/components/jaffle_shop" + in component_yaml_path.read_text() + ) + + +# ######################## +# ##### LIST +# ######################## + + +def test_list_components_succeeds(): + with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner): + result = runner.invoke( + "component", + "generate", + "dagster_components.test.all_metadata_empty_asset", + "qux", + ) + assert_runner_result(result) + result = runner.invoke("component", "list") + assert_runner_result(result) + assert ( + result.output.strip() + == textwrap.dedent(""" + qux + """).strip() + ) + + +def test_list_components_command_outside_code_location_fails() -> None: + with ProxyRunner.test() as runner, runner.isolated_filesystem(): + result = runner.invoke("component", "list") + assert_runner_result(result, exit_0=False) + assert "must be run inside a Dagster code location directory" in result.output diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_info_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_component_type_commands.py similarity index 52% rename from python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_info_commands.py rename to python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_component_type_commands.py index 27d9005156ccb..c1cef1d09fd71 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_info_commands.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_component_type_commands.py @@ -1,5 +1,8 @@ import textwrap +from pathlib import Path +import pytest +from dagster_dg.context import CodeLocationDirectoryContext, DgContext from dagster_dg.utils import ensure_dagster_dg_tests_import ensure_dagster_dg_tests_import() @@ -8,14 +11,51 @@ ProxyRunner, assert_runner_result, isolated_example_code_location_bar, + isolated_example_deployment_foo, ) +# ######################## +# ##### GENERATE +# ######################## -def test_info_component_type_all_metadata_success(): + +@pytest.mark.parametrize("in_deployment", [True, False]) +def test_component_type_generate_success(in_deployment: bool) -> None: + with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner, in_deployment): + result = runner.invoke("component-type", "generate", "baz") + assert_runner_result(result) + assert Path("bar/lib/baz.py").exists() + context = CodeLocationDirectoryContext.from_path(Path.cwd(), DgContext.default()) + assert context.has_component_type("bar.baz") + + +def test_component_type_generate_outside_code_location_fails() -> None: + with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner): + result = runner.invoke("component-type", "generate", "baz") + assert_runner_result(result, exit_0=False) + assert "must be run inside a Dagster code location directory" in result.output + + +@pytest.mark.parametrize("in_deployment", [True, False]) +def test_component_type_generate_already_exists_fails(in_deployment: bool) -> None: + with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner, in_deployment): + result = runner.invoke("component-type", "generate", "baz") + assert_runner_result(result) + result = runner.invoke("component-type", "generate", "baz") + assert_runner_result(result, exit_0=False) + assert "already exists" in result.output + + +# ######################## +# ##### INFO +# ######################## + + +def test_component_type_info_all_metadata_success(): with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner): result = runner.invoke( - "info", "component-type", + "info", "dagster_components.test.simple_pipes_script_asset", ) assert_runner_result(result) @@ -75,27 +115,27 @@ def test_info_component_type_all_metadata_success(): ) -def test_info_component_type_all_metadata_empty_success(): +def test_component_type_info_all_metadata_empty_success(): with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner): result = runner.invoke( - "info", "component-type", + "info", "dagster_components.test.all_metadata_empty_asset", ) assert_runner_result(result) assert ( result.output.strip() == textwrap.dedent(""" - dagster_components.test.all_metadata_empty_asset - """).strip() + dagster_components.test.all_metadata_empty_asset + """).strip() ) -def test_info_component_type_flag_fields_success(): +def test_component_type_info_flag_fields_success(): with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner): result = runner.invoke( - "info", "component-type", + "info", "dagster_components.test.simple_pipes_script_asset", "--description", ) @@ -110,8 +150,8 @@ def test_info_component_type_flag_fields_success(): ) result = runner.invoke( - "info", "component-type", + "info", "dagster_components.test.simple_pipes_script_asset", "--generate-params-schema", ) @@ -119,30 +159,30 @@ def test_info_component_type_flag_fields_success(): assert ( result.output.strip() == textwrap.dedent(""" - { - "properties": { - "asset_key": { - "title": "Asset Key", - "type": "string" + { + "properties": { + "asset_key": { + "title": "Asset Key", + "type": "string" + }, + "filename": { + "title": "Filename", + "type": "string" + } }, - "filename": { - "title": "Filename", - "type": "string" - } - }, - "required": [ - "asset_key", - "filename" - ], - "title": "SimplePipesScriptAssetParams", - "type": "object" - } - """).strip() + "required": [ + "asset_key", + "filename" + ], + "title": "SimplePipesScriptAssetParams", + "type": "object" + } + """).strip() ) result = runner.invoke( - "info", "component-type", + "info", "dagster_components.test.simple_pipes_script_asset", "--component-params-schema", ) @@ -150,33 +190,33 @@ def test_info_component_type_flag_fields_success(): assert ( result.output.strip() == textwrap.dedent(""" - { - "properties": { - "asset_key": { - "title": "Asset Key", - "type": "string" + { + "properties": { + "asset_key": { + "title": "Asset Key", + "type": "string" + }, + "filename": { + "title": "Filename", + "type": "string" + } }, - "filename": { - "title": "Filename", - "type": "string" - } - }, - "required": [ - "asset_key", - "filename" - ], - "title": "SimplePipesScriptAssetParams", - "type": "object" - } - """).strip() + "required": [ + "asset_key", + "filename" + ], + "title": "SimplePipesScriptAssetParams", + "type": "object" + } + """).strip() ) -def test_info_component_type_outside_code_location_fails() -> None: +def test_component_type_info_outside_code_location_fails() -> None: with ProxyRunner.test() as runner, runner.isolated_filesystem(): result = runner.invoke( - "info", "component-type", + "info", "dagster_components.test.simple_pipes_script_asset", "--component-params-schema", ) @@ -184,11 +224,11 @@ def test_info_component_type_outside_code_location_fails() -> None: assert "must be run inside a Dagster code location directory" in result.output -def test_info_component_type_multiple_flags_fails() -> None: +def test_component_type_info_multiple_flags_fails() -> None: with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner): result = runner.invoke( - "info", "component-type", + "info", "dagster_components.test.simple_pipes_script_asset", "--description", "--generate-params-schema", @@ -198,3 +238,31 @@ def test_info_component_type_multiple_flags_fails() -> None: "Only one of --description, --generate-params-schema, and --component-params-schema can be specified." in result.output ) + + +# ######################## +# ##### LIST +# ######################## + + +def test_list_component_types_success(): + with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner): + result = runner.invoke("component-type", "list") + assert_runner_result(result) + assert ( + result.output.strip() + == textwrap.dedent(""" + dagster_components.test.all_metadata_empty_asset + dagster_components.test.simple_asset + A simple asset that returns a constant string value. + dagster_components.test.simple_pipes_script_asset + A simple asset that runs a Python script with the Pipes subprocess client. + """).strip() + ) + + +def test_list_component_types_outside_code_location_fails() -> None: + with ProxyRunner.test() as runner, runner.isolated_filesystem(): + result = runner.invoke("component-type", "list") + assert_runner_result(result, exit_0=False) + assert "must be run inside a Dagster code location directory" in result.output diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_deployment_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_deployment_commands.py new file mode 100644 index 0000000000000..1edee38c667ac --- /dev/null +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_deployment_commands.py @@ -0,0 +1,32 @@ +import os +from pathlib import Path + +from dagster_dg.utils import ensure_dagster_dg_tests_import + +ensure_dagster_dg_tests_import() + +from dagster_dg_tests.utils import ProxyRunner, assert_runner_result + +# ######################## +# ##### GENERATE +# ######################## + + +def test_generate_deployment_command_success() -> None: + with ProxyRunner.test() as runner, runner.isolated_filesystem(): + result = runner.invoke("deployment", "generate", "foo") + assert_runner_result(result) + assert Path("foo").exists() + assert Path("foo/.github").exists() + assert Path("foo/.github/workflows").exists() + assert Path("foo/.github/workflows/dagster-cloud-deploy.yaml").exists() + assert Path("foo/dagster_cloud.yaml").exists() + assert Path("foo/code_locations").exists() + + +def test_generate_deployment_command_already_exists_fails() -> None: + with ProxyRunner.test() as runner, runner.isolated_filesystem(): + os.mkdir("foo") + result = runner.invoke("deployment", "generate", "foo") + assert_runner_result(result, exit_0=False) + assert "already exists" in result.output diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_generate_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_generate_commands.py deleted file mode 100644 index 3d51879fc0f79..0000000000000 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_generate_commands.py +++ /dev/null @@ -1,379 +0,0 @@ -import json -import os -import subprocess -import textwrap -from pathlib import Path - -import pytest -import tomli -from dagster_dg.context import CodeLocationDirectoryContext, DgContext -from dagster_dg.utils import discover_git_root, ensure_dagster_dg_tests_import, pushd - -ensure_dagster_dg_tests_import() - -from dagster_dg_tests.utils import ( - ProxyRunner, - assert_runner_result, - isolated_example_code_location_bar, - isolated_example_deployment_foo, -) - - -def test_generate_deployment_command_success() -> None: - with ProxyRunner.test() as runner, runner.isolated_filesystem(): - result = runner.invoke("generate", "deployment", "foo") - assert_runner_result(result) - assert Path("foo").exists() - assert Path("foo/.github").exists() - assert Path("foo/.github/workflows").exists() - assert Path("foo/.github/workflows/dagster-cloud-deploy.yaml").exists() - assert Path("foo/dagster_cloud.yaml").exists() - assert Path("foo/code_locations").exists() - - -def test_generate_deployment_command_already_exists_fails() -> None: - with ProxyRunner.test() as runner, runner.isolated_filesystem(): - os.mkdir("foo") - result = runner.invoke("generate", "deployment", "foo") - assert_runner_result(result, exit_0=False) - assert "already exists" in result.output - - -def test_generate_code_location_inside_deployment_success() -> None: - # Don't use the test component lib because it is not present in published dagster-components, - # which this test is currently accessing since we are not doing an editable install. - with ( - ProxyRunner.test(use_test_component_lib=False) as runner, - isolated_example_deployment_foo(runner), - ): - result = runner.invoke("generate", "code-location", "bar") - assert_runner_result(result) - assert Path("code_locations/bar").exists() - assert Path("code_locations/bar/bar").exists() - assert Path("code_locations/bar/bar/lib").exists() - assert Path("code_locations/bar/bar/components").exists() - assert Path("code_locations/bar/bar_tests").exists() - assert Path("code_locations/bar/pyproject.toml").exists() - - # Check venv created - assert Path("code_locations/bar/.venv").exists() - assert Path("code_locations/bar/uv.lock").exists() - - with open("code_locations/bar/pyproject.toml") as f: - toml = tomli.loads(f.read()) - - # No tool.uv.sources added without --use-editable-dagster - assert "uv" not in toml["tool"] - - # Check cache was populated - with pushd("code_locations/bar"): - result = runner.invoke("--verbose", "list", "component-types") - assert "CACHE [hit]" in result.output - - -def test_generate_code_location_outside_deployment_success() -> None: - # Don't use the test component lib because it is not present in published dagster-components, - # which this test is currently accessing since we are not doing an editable install. - with ProxyRunner.test(use_test_component_lib=False) as runner, runner.isolated_filesystem(): - result = runner.invoke("generate", "code-location", "bar") - assert_runner_result(result) - assert Path("bar").exists() - assert Path("bar/bar").exists() - assert Path("bar/bar/lib").exists() - assert Path("bar/bar/components").exists() - assert Path("bar/bar_tests").exists() - assert Path("bar/pyproject.toml").exists() - - # Check venv created - assert Path("bar/.venv").exists() - assert Path("bar/uv.lock").exists() - - -@pytest.mark.parametrize("mode", ["env_var", "arg"]) -def test_generate_code_location_editable_dagster_success(mode: str, monkeypatch) -> None: - dagster_git_repo_dir = discover_git_root(Path(__file__)) - if mode == "env_var": - monkeypatch.setenv("DAGSTER_GIT_REPO_DIR", str(dagster_git_repo_dir)) - editable_args = ["--use-editable-dagster", "--"] - else: - editable_args = ["--use-editable-dagster", str(dagster_git_repo_dir)] - with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner): - result = runner.invoke("generate", "code-location", *editable_args, "bar") - assert_runner_result(result) - assert Path("code_locations/bar").exists() - assert Path("code_locations/bar/pyproject.toml").exists() - with open("code_locations/bar/pyproject.toml") as f: - toml = tomli.loads(f.read()) - assert toml["tool"]["uv"]["sources"]["dagster"] == { - "path": f"{dagster_git_repo_dir}/python_modules/dagster", - "editable": True, - } - assert toml["tool"]["uv"]["sources"]["dagster-pipes"] == { - "path": f"{dagster_git_repo_dir}/python_modules/dagster-pipes", - "editable": True, - } - assert toml["tool"]["uv"]["sources"]["dagster-webserver"] == { - "path": f"{dagster_git_repo_dir}/python_modules/dagster-webserver", - "editable": True, - } - assert toml["tool"]["uv"]["sources"]["dagster-components"] == { - "path": f"{dagster_git_repo_dir}/python_modules/libraries/dagster-components", - "editable": True, - } - # Check for presence of one random package with no component to ensure we are - # preemptively adding all packages - assert toml["tool"]["uv"]["sources"]["dagstermill"] == { - "path": f"{dagster_git_repo_dir}/python_modules/libraries/dagstermill", - "editable": True, - } - - -def test_generate_code_location_skip_venv_success() -> None: - # Don't use the test component lib because it is not present in published dagster-components, - # which this test is currently accessing since we are not doing an editable install. - with ProxyRunner.test() as runner, runner.isolated_filesystem(): - result = runner.invoke("generate", "code-location", "--skip-venv", "bar") - assert_runner_result(result) - assert Path("bar").exists() - assert Path("bar/bar").exists() - assert Path("bar/bar/lib").exists() - assert Path("bar/bar/components").exists() - assert Path("bar/bar_tests").exists() - assert Path("bar/pyproject.toml").exists() - - # Check venv created - assert not Path("bar/.venv").exists() - assert not Path("bar/uv.lock").exists() - - -def test_generate_code_location_editable_dagster_no_env_var_no_value_fails(monkeypatch) -> None: - monkeypatch.setenv("DAGSTER_GIT_REPO_DIR", "") - with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner): - result = runner.invoke("generate", "code-location", "--use-editable-dagster", "--", "bar") - assert_runner_result(result, exit_0=False) - assert "requires the `DAGSTER_GIT_REPO_DIR`" in result.output - - -def test_generate_code_location_already_exists_fails() -> None: - with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner): - result = runner.invoke("generate", "code-location", "bar", "--skip-venv") - assert_runner_result(result) - result = runner.invoke("generate", "code-location", "bar", "--skip-venv") - assert_runner_result(result, exit_0=False) - assert "already exists" in result.output - - -@pytest.mark.parametrize("in_deployment", [True, False]) -def test_generate_component_type_success(in_deployment: bool) -> None: - with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner, in_deployment): - result = runner.invoke("generate", "component-type", "baz") - assert_runner_result(result) - assert Path("bar/lib/baz.py").exists() - context = CodeLocationDirectoryContext.from_path(Path.cwd(), DgContext.default()) - assert context.has_component_type("bar.baz") - - -def test_generate_component_type_outside_code_location_fails() -> None: - with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner): - result = runner.invoke("generate", "component-type", "baz") - assert_runner_result(result, exit_0=False) - assert "must be run inside a Dagster code location directory" in result.output - - -@pytest.mark.parametrize("in_deployment", [True, False]) -def test_generate_component_type_already_exists_fails(in_deployment: bool) -> None: - with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner, in_deployment): - result = runner.invoke("generate", "component-type", "baz") - assert_runner_result(result) - result = runner.invoke("generate", "component-type", "baz") - assert_runner_result(result, exit_0=False) - assert "already exists" in result.output - - -def test_generate_component_dynamic_subcommand_generation() -> None: - with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner): - result = runner.invoke("generate", "component", "--help") - assert_runner_result(result) - assert ( - textwrap.dedent(""" - Commands: - dagster_components.test.all_metadata_empty_asset - dagster_components.test.simple_asset - dagster_components.test.simple_pipes_script_asset - """).strip() - in result.output - ) - - -@pytest.mark.parametrize("in_deployment", [True, False]) -def test_generate_component_no_params_success(in_deployment: bool) -> None: - with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner, in_deployment): - result = runner.invoke( - "generate", - "component", - "dagster_components.test.all_metadata_empty_asset", - "qux", - ) - assert_runner_result(result) - assert Path("bar/components/qux").exists() - component_yaml_path = Path("bar/components/qux/component.yaml") - assert component_yaml_path.exists() - assert ( - "type: dagster_components.test.all_metadata_empty_asset" - in component_yaml_path.read_text() - ) - - -@pytest.mark.parametrize("in_deployment", [True, False]) -def test_generate_component_json_params_success(in_deployment: bool) -> None: - with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner, in_deployment): - result = runner.invoke( - "generate", - "component", - "dagster_components.test.simple_pipes_script_asset", - "qux", - "--json-params", - '{"asset_key": "foo", "filename": "hello.py"}', - ) - assert_runner_result(result) - assert Path("bar/components/qux").exists() - assert Path("bar/components/qux/hello.py").exists() - component_yaml_path = Path("bar/components/qux/component.yaml") - assert component_yaml_path.exists() - assert ( - "type: dagster_components.test.simple_pipes_script_asset" - in component_yaml_path.read_text() - ) - - -@pytest.mark.parametrize("in_deployment", [True, False]) -def test_generate_component_key_value_params_success(in_deployment: bool) -> None: - with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner, in_deployment): - result = runner.invoke( - "generate", - "component", - "dagster_components.test.simple_pipes_script_asset", - "qux", - "--asset-key=foo", - "--filename=hello.py", - ) - assert_runner_result(result) - assert Path("bar/components/qux").exists() - assert Path("bar/components/qux/hello.py").exists() - component_yaml_path = Path("bar/components/qux/component.yaml") - assert component_yaml_path.exists() - assert ( - "type: dagster_components.test.simple_pipes_script_asset" - in component_yaml_path.read_text() - ) - - -def test_generate_component_json_params_and_key_value_params_fails() -> None: - with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner): - result = runner.invoke( - "generate", - "component", - "dagster_components.test.simple_pipes_script_asset", - "qux", - "--json-params", - '{"filename": "hello.py"}', - "--filename=hello.py", - ) - assert_runner_result(result, exit_0=False) - assert ( - "Detected params passed as both --json-params and individual options" in result.output - ) - - -def test_generate_component_outside_code_location_fails() -> None: - with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner): - result = runner.invoke("generate", "component", "bar.baz", "qux") - assert_runner_result(result, exit_0=False) - assert "must be run inside a Dagster code location directory" in result.output - - -@pytest.mark.parametrize("in_deployment", [True, False]) -def test_generate_component_already_exists_fails(in_deployment: bool) -> None: - with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner, in_deployment): - result = runner.invoke( - "generate", - "component", - "dagster_components.test.all_metadata_empty_asset", - "qux", - ) - assert_runner_result(result) - result = runner.invoke( - "generate", - "component", - "dagster_components.test.all_metadata_empty_asset", - "qux", - ) - assert_runner_result(result, exit_0=False) - assert "already exists" in result.output - - -# ######################## -# ##### REAL COMPONENTS -# ######################## - - -def test_generate_sling_replication_instance() -> None: - with ( - ProxyRunner.test(use_test_component_lib=False) as runner, - isolated_example_code_location_bar(runner), - ): - # We need to add dagster-embedded-elt also because we are using editable installs. Only - # direct dependencies will be resolved by uv.tool.sources. - subprocess.run( - ["uv", "add", "dagster-components[sling]", "dagster-embedded-elt"], check=True - ) - result = runner.invoke( - "generate", "component", "dagster_components.sling_replication", "file_ingest" - ) - assert_runner_result(result) - assert Path("bar/components/file_ingest").exists() - - component_yaml_path = Path("bar/components/file_ingest/component.yaml") - assert component_yaml_path.exists() - assert "type: dagster_components.sling_replication" in component_yaml_path.read_text() - - replication_path = Path("bar/components/file_ingest/replication.yaml") - assert replication_path.exists() - assert "source: " in replication_path.read_text() - - -dbt_project_path = "../stub_code_locations/dbt_project_location/components/jaffle_shop" - - -@pytest.mark.parametrize( - "params", - [ - ["--json-params", json.dumps({"project_path": str(dbt_project_path)})], - ["--project-path", dbt_project_path], - ], -) -def test_generate_dbt_project_instance(params) -> None: - with ( - ProxyRunner.test(use_test_component_lib=False) as runner, - isolated_example_code_location_bar(runner), - ): - # We need to add dagster-dbt also because we are using editable installs. Only - # direct dependencies will be resolved by uv.tool.sources. - subprocess.run(["uv", "add", "dagster-components[dbt]", "dagster-dbt"], check=True) - result = runner.invoke( - "generate", - "component", - "dagster_components.dbt_project", - "my_project", - *params, - ) - assert_runner_result(result) - assert Path("bar/components/my_project").exists() - - component_yaml_path = Path("bar/components/my_project/component.yaml") - assert component_yaml_path.exists() - assert "type: dagster_components.dbt_project" in component_yaml_path.read_text() - assert ( - "stub_code_locations/dbt_project_location/components/jaffle_shop" - in component_yaml_path.read_text() - ) diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_list_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_list_commands.py deleted file mode 100644 index abb22d8934b93..0000000000000 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_list_commands.py +++ /dev/null @@ -1,83 +0,0 @@ -import textwrap - -from dagster_dg.utils import ensure_dagster_dg_tests_import - -ensure_dagster_dg_tests_import() - -from dagster_dg_tests.utils import ( - ProxyRunner, - assert_runner_result, - isolated_example_code_location_bar, - isolated_example_deployment_foo, -) - - -def test_list_code_locations_success(): - with ProxyRunner.test() as runner, isolated_example_deployment_foo(runner): - runner.invoke("generate", "code-location", "foo") - runner.invoke("generate", "code-location", "bar") - result = runner.invoke("list", "code-locations") - assert_runner_result(result) - assert ( - result.output.strip() - == textwrap.dedent(""" - bar - foo - """).strip() - ) - - -def test_list_code_locations_outside_deployment_fails() -> None: - with ProxyRunner.test() as runner, runner.isolated_filesystem(): - result = runner.invoke("list", "code-locations") - assert_runner_result(result, exit_0=False) - assert "must be run inside a Dagster deployment directory" in result.output - - -def test_list_component_types_success(): - with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner): - result = runner.invoke("list", "component-types") - assert_runner_result(result) - assert ( - result.output.strip() - == textwrap.dedent(""" - dagster_components.test.all_metadata_empty_asset - dagster_components.test.simple_asset - A simple asset that returns a constant string value. - dagster_components.test.simple_pipes_script_asset - A simple asset that runs a Python script with the Pipes subprocess client. - """).strip() - ) - - -def test_list_component_types_outside_code_location_fails() -> None: - with ProxyRunner.test() as runner, runner.isolated_filesystem(): - result = runner.invoke("list", "component-types") - assert_runner_result(result, exit_0=False) - assert "must be run inside a Dagster code location directory" in result.output - - -def test_list_components_succeeds(): - with ProxyRunner.test() as runner, isolated_example_code_location_bar(runner): - result = runner.invoke( - "generate", - "component", - "dagster_components.test.all_metadata_empty_asset", - "qux", - ) - assert_runner_result(result) - result = runner.invoke("list", "components") - assert_runner_result(result) - assert ( - result.output.strip() - == textwrap.dedent(""" - qux - """).strip() - ) - - -def test_list_components_command_outside_code_location_fails() -> None: - with ProxyRunner.test() as runner, runner.isolated_filesystem(): - result = runner.invoke("list", "components") - assert_runner_result(result, exit_0=False) - assert "must be run inside a Dagster code location directory" in result.output diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/test_cache.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/test_cache.py index b3c4964b4a28c..e7448c0f7dfef 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/test_cache.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/test_cache.py @@ -17,54 +17,54 @@ def test_load_from_cache(): with ProxyRunner.test(verbose=True) as runner, example_code_location(runner): - result = runner.invoke("list", "component-types") + result = runner.invoke("component-type", "list") assert_runner_result(result) assert "CACHE [miss]" in result.output assert "CACHE [write]" in result.output - result = runner.invoke("list", "component-types") + result = runner.invoke("component-type", "list") assert_runner_result(result) assert "CACHE [hit]" in result.output def test_cache_invalidation_uv_lock(): with ProxyRunner.test(verbose=True) as runner, example_code_location(runner): - result = runner.invoke("list", "component-types") + result = runner.invoke("component-type", "list") assert_runner_result(result) assert "CACHE [miss]" in result.output assert "CACHE [write]" in result.output subprocess.run(["uv", "add", "dagster-components[dbt]", "dagster-dbt"], check=True) - result = runner.invoke("list", "component-types") + result = runner.invoke("component-type", "list") assert_runner_result(result) assert "CACHE [miss]" in result.output def test_cache_invalidation_modified_lib(): with ProxyRunner.test(verbose=True) as runner, example_code_location(runner): - result = runner.invoke("list", "component-types") + result = runner.invoke("component-type", "list") assert_runner_result(result) assert "CACHE [miss]" in result.output assert "CACHE [write]" in result.output - result = runner.invoke("generate", "component-type", "my_component") + result = runner.invoke("component-type", "generate", "my_component") assert_runner_result(result) - result = runner.invoke("list", "component-types") + result = runner.invoke("component-type", "list") assert_runner_result(result) assert "CACHE [miss]" in result.output def test_cache_no_invalidation_modified_pkg(): with ProxyRunner.test(verbose=True) as runner, example_code_location(runner): - result = runner.invoke("list", "component-types") + result = runner.invoke("component-type", "list") assert_runner_result(result) assert "CACHE [miss]" in result.output assert "CACHE [write]" in result.output Path("bar/submodule.py").write_text("print('hello')") - result = runner.invoke("list", "component-types") + result = runner.invoke("component-type", "list") assert_runner_result(result) assert "CACHE [hit]" in result.output @@ -72,19 +72,19 @@ def test_cache_no_invalidation_modified_pkg(): @pytest.mark.parametrize("with_command", [True, False]) def test_cache_clear(with_command: bool): with ProxyRunner.test(verbose=True) as runner, example_code_location(runner): - result = runner.invoke("list", "component-types") + result = runner.invoke("component-type", "list") assert_runner_result(result) assert "CACHE [miss]" in result.output assert "CACHE [write]" in result.output if with_command: - result = runner.invoke("--clear-cache", "list", "component-types") + result = runner.invoke("--clear-cache", "component-type", "list") assert "CACHE [clear-all]" in result.output else: result = runner.invoke("--clear-cache") assert_runner_result(result) assert "CACHE [clear-all]" in result.output - result = runner.invoke("list", "component-types") + result = runner.invoke("component-type", "list") assert_runner_result(result) assert "CACHE [miss]" in result.output @@ -100,7 +100,7 @@ def test_rebuild_component_registry_success(): assert_runner_result(result) assert "CACHE [clear-key]" in result.output - result = runner.invoke("list", "component-types") + result = runner.invoke("component-type", "list") assert_runner_result(result) assert "CACHE [hit]" in result.output @@ -114,7 +114,7 @@ def test_rebuild_component_registry_fails_outside_code_location(): def test_rebuild_component_registry_fails_with_subcommand(): with ProxyRunner.test(verbose=True) as runner, example_code_location(runner): - result = runner.invoke("--rebuild-component-registry", "list", "component-types") + result = runner.invoke("--rebuild-component-registry", "component-type", "list") assert_runner_result(result, exit_0=False) assert "Cannot specify --rebuild-component-registry with a subcommand." in result.output @@ -138,6 +138,6 @@ def test_cache_disabled(): ProxyRunner.test(verbose=True, disable_cache=True) as runner, example_code_location(runner), ): - result = runner.invoke("list", "component-types") + result = runner.invoke("component-type", "list") assert_runner_result(result) assert "CACHE" not in result.output diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/utils.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/utils.py index 0d695570d5b87..45dbdebbf3216 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/utils.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/utils.py @@ -16,7 +16,7 @@ def isolated_example_deployment_foo(runner: Union[CliRunner, "ProxyRunner"]) -> Iterator[None]: runner = ProxyRunner(runner) if isinstance(runner, CliRunner) else runner with runner.isolated_filesystem(): - runner.invoke("generate", "deployment", "foo") + runner.invoke("deployment", "generate", "foo") with pushd("foo"): yield @@ -30,8 +30,8 @@ def isolated_example_code_location_bar( if in_deployment: with isolated_example_deployment_foo(runner): runner.invoke( - "generate", "code-location", + "generate", "--use-editable-dagster", dagster_git_repo_dir, *(["--skip-venv"] if skip_venv else []), @@ -42,8 +42,8 @@ def isolated_example_code_location_bar( else: with runner.isolated_filesystem(): runner.invoke( - "generate", "code-location", + "generate", "--use-editable-dagster", dagster_git_repo_dir, *(["--skip-venv"] if skip_venv else []),