From cecbfdb8d6041b9c47023ba26f5b84fa150c321f Mon Sep 17 00:00:00 2001 From: Nick Schrock Date: Thu, 19 Dec 2024 21:20:59 -0500 Subject: [PATCH] Rename build_defs_from_toplevel_components_folder to build_component_defs and make it less magical --- .../hello_world/hello_world/definitions.py | 6 ++--- .../dagster_components/__init__.py | 2 +- .../dagster_components/cli/generate.py | 5 +++-- .../dagster_components/cli/list.py | 5 +++-- .../core/component_decl_builder.py | 2 +- .../core/component_defs_builder.py | 15 ++++++++----- .../dagster_components/core/deployment.py | 22 ++++++++++++++----- .../custom_sling_location/definitions.py | 6 ++--- .../definitions.py | 8 ++----- .../COMPONENT_TYPE_NAME_PLACEHOLDER.py.jinja | 1 - 10 files changed, 41 insertions(+), 31 deletions(-) diff --git a/examples/experimental/components/examples/hello_world_deployment/code_locations/hello_world/hello_world/definitions.py b/examples/experimental/components/examples/hello_world_deployment/code_locations/hello_world/hello_world/definitions.py index b7a4a42f65ee3..36eb05ef9314b 100644 --- a/examples/experimental/components/examples/hello_world_deployment/code_locations/hello_world/hello_world/definitions.py +++ b/examples/experimental/components/examples/hello_world_deployment/code_locations/hello_world/hello_world/definitions.py @@ -1,13 +1,13 @@ from pathlib import Path import dagster as dg -from dagster_components import ComponentRegistry, build_defs_from_toplevel_components_folder +from dagster_components import ComponentRegistry, build_component_defs from dagster_components.lib.pipes_subprocess_script_collection import ( PipesSubprocessScriptCollection, ) -defs = build_defs_from_toplevel_components_folder( - path=Path(__file__).parent, +defs = build_component_defs( + code_location_root=Path(__file__).parent.parent, registry=ComponentRegistry( {"pipes_subprocess_script_collection": PipesSubprocessScriptCollection} ), diff --git a/python_modules/libraries/dagster-components/dagster_components/__init__.py b/python_modules/libraries/dagster-components/dagster_components/__init__.py index 13b5b3d5758ad..d5eaadc8d1248 100644 --- a/python_modules/libraries/dagster-components/dagster_components/__init__.py +++ b/python_modules/libraries/dagster-components/dagster_components/__init__.py @@ -6,6 +6,6 @@ component as component, ) from dagster_components.core.component_defs_builder import ( - build_defs_from_toplevel_components_folder as build_defs_from_toplevel_components_folder, + build_component_defs as build_component_defs, ) from dagster_components.version import __version__ as __version__ diff --git a/python_modules/libraries/dagster-components/dagster_components/cli/generate.py b/python_modules/libraries/dagster-components/dagster_components/cli/generate.py index 4da9481d312b2..d8d1d9b61d7da 100644 --- a/python_modules/libraries/dagster-components/dagster_components/cli/generate.py +++ b/python_modules/libraries/dagster-components/dagster_components/cli/generate.py @@ -8,6 +8,7 @@ from dagster_components import ComponentRegistry from dagster_components.core.deployment import ( CodeLocationProjectContext, + find_enclosing_code_location_root_path, is_inside_code_location_project, ) from dagster_components.generate import generate_component_instance @@ -39,8 +40,8 @@ def generate_component_command( ) sys.exit(1) - context = CodeLocationProjectContext.from_path( - Path.cwd(), + context = CodeLocationProjectContext.from_code_location_path( + find_enclosing_code_location_root_path(Path.cwd()), ComponentRegistry.from_entry_point_discovery(builtin_component_lib=builtin_component_lib), ) if not context.has_component_type(component_type): diff --git a/python_modules/libraries/dagster-components/dagster_components/cli/list.py b/python_modules/libraries/dagster-components/dagster_components/cli/list.py index 28957aaa33aad..13317d6a32042 100644 --- a/python_modules/libraries/dagster-components/dagster_components/cli/list.py +++ b/python_modules/libraries/dagster-components/dagster_components/cli/list.py @@ -8,6 +8,7 @@ from dagster_components.core.component import ComponentMetadata, ComponentRegistry from dagster_components.core.deployment import ( CodeLocationProjectContext, + find_enclosing_code_location_root_path, is_inside_code_location_project, ) from dagster_components.utils import CLI_BUILTIN_COMPONENT_LIB_KEY @@ -31,8 +32,8 @@ def list_component_types_command(ctx: click.Context) -> None: ) sys.exit(1) - context = CodeLocationProjectContext.from_path( - Path.cwd(), + context = CodeLocationProjectContext.from_code_location_path( + find_enclosing_code_location_root_path(Path.cwd()), ComponentRegistry.from_entry_point_discovery(builtin_component_lib=builtin_component_lib), ) output: Dict[str, Any] = {} diff --git a/python_modules/libraries/dagster-components/dagster_components/core/component_decl_builder.py b/python_modules/libraries/dagster-components/dagster_components/core/component_decl_builder.py index 160ad0b8ef93a..33b3a3fb77b7d 100644 --- a/python_modules/libraries/dagster-components/dagster_components/core/component_decl_builder.py +++ b/python_modules/libraries/dagster-components/dagster_components/core/component_decl_builder.py @@ -47,4 +47,4 @@ def path_to_decl_node(path: Path) -> Optional[ComponentDeclNode]: if component: subs.append(component) - return ComponentFolder(path=path, sub_decls=subs) if subs else None + return ComponentFolder(path=path, sub_decls=subs) diff --git a/python_modules/libraries/dagster-components/dagster_components/core/component_defs_builder.py b/python_modules/libraries/dagster-components/dagster_components/core/component_defs_builder.py index 6021344d9cf39..70896fe70d0de 100644 --- a/python_modules/libraries/dagster-components/dagster_components/core/component_defs_builder.py +++ b/python_modules/libraries/dagster-components/dagster_components/core/component_defs_builder.py @@ -125,16 +125,21 @@ def defs_from_components( # Public method so optional Nones are fine @suppress_dagster_warnings -def build_defs_from_toplevel_components_folder( - path: Path, +def build_component_defs( + code_location_root: Path, resources: Optional[Mapping[str, object]] = None, registry: Optional["ComponentRegistry"] = None, ) -> "Definitions": - """Build a Definitions object from an entire component hierarchy.""" + """Build a Definitions object for all the component instances in a given code location. + + Args: + code_location_root (Path): The path to the code location root. + The path must be a code location directory that has a pyproject.toml with a [dagster] section. + """ from dagster._core.definitions.definitions_class import Definitions - context = CodeLocationProjectContext.from_path( - path, registry or ComponentRegistry.from_entry_point_discovery() + context = CodeLocationProjectContext.from_code_location_path( + code_location_root, registry or ComponentRegistry.from_entry_point_discovery() ) all_defs: List[Definitions] = [] diff --git a/python_modules/libraries/dagster-components/dagster_components/core/deployment.py b/python_modules/libraries/dagster-components/dagster_components/core/deployment.py index de95b5875a0ad..fe8d41bb8e9a5 100644 --- a/python_modules/libraries/dagster-components/dagster_components/core/deployment.py +++ b/python_modules/libraries/dagster-components/dagster_components/core/deployment.py @@ -15,13 +15,19 @@ def is_inside_code_location_project(path: Path) -> bool: try: - _resolve_code_location_root_path(path) + find_enclosing_code_location_root_path(path) return True except DagsterError: return False -def _resolve_code_location_root_path(path: Path) -> Path: +def find_enclosing_code_location_root_path(path: Path) -> Path: + """Given a path, locate the code location root directory that contains it. It + determines this by finding a pyproject.toml with a [tool.dagster] section. + + Searches parent directory recursively until it finds it. It if navigates + to the root directory, it throws an error. + """ current_path = path.absolute() while not _is_code_location_root(current_path): current_path = current_path.parent @@ -40,11 +46,15 @@ def _is_code_location_root(path: Path) -> bool: class CodeLocationProjectContext: @classmethod - def from_path(cls, path: Path, component_registry: "ComponentRegistry") -> Self: - root_path = _resolve_code_location_root_path(path) + def from_code_location_path(cls, path: Path, component_registry: "ComponentRegistry") -> Self: + if not _is_code_location_root(path): + raise DagsterError( + f"Path {path} is not a code location root. Must have a pyproject.toml with a [tool.dagster] section." + ) + return cls( - root_path=str(root_path), - name=os.path.basename(root_path), + root_path=str(path), + name=os.path.basename(path), component_registry=component_registry, ) diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/custom_sling_location/custom_sling_location/definitions.py b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/custom_sling_location/custom_sling_location/definitions.py index dd49b214d9757..25f8151a3898c 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/custom_sling_location/custom_sling_location/definitions.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/custom_sling_location/custom_sling_location/definitions.py @@ -1,11 +1,9 @@ from pathlib import Path from dagster._core.definitions.definitions_class import Definitions -from dagster_components.core.component_defs_builder import ( - build_defs_from_toplevel_components_folder, -) +from dagster_components.core.component_defs_builder import build_component_defs -defs = build_defs_from_toplevel_components_folder(path=Path(__file__).parent) +defs = build_component_defs(code_location_root=Path(__file__).parent.parent) if __name__ == "__main__": Definitions.validate_loadable(defs) diff --git a/python_modules/libraries/dagster-dg/dagster_dg/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/definitions.py b/python_modules/libraries/dagster-dg/dagster_dg/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/definitions.py index 928050abfadf2..09a7105e350ea 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/definitions.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/definitions.py @@ -1,9 +1,5 @@ from pathlib import Path -from dagster_components.core.component_defs_builder import ( - build_defs_from_toplevel_components_folder, -) +from dagster_components import build_component_defs -defs = build_defs_from_toplevel_components_folder( - path=Path(__file__).parent, -) +defs = build_component_defs(code_location_root=Path(__file__).parent.parent) diff --git a/python_modules/libraries/dagster-dg/dagster_dg/templates/COMPONENT_TYPE/COMPONENT_TYPE_NAME_PLACEHOLDER.py.jinja b/python_modules/libraries/dagster-dg/dagster_dg/templates/COMPONENT_TYPE/COMPONENT_TYPE_NAME_PLACEHOLDER.py.jinja index 9ac7ac51684c0..7bb300ced1339 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/templates/COMPONENT_TYPE/COMPONENT_TYPE_NAME_PLACEHOLDER.py.jinja +++ b/python_modules/libraries/dagster-dg/dagster_dg/templates/COMPONENT_TYPE/COMPONENT_TYPE_NAME_PLACEHOLDER.py.jinja @@ -3,7 +3,6 @@ from dagster_components import ( Component, ComponentRegistry, ComponentLoadContext, - build_defs_from_toplevel_components_folder, component, )