ref(docker-compose): Starting to refactor docker compose logic (#198)

* ref(docker-compose): Starting to refactor get_docker_compose_commands_to_run * Removing old code
getsentry · Jan 6, 2025 · 2b513fc · 2b513fc
1 parent 4be853f
commit 2b513fc
Show file tree

Hide file tree

Showing 7 changed files with 217 additions and 523 deletions.
diff --git a/devservices/commands/down.py b/devservices/commands/down.py
@@ -9,6 +9,7 @@
 
 from sentry_sdk import capture_exception
 
+from devservices.configs.service_config import load_service_config_from_file
 from devservices.constants import CONFIG_FILE_NAME
 from devservices.constants import DEPENDENCY_CONFIG_VERSION
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
@@ -24,8 +25,9 @@
 from devservices.utils.dependencies import get_non_shared_remote_dependencies
 from devservices.utils.dependencies import install_and_verify_dependencies
 from devservices.utils.dependencies import InstalledRemoteDependency
+from devservices.utils.docker_compose import create_docker_compose_command
 from devservices.utils.docker_compose import DockerComposeCommand
-from devservices.utils.docker_compose import get_docker_compose_commands_to_run
+from devservices.utils.docker_compose import get_non_remote_services
 from devservices.utils.docker_compose import run_cmd
 from devservices.utils.services import find_matching_service
 from devservices.utils.services import Service
@@ -163,3 +165,51 @@ def _down(
         ]
         for future in concurrent.futures.as_completed(futures):
             cmd_outputs.append(future.result())
+
+
+def get_docker_compose_commands_to_run(
+    service: Service,
+    remote_dependencies: list[InstalledRemoteDependency],
+    current_env: dict[str, str],
+    command: str,
+    options: list[str],
+    service_config_file_path: str,
+    mode_dependencies: list[str],
+) -> list[DockerComposeCommand]:
+    docker_compose_commands = []
+    for dependency in remote_dependencies:
+        # TODO: Consider passing in service config in InstalledRemoteDependency instead of loading it here
+        dependency_service_config = load_service_config_from_file(dependency.repo_path)
+        dependency_config_path = os.path.join(
+            dependency.repo_path, DEVSERVICES_DIR_NAME, CONFIG_FILE_NAME
+        )
+        non_remote_services = get_non_remote_services(
+            dependency_config_path, current_env
+        )
+        services_to_use = non_remote_services.intersection(
+            set(dependency_service_config.modes[dependency.mode])
+        )
+        docker_compose_commands.append(
+            create_docker_compose_command(
+                dependency_service_config.service_name,
+                dependency_config_path,
+                services_to_use,
+                command,
+                options,
+            )
+        )
+
+    # Add docker compose command for the top level service
+    non_remote_services = get_non_remote_services(service_config_file_path, current_env)
+    services_to_use = non_remote_services.intersection(set(mode_dependencies))
+    if len(services_to_use) > 0:
+        docker_compose_commands.append(
+            create_docker_compose_command(
+                service.name,
+                service_config_file_path,
+                services_to_use,
+                command,
+                options,
+            )
+        )
+    return docker_compose_commands
diff --git a/devservices/commands/logs.py b/devservices/commands/logs.py
@@ -9,6 +9,7 @@
 
 from sentry_sdk import capture_exception
 
+from devservices.configs.service_config import load_service_config_from_file
 from devservices.constants import CONFIG_FILE_NAME
 from devservices.constants import DEPENDENCY_CONFIG_VERSION
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
@@ -23,7 +24,9 @@
 from devservices.utils.console import Console
 from devservices.utils.dependencies import install_and_verify_dependencies
 from devservices.utils.dependencies import InstalledRemoteDependency
-from devservices.utils.docker_compose import get_docker_compose_commands_to_run
+from devservices.utils.docker_compose import create_docker_compose_command
+from devservices.utils.docker_compose import DockerComposeCommand
+from devservices.utils.docker_compose import get_non_remote_services
 from devservices.utils.docker_compose import run_cmd
 from devservices.utils.services import find_matching_service
 from devservices.utils.services import Service
@@ -128,3 +131,51 @@ def _logs(
             cmd_outputs.append(future.result())
 
     return cmd_outputs
+
+
+def get_docker_compose_commands_to_run(
+    service: Service,
+    remote_dependencies: list[InstalledRemoteDependency],
+    current_env: dict[str, str],
+    command: str,
+    options: list[str],
+    service_config_file_path: str,
+    mode_dependencies: list[str],
+) -> list[DockerComposeCommand]:
+    docker_compose_commands = []
+    for dependency in remote_dependencies:
+        # TODO: Consider passing in service config in InstalledRemoteDependency instead of loading it here
+        dependency_service_config = load_service_config_from_file(dependency.repo_path)
+        dependency_config_path = os.path.join(
+            dependency.repo_path, DEVSERVICES_DIR_NAME, CONFIG_FILE_NAME
+        )
+        non_remote_services = get_non_remote_services(
+            dependency_config_path, current_env
+        )
+        services_to_use = non_remote_services.intersection(
+            set(dependency_service_config.modes[dependency.mode])
+        )
+        docker_compose_commands.append(
+            create_docker_compose_command(
+                dependency_service_config.service_name,
+                dependency_config_path,
+                services_to_use,
+                command,
+                options,
+            )
+        )
+
+    # Add docker compose command for the top level service
+    non_remote_services = get_non_remote_services(service_config_file_path, current_env)
+    services_to_use = non_remote_services.intersection(set(mode_dependencies))
+    if len(services_to_use) > 0:
+        docker_compose_commands.append(
+            create_docker_compose_command(
+                service.name,
+                service_config_file_path,
+                services_to_use,
+                command,
+                options,
+            )
+        )
+    return docker_compose_commands
diff --git a/devservices/commands/status.py b/devservices/commands/status.py
@@ -11,6 +11,7 @@
 
 from sentry_sdk import capture_exception
 
+from devservices.configs.service_config import load_service_config_from_file
 from devservices.constants import CONFIG_FILE_NAME
 from devservices.constants import DEPENDENCY_CONFIG_VERSION
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
@@ -24,7 +25,9 @@
 from devservices.utils.console import Console
 from devservices.utils.dependencies import install_and_verify_dependencies
 from devservices.utils.dependencies import InstalledRemoteDependency
-from devservices.utils.docker_compose import get_docker_compose_commands_to_run
+from devservices.utils.docker_compose import create_docker_compose_command
+from devservices.utils.docker_compose import DockerComposeCommand
+from devservices.utils.docker_compose import get_non_remote_services
 from devservices.utils.docker_compose import run_cmd
 from devservices.utils.services import find_matching_service
 from devservices.utils.services import Service
@@ -176,3 +179,51 @@ def _status(
             cmd_outputs.append(future.result())
 
     return cmd_outputs
+
+
+def get_docker_compose_commands_to_run(
+    service: Service,
+    remote_dependencies: list[InstalledRemoteDependency],
+    current_env: dict[str, str],
+    command: str,
+    options: list[str],
+    service_config_file_path: str,
+    mode_dependencies: list[str],
+) -> list[DockerComposeCommand]:
+    docker_compose_commands = []
+    for dependency in remote_dependencies:
+        # TODO: Consider passing in service config in InstalledRemoteDependency instead of loading it here
+        dependency_service_config = load_service_config_from_file(dependency.repo_path)
+        dependency_config_path = os.path.join(
+            dependency.repo_path, DEVSERVICES_DIR_NAME, CONFIG_FILE_NAME
+        )
+        non_remote_services = get_non_remote_services(
+            dependency_config_path, current_env
+        )
+        services_to_use = non_remote_services.intersection(
+            set(dependency_service_config.modes[dependency.mode])
+        )
+        docker_compose_commands.append(
+            create_docker_compose_command(
+                dependency_service_config.service_name,
+                dependency_config_path,
+                services_to_use,
+                command,
+                options,
+            )
+        )
+
+    # Add docker compose command for the top level service
+    non_remote_services = get_non_remote_services(service_config_file_path, current_env)
+    services_to_use = non_remote_services.intersection(set(mode_dependencies))
+    if len(services_to_use) > 0:
+        docker_compose_commands.append(
+            create_docker_compose_command(
+                service.name,
+                service_config_file_path,
+                services_to_use,
+                command,
+                options,
+            )
+        )
+    return docker_compose_commands
diff --git a/devservices/commands/up.py b/devservices/commands/up.py
@@ -9,6 +9,7 @@
 
 from sentry_sdk import capture_exception
 
+from devservices.configs.service_config import load_service_config_from_file
 from devservices.constants import CONFIG_FILE_NAME
 from devservices.constants import DEPENDENCY_CONFIG_VERSION
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
@@ -27,9 +28,10 @@
 from devservices.utils.dependencies import install_and_verify_dependencies
 from devservices.utils.dependencies import InstalledRemoteDependency
 from devservices.utils.docker import check_all_containers_healthy
+from devservices.utils.docker_compose import create_docker_compose_command
 from devservices.utils.docker_compose import DockerComposeCommand
 from devservices.utils.docker_compose import get_container_names_for_project
-from devservices.utils.docker_compose import get_docker_compose_commands_to_run
+from devservices.utils.docker_compose import get_non_remote_services
 from devservices.utils.docker_compose import run_cmd
 from devservices.utils.services import find_matching_service
 from devservices.utils.services import Service
@@ -187,3 +189,51 @@ def _create_devservices_network() -> None:
         stderr=subprocess.DEVNULL,
         check=True,
     )
+
+
+def get_docker_compose_commands_to_run(
+    service: Service,
+    remote_dependencies: list[InstalledRemoteDependency],
+    current_env: dict[str, str],
+    command: str,
+    options: list[str],
+    service_config_file_path: str,
+    mode_dependencies: list[str],
+) -> list[DockerComposeCommand]:
+    docker_compose_commands = []
+    for dependency in remote_dependencies:
+        # TODO: Consider passing in service config in InstalledRemoteDependency instead of loading it here
+        dependency_service_config = load_service_config_from_file(dependency.repo_path)
+        dependency_config_path = os.path.join(
+            dependency.repo_path, DEVSERVICES_DIR_NAME, CONFIG_FILE_NAME
+        )
+        non_remote_services = get_non_remote_services(
+            dependency_config_path, current_env
+        )
+        services_to_use = non_remote_services.intersection(
+            set(dependency_service_config.modes[dependency.mode])
+        )
+        docker_compose_commands.append(
+            create_docker_compose_command(
+                dependency_service_config.service_name,
+                dependency_config_path,
+                services_to_use,
+                command,
+                options,
+            )
+        )
+
+    # Add docker compose command for the top level service
+    non_remote_services = get_non_remote_services(service_config_file_path, current_env)
+    services_to_use = non_remote_services.intersection(set(mode_dependencies))
+    if len(services_to_use) > 0:
+        docker_compose_commands.append(
+            create_docker_compose_command(
+                service.name,
+                service_config_file_path,
+                services_to_use,
+                command,
+                options,
+            )
+        )
+    return docker_compose_commands
diff --git a/devservices/utils/docker_compose.py b/devservices/utils/docker_compose.py
@@ -5,15 +5,11 @@
 import platform
 import re
 import subprocess
-from collections.abc import Callable
 from typing import cast
 from typing import NamedTuple
 
 from packaging import version
 
-from devservices.configs.service_config import load_service_config_from_file
-from devservices.constants import CONFIG_FILE_NAME
-from devservices.constants import DEVSERVICES_DIR_NAME
 from devservices.constants import DOCKER_COMPOSE_DOWNLOAD_URL
 from devservices.constants import DOCKER_USER_PLUGIN_DIR
 from devservices.constants import LOGGER_NAME
@@ -22,10 +18,8 @@
 from devservices.exceptions import DockerComposeError
 from devservices.exceptions import DockerComposeInstallationError
 from devservices.utils.console import Console
-from devservices.utils.dependencies import InstalledRemoteDependency
 from devservices.utils.docker import check_docker_daemon_running
 from devservices.utils.install_binary import install_binary
-from devservices.utils.services import Service
 
 
 class DockerComposeCommand(NamedTuple):
@@ -170,7 +164,7 @@ def check_docker_compose_version() -> None:
 
 
 # TODO: Consider removing this in favor of in house logic for determining non-remote services
-def _get_non_remote_services(
+def get_non_remote_services(
     service_config_path: str, current_env: dict[str, str]
 ) -> set[str]:
     config_command = [
@@ -195,19 +189,14 @@ def _get_non_remote_services(
     return set(config_services.splitlines())
 
 
-def get_docker_compose_commands_to_run(
-    service: Service,
-    remote_dependencies: list[InstalledRemoteDependency],
-    current_env: dict[str, str],
+def create_docker_compose_command(
+    name: str,
+    config_path: str,
+    services_to_use: set[str],
     command: str,
     options: list[str],
-    service_config_file_path: str,
-    mode_dependencies: list[str],
-) -> list[DockerComposeCommand]:
-    docker_compose_commands = []
-    create_docker_compose_command: Callable[
-        [str, str, set[str]], DockerComposeCommand
-    ] = lambda name, config_path, services_to_use: DockerComposeCommand(
+) -> DockerComposeCommand:
+    return DockerComposeCommand(
         full_command=[
             "docker",
             "compose",
@@ -223,38 +212,6 @@ def get_docker_compose_commands_to_run(
         config_path=config_path,
         services=sorted(list(services_to_use)),
     )
-    for dependency in remote_dependencies:
-        # TODO: Consider passing in service config in InstalledRemoteDependency instead of loading it here
-        dependency_service_config = load_service_config_from_file(dependency.repo_path)
-        dependency_config_path = os.path.join(
-            dependency.repo_path, DEVSERVICES_DIR_NAME, CONFIG_FILE_NAME
-        )
-        non_remote_services = _get_non_remote_services(
-            dependency_config_path, current_env
-        )
-        services_to_use = non_remote_services.intersection(
-            set(dependency_service_config.modes[dependency.mode])
-        )
-        docker_compose_commands.append(
-            create_docker_compose_command(
-                dependency_service_config.service_name,
-                dependency_config_path,
-                services_to_use,
-            )
-        )
-
-    # Add docker compose command for the top level service
-    non_remote_services = _get_non_remote_services(
-        service_config_file_path, current_env
-    )
-    services_to_use = non_remote_services.intersection(set(mode_dependencies))
-    if len(services_to_use) > 0:
-        docker_compose_commands.append(
-            create_docker_compose_command(
-                service.name, service_config_file_path, services_to_use
-            )
-        )
-    return docker_compose_commands
 
 
 def run_cmd(cmd: list[str], env: dict[str, str]) -> subprocess.CompletedProcess[str]:

diff --git a/tests/commands/test_logs.py b/tests/commands/test_logs.py
@@ -67,7 +67,7 @@ def test_logs_no_specified_service_not_running(
 @mock.patch("devservices.commands.logs.find_matching_service")
 @mock.patch("devservices.utils.state.State.get_started_services")
 @mock.patch("devservices.commands.logs.install_and_verify_dependencies")
-@mock.patch("devservices.utils.docker_compose._get_non_remote_services")
+@mock.patch("devservices.commands.logs.get_non_remote_services")
 def test_logs_no_specified_service_success(
     mock_get_non_remote_services: mock.Mock,
     mock_install_and_verify_dependencies: mock.Mock,