NVIDIA-NeMo · mckornfield · Jul 30, 2026 · Jul 30, 2026 · Jul 31, 2026 · Jul 31, 2026
@@ -28,6 +28,7 @@
 import yaml as _yaml
 from nemo_platform import NeMoPlatform
 from nemo_platform_plugin.client.adapter import client_from_platform
+from nemo_platform_plugin.config import validate_docker_available
 from nemo_platform_plugin.secrets.client import SecretsClient
 from nemo_platform_plugin.secrets.types import PlatformSecretCreateRequest, PlatformSecretUpdateRequest
 from nmp.common.config import nmp_user_data_dir
@@ -808,17 +809,23 @@ def _wait_for_platform(
     timeout: int = _SERVICE_STARTUP_TIMEOUT_SECONDS,
     poll_interval: float = _SERVICE_STARTUP_POLL_INTERVAL,
     log_path: Path | None = None,
+    proc: subprocess.Popen | None = None,
 ) -> bool:
     """Poll until the platform health endpoint responds. Returns True on success.
 
     When *log_path* is provided, the spinner shows the last service that
     finished loading so users see that progress is being made during a
     slow cold start.
+
+    When *proc* is provided, return False immediately if the service process
+    exits before the platform becomes ready (avoid waiting the full timeout).
     """
     start = time.monotonic()
     deadline = start + timeout
     with console.status("[bold cyan]Waiting for platform...") as status:
         while time.monotonic() < deadline:
+            if proc is not None and proc.poll() is not None:
+                return False
             elapsed = int(time.monotonic() - start)
             svc = _last_startup_service(log_path)
             hint = f" — loaded {svc}" if svc else ""
@@ -829,6 +836,39 @@ def _wait_for_platform(
     return False
 
 
+_DOCKER_FAILURE_LOG_MARKERS = (
+    "Docker daemon is unavailable",
+    "Docker is unavailable",
+    "docker.from_env",
+    "Error while fetching server API version",
+)
+
+
+def _services_log_suggests_docker_failure(log_path: Path | None) -> bool:
+    """Return True when services.log contains known Docker soft-skip / daemon errors."""
+    if log_path is None or not log_path.is_file():
+        return False
+    try:
+        text = log_path.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return False
+    return any(marker in text for marker in _DOCKER_FAILURE_LOG_MARKERS)
+
+
+def _should_hint_docker_unavailable(*, exit_code: int | None, log_path: Path | None) -> bool:
+    """Decide whether to print a Docker-missing hint after a failed startup wait.
+
+    Prefer log evidence. Otherwise only hint when the process exited early and
+    a Docker ping confirms the daemon is unavailable — not on a pure readiness
+    timeout while the process is still alive.
+    """
+    if _services_log_suggests_docker_failure(log_path):
+        return True
+    if exit_code is not None and not validate_docker_available():
+        return True
+    return False
+
+
 def _kill_existing_services(base_url: str) -> None:
     """Find and kill any running ``nemo services run`` processes.
 
@@ -938,14 +978,19 @@ def _maybe_start_services(
 
     log = log_path_for(compute_scope(port=_resolve_services_port(base_url)))
 
-    if not _wait_for_platform(base_url, timeout=timeout, log_path=log):
+    if not _wait_for_platform(base_url, timeout=timeout, log_path=log, proc=proc):
         exit_code = proc.poll()
         if exit_code is not None:
             console.print(f"{CROSS} Service process exited early (exit code {exit_code})")
         else:
             proc.terminate()
             console.print(f"{CROSS} Platform did not become ready within {timeout}s")
         console.print(f"  Check {log} for details.")
+        if _should_hint_docker_unavailable(exit_code=exit_code, log_path=log):
+            console.print(
+                "  Docker does not appear to be available. "
+                "Install and start Docker, or configure non-Docker executors, then retry."
+            )
         raise typer.Exit(1)
 
     console.print(f"{CHECK} Platform running at {base_url} (pid {proc.pid})\n")

@@ -760,6 +760,66 @@ def test_allows_start_when_port_free(self, maybe_start_preflight_mocks):
             _maybe_start_services("http://localhost:8080", auto=False, start_services=True)
         maybe_start_preflight_mocks.assert_called_once()
 
+    def test_early_exit_prints_docker_hint_when_daemon_unavailable(self, maybe_start_preflight_mocks, capsys):
+        dead = MagicMock(pid=999)
+        dead.poll.return_value = 3
+        wait = MagicMock(return_value=False)
+        with (
+            patch(f"{SETUP_MOD}.check_port_available_for_start", return_value=None),
+            patch(f"{SETUP_MOD}._wait_for_platform", wait),
+            patch(f"{SETUP_MOD}.validate_docker_available", return_value=False),
+            patch(f"{SETUP_MOD}.log_path_for", return_value=MagicMock(__str__=lambda self: "/tmp/services.log")),
+            patch(f"{SETUP_MOD}._pause"),
+            pytest.raises(ClickExit),
+        ):
+            maybe_start_preflight_mocks.return_value = dead
+            _maybe_start_services("http://localhost:8080", auto=False, start_services=True)
+        wait.assert_called_once()
+        assert wait.call_args.kwargs.get("proc") is dead
+        captured = capsys.readouterr()
+        assert "exited early (exit code 3)" in captured.err
+        assert "Check /tmp/services.log for details." in captured.err
+        assert "Docker does not appear to be available" in captured.err
+
+    def test_readiness_timeout_does_not_hint_docker_without_evidence(
+        self, maybe_start_preflight_mocks, capsys, tmp_path
+    ):
+        alive = MagicMock(pid=999)
+        alive.poll.return_value = None
+        log = tmp_path / "services.log"
+        log.write_text("starting auth\n", encoding="utf-8")
+        with (
+            patch(f"{SETUP_MOD}.check_port_available_for_start", return_value=None),
+            patch(f"{SETUP_MOD}._wait_for_platform", return_value=False),
+            patch(f"{SETUP_MOD}.validate_docker_available", return_value=False),
+            patch(f"{SETUP_MOD}.log_path_for", return_value=log),
+            patch(f"{SETUP_MOD}._pause"),
+            pytest.raises(ClickExit),
+        ):
+            maybe_start_preflight_mocks.return_value = alive
+            _maybe_start_services("http://localhost:8080", auto=False, start_services=True)
+        captured = capsys.readouterr()
+        assert "did not become ready" in captured.err
+        assert "Docker does not appear to be available" not in captured.err
+
+    def test_docker_hint_from_services_log_markers(self, maybe_start_preflight_mocks, capsys, tmp_path):
+        alive = MagicMock(pid=999)
+        alive.poll.return_value = None
+        log = tmp_path / "services.log"
+        log.write_text("Skipping executor 'local-docker': Docker daemon is unavailable\n", encoding="utf-8")
+        with (
+            patch(f"{SETUP_MOD}.check_port_available_for_start", return_value=None),
+            patch(f"{SETUP_MOD}._wait_for_platform", return_value=False),
+            patch(f"{SETUP_MOD}.validate_docker_available", return_value=True),
+            patch(f"{SETUP_MOD}.log_path_for", return_value=log),
+            patch(f"{SETUP_MOD}._pause"),
+            pytest.raises(ClickExit),
+        ):
+            maybe_start_preflight_mocks.return_value = alive
+            _maybe_start_services("http://localhost:8080", auto=False, start_services=True)
+        captured = capsys.readouterr()
+        assert "Docker does not appear to be available" in captured.err
+
 
 class TestRemoteConnection:
     def test_prompts_again_until_platform_is_reachable(self, capsys):
@@ -2019,6 +2079,21 @@ def test_spinner_works_without_log_path(self, spinner_console):
         update_texts = [c.args[0] for c in mock_status.update.call_args_list]
         assert all("loaded" not in t for t in update_texts)
 
+    def test_returns_false_immediately_when_process_exits(self, spinner_console):
+        """Do not wait the full timeout when the service process already exited."""
+        dead = MagicMock()
+        dead.poll.return_value = 3
+        with (
+            patch(f"{self._MOD}._pause") as pause,
+            patch(f"{self._MOD}.time.monotonic", side_effect=[0, 0, 1, 2]),
+            patch(f"{self._MOD}._check_platform_reachable") as reachable,
+        ):
+            result = _wait_for_platform("http://localhost:8080", timeout=120, proc=dead)
+
+        assert result is False
+        reachable.assert_not_called()
+        pause.assert_not_called()
+
 
 # ---------------------------------------------------------------------------
 # _last_startup_service tests

@@ -82,6 +82,7 @@ def test_something():
 from pydantic import BaseModel, Field, field_validator, model_validator
 from pydantic._internal._model_construction import ModelMetaclass
 from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict
+from requests.exceptions import ConnectionError as RequestsConnectionError
 from requests.exceptions import Timeout as RequestsTimeout
 
 import docker
@@ -388,7 +389,7 @@ def validate_docker_available() -> bool:
         client = docker.from_env(timeout=5)
         client.ping()
         return True
-    except (DockerException, RequestsTimeout):
+    except (DockerException, RequestsConnectionError, RequestsTimeout, OSError):
         return False
     finally:
         if client:

@@ -268,3 +268,25 @@ class _RegisteredPlatformConfig(NemoPlatformConfig):
 def test_get_nemo_platform_config_alias() -> None:
     """get_nemo_platform_config is an alias for get_platform_config."""
     assert get_nemo_platform_config is get_platform_config
+
+
+def test_validate_docker_available_returns_false_on_connection_failures() -> None:
+    from unittest.mock import MagicMock, patch
+
+    from docker.errors import DockerException
+    from nemo_platform_plugin.config import validate_docker_available
+    from requests.exceptions import ConnectionError as RequestsConnectionError
+
+    for exc in (
+        DockerException("boom"),
+        RequestsConnectionError("refused"),
+        OSError("no such file"),
+    ):
+        with patch("nemo_platform_plugin.config.docker.from_env", side_effect=exc):
+            assert validate_docker_available() is False
+
+        client = MagicMock()
+        client.ping.side_effect = exc
+        with patch("nemo_platform_plugin.config.docker.from_env", return_value=client):
+            assert validate_docker_available() is False
+        client.close.assert_called()
@@ -16,13 +16,13 @@
 
 
 class MissingBackendDependencyError(RuntimeError):
-    """A backend's optional dependency (its packaging extra) is not installed.
+    """A backend cannot initialize because a required capability is unavailable.
 
-    Backends whose substrate SDK ships as an optional extra (e.g. ``openshell``)
-    raise this when that SDK is absent, so the executor registry can skip the
-    executor with a warning instead of failing the whole deployments service at
-    startup. Subclasses ``RuntimeError`` so existing ``except RuntimeError`` paths
-    keep working.
+    Raised when an optional packaging extra is missing (e.g. ``openshell``) or when
+    a runtime substrate is unreachable (e.g. Docker daemon/socket). The executor
+    registry catches this, skips that executor with a warning, and continues
+    starting the deployments service. Subclasses ``RuntimeError`` so existing
+    ``except RuntimeError`` paths keep working.
     """
 
 

@@ -14,6 +14,7 @@
     BackendStatusUpdate,
     DeploymentBackend,
     LogResult,
+    MissingBackendDependencyError,
     VolumeStatusUpdate,
 )
 from nemo_deployments_plugin.backends.docker import volumes as volume_ops
@@ -65,6 +66,7 @@
 from nemo_platform_plugin.entity_client import NemoEntitiesClient, NemoEntityNotFoundError
 from requests.exceptions import ConnectionError as RequestsConnectionError
 from requests.exceptions import ReadTimeout
+from requests.exceptions import Timeout as RequestsTimeout
 from urllib3.exceptions import ReadTimeoutError as Urllib3ReadTimeoutError
 
 if TYPE_CHECKING:
@@ -97,10 +99,12 @@ class DockerDeploymentBackend(DeploymentBackend):
 
     def init(self) -> None:
         try:
+            from docker.errors import DockerException
+
             import docker
             from docker import errors as docker_errors
         except ImportError as exc:
-            raise RuntimeError(
+            raise MissingBackendDependencyError(
                 "docker package is required for DockerDeploymentBackend. "
                 "Install with: uv sync --package nemo-deployments-plugin --extra docker"
             ) from exc
@@ -110,14 +114,20 @@ def init(self) -> None:
         self._executor_config = DockerExecutorConfig.model_validate(self._config)
         self._entities = NemoEntitiesClient(client_from_platform(self._sdk, AsyncEntitiesClient))
         self._gpu_pool = get_shared_gpu_pool()
-        self._client = self._create_client()
+        try:
+            self._client = self._create_client()
+        except (DockerException, RequestsConnectionError, RequestsTimeout, OSError) as exc:
+            raise MissingBackendDependencyError(
+                f"Docker daemon is unavailable ({exc}). Docker-backed deployments will be disabled."
+            ) from exc
 
     def _create_client(self) -> docker.DockerClient:
         kwargs: dict[str, Any] = {"timeout": self._executor_config.docker_timeout}
         if self._executor_config.docker_host:
             kwargs["base_url"] = self._executor_config.docker_host
         client = self._docker.from_env(**kwargs)
         client.api.timeout = self._executor_config.docker_timeout
+        client.ping()
         return client
 
     def shutdown(self) -> None:

@@ -66,19 +66,23 @@ def from_config(
                 try:
                     executors[spec.name] = classes[spec.backend](sdk, spec.config)
                 except MissingBackendDependencyError as exc:
-                    # An opt-in backend whose optional extra isn't installed (e.g.
-                    # 'openshell') must not take down the whole deployments service:
-                    # skip just that executor. Resolving it later raises
-                    # ExecutorNotFoundError, and a `default_executor` that can't be
-                    # built still fails fast via the check below.
+                    # Capability missing (optional packaging extra, unreachable Docker
+                    # daemon, etc.): skip just that executor so the deployments service
+                    # can still boot. Resolving a skipped name later raises
+                    # ExecutorNotFoundError; a skipped default is cleared below.
                     logger.warning(
                         "Skipping executor '%s': backend '%s' is unavailable (%s)",
                         spec.name,
                         spec.backend,
                         exc,
                     )
             if default_executor and default_executor not in executors:
-                raise ExecutorNotFoundError(f"default_executor '{default_executor}' is not registered.")
+                logger.warning(
+                    "Clearing default_executor '%s' because it is not registered "
+                    "(unavailable backend or missing from executor config).",
+                    default_executor,
+                )
+                default_executor = None
         except Exception:
             for backend in executors.values():
                 backend.shutdown()

@@ -14,8 +14,8 @@
     published_port_config,
     sample_config,
 )
-from docker.errors import APIError, NotFound
-from nemo_deployments_plugin.backends.base import BackendStatusUpdate
+from docker.errors import APIError, DockerException, NotFound
+from nemo_deployments_plugin.backends.base import BackendStatusUpdate, MissingBackendDependencyError
 from nemo_deployments_plugin.backends.docker import ports as ports_mod
 from nemo_deployments_plugin.backends.docker.backend import (
     _PORT_CONFLICT_ATTEMPTS,
@@ -43,6 +43,30 @@
 from requests.exceptions import ReadTimeout
 
 
+def test_init_raises_missing_dependency_when_docker_daemon_unavailable(mock_sdk: MagicMock) -> None:
+    with (
+        patch("nemo_deployments_plugin.backends.docker.backend.client_from_platform"),
+        patch("nemo_deployments_plugin.backends.docker.backend.NemoEntitiesClient"),
+        patch("nemo_deployments_plugin.backends.docker.backend.get_shared_gpu_pool", return_value=None),
+        patch("docker.from_env", side_effect=DockerException("Error while fetching server API version")),
+    ):
+        with pytest.raises(MissingBackendDependencyError, match="Docker daemon is unavailable"):
+            DockerDeploymentBackend(mock_sdk, {"docker_timeout": 5, "pull_images": False})
+
+
+def test_init_raises_missing_dependency_when_docker_ping_fails(mock_sdk: MagicMock) -> None:
+    client = MagicMock()
+    client.ping.side_effect = RequestsConnectionError("Connection refused")
+    with (
+        patch("nemo_deployments_plugin.backends.docker.backend.client_from_platform"),
+        patch("nemo_deployments_plugin.backends.docker.backend.NemoEntitiesClient"),
+        patch("nemo_deployments_plugin.backends.docker.backend.get_shared_gpu_pool", return_value=None),
+        patch("docker.from_env", return_value=client),
+    ):
+        with pytest.raises(MissingBackendDependencyError, match="Docker daemon is unavailable"):
+            DockerDeploymentBackend(mock_sdk, {"docker_timeout": 5, "pull_images": False})
+
+
 @pytest.mark.asyncio
 async def test_create_deployment_starts_container(
     docker_backend: DockerDeploymentBackend,