refactor(lifecycle): run hooks via the Docker exec API (drop docker exec shell-out)

The lifecycle shell server no longer shells out to `docker exec`. It runs each
command as an independent `/bin/sh -c` exec through the moby client exec API,
added as EngineClient.ExecInContainer (ExecCreate → ExecAttach → stdcopy demux →
ExecInspect for the exit code). The DockerAPI interface gains ExecCreate/
ExecAttach/ExecInspect; the mock and a focused unit test (stdcopy framing) cover it.

This drops the hand-rolled EOT framing protocol and the persistent-shell
management from internal/lifecycle/shell.go. It is behavior-preserving: no
lifecycle hook or userEnvProbe command depends on shell state carried across
commands (each probe runs its own login/interactive shell; hooks are
independent), so one-exec-per-command is equivalent while providing user/env per
exec natively. NewShellServer now takes (ctx, ContainerExecutor) instead of a
docker path; all nine call sites pass the engine client.

Validated: 35 run-user-commands + exec runtime parity cases match against the TS
oracle (lifecycle hooks, userEnvProbe, secrets, dotfiles, subfolder config).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: aledbf

docs/migration/REMAINING-WORK.md

@@ -426,6 +426,30 @@ with the TS oracle except where indicated; they are sequenced by value/risk.
   and a real-daemon smoke (`up` + `up --remove-existing-container`) exercises the
   reshaped `RemoveContainer`/`Events`/`ListContainers`/`PullImage` paths.
 
+## Self-containment (fewer external commands)
+
+The CLI drives containers through Go libraries where it is parity-neutral to do
+so, keeping the big shell-outs only where a library would regress behavior or
+bloat the binary.
+
+- **Docker context + git root — ✅ DONE (zero new deps).** `docker context inspect`
+  is replaced by a direct read of the context store (`internal/docker/context.go`);
+  `git rev-parse --show-cdup` by a lexical `.git` parent walk. Removes the last hard
+  runtime deps on the external `docker`/`git` binaries in the connect/config paths.
+- **Lifecycle shell → Docker exec API — ✅ DONE.** The lifecycle shell server (hooks
+  + userEnvProbe) no longer shells out to `docker exec`; it runs each command as an
+  independent `/bin/sh -c` via the moby client exec API (`EngineClient.ExecInContainer`
+  — ExecCreate/Attach/Inspect + stdcopy demux), dropping the EOT framing protocol.
+  Safe because no hook/probe relies on shell state carried across commands (each
+  probe command spawns its own login/interactive shell). Validated: 35
+  run-user-commands + exec runtime cases match against the oracle.
+- **Kept as shell-outs (deliberate):** `docker buildx build` (buildx feature breadth
+  + the user's builder/context; a library would regress buildx or pull the heavy
+  buildkit client), `docker compose` (its output is not in the compared stream, so a
+  library adds huge deps for zero parity gain), interactive `docker exec -it` (the
+  inherited-TTY design avoids re-adding PTY/SIGWINCH/128+N complexity), and the
+  credential-helper protocol (external executables by design).
+
 ## Decisions that must be made explicit
 
 Decisions already taken (firm):

internal/cli/exec.go

@@ -225,11 +225,7 @@ func runExec(ctx context.Context, opts *execOpts, cmdArgs []string) error {
 
 	// Probed env (lowest priority)
 	if probeStrategy != lifecycle.ProbeNone {
-		dockerPath := opts.dockerPath
-		if dockerPath == "" {
-			dockerPath = "docker"
-		}
-		probeServer, probeErr := lifecycle.NewShellServer(dockerPath, containerID, remoteUser, logger)
+		probeServer, probeErr := lifecycle.NewShellServer(ctx, engine, containerID, remoteUser, logger)
 		if probeErr == nil {
 			probedEnv, _ := lifecycle.ProbeRemoteEnv(logger, probeServer, probeStrategy, remoteUser)
 			probeServer.Close()

internal/cli/run_user_commands.go

@@ -229,7 +229,7 @@ func runUserCommands(ctx context.Context, out Output, opts *runUserCommandsOpts)
 
 	var mergedRemoteEnv []string
 	if probeStrategy != lifecycle.ProbeNone {
-		probeServer, probeErr := lifecycle.NewShellServer(opts.dockerPath, containerID, remoteUser, logger)
+		probeServer, probeErr := lifecycle.NewShellServer(ctx, engine, containerID, remoteUser, logger)
 		if probeErr == nil {
 			probedEnv, _ := lifecycle.ProbeRemoteEnv(logger, probeServer, probeStrategy, remoteUser, opts.containerSessionDataFolder)
 			probeServer.Close()
@@ -252,7 +252,7 @@ func runUserCommands(ctx context.Context, out Output, opts *runUserCommandsOpts)
 		}
 	}
 
-	shellServer, err := lifecycle.NewShellServer(opts.dockerPath, containerID, remoteUser, logger, mergedRemoteEnv...)
+	shellServer, err := lifecycle.NewShellServer(ctx, engine, containerID, remoteUser, logger, mergedRemoteEnv...)
 	if err != nil {
 		return writeErrorResult(out, fmt.Sprintf("Failed to start shell server: %v", err))
 	}

internal/cli/setup.go

@@ -129,7 +129,7 @@ func newSetUpCmd() *cobra.Command {
 
 			var mergedRemoteEnv []string
 			if probeStrategy != lifecycle.ProbeNone {
-				probeServer, probeErr := lifecycle.NewShellServer(dockerPath, containerID, remoteUser, logger)
+				probeServer, probeErr := lifecycle.NewShellServer(ctx, engine, containerID, remoteUser, logger)
 				if probeErr == nil {
 					probedEnv, _ := lifecycle.ProbeRemoteEnv(logger, probeServer, probeStrategy, remoteUser, containerSessionDataFolder)
 					probeServer.Close()
@@ -147,7 +147,7 @@ func newSetUpCmd() *cobra.Command {
 				}
 			}
 
-			shellServer, err := lifecycle.NewShellServer(dockerPath, containerID, remoteUser, logger, mergedRemoteEnv...)
+			shellServer, err := lifecycle.NewShellServer(ctx, engine, containerID, remoteUser, logger, mergedRemoteEnv...)
 			if err != nil {
 				return writeErrorResult(out, fmt.Sprintf("Failed to start shell: %v", err))
 			}

internal/cli/up.go

@@ -615,7 +615,7 @@ func (r *upRunner) runReattachLifecycle(containerID string) {
 		probeStrategy = lifecycle.UserEnvProbeStrategy(merged.UserEnvProbe)
 	}
 	if probeStrategy != lifecycle.ProbeNone {
-		if probeServer, probeErr := lifecycle.NewShellServer(opts.dockerPath, containerID, remoteUser, logger); probeErr == nil {
+		if probeServer, probeErr := lifecycle.NewShellServer(ctx, engine, containerID, remoteUser, logger); probeErr == nil {
 			probedEnv, _ := lifecycle.ProbeRemoteEnv(logger, probeServer, probeStrategy, remoteUser, opts.containerSessionDataFolder)
 			probeServer.Close()
 			for k, v := range probedEnv {
@@ -635,7 +635,7 @@ func (r *upRunner) runReattachLifecycle(containerID string) {
 		}
 	}
 
-	shellServer, err := lifecycle.NewShellServer(opts.dockerPath, containerID, remoteUser, logger, mergedRemoteEnv...)
+	shellServer, err := lifecycle.NewShellServer(ctx, engine, containerID, remoteUser, logger, mergedRemoteEnv...)
 	if err != nil {
 		logger.Write(fmt.Sprintf("Could not start shell server for reattach hooks: %v", err), log.LevelWarning)
 		return
@@ -1566,7 +1566,7 @@ func (r *upRunner) fromCompose(cfg *config.DevContainerConfig, loadResult *confi
 // a non-nil error (a *coreerrors.ContainerError with a TS-compatible description)
 // only when a lifecycle hook fails; probe/dotfiles issues stay non-fatal warnings.
 func (r *upRunner) runLifecycleForUp(containerID string, cfg *config.DevContainerConfig, workDir string, remoteUser string) error {
-	ctx, logger, dockerClient, engine, opts := r.ctx, r.log, r.docker, r.engine, r.opts
+	ctx, logger, engine, opts := r.ctx, r.log, r.engine, r.opts
 	// Build merged remote env: probed shell env → CLI --remote-env → config remoteEnv
 	probeStrategy := lifecycle.UserEnvProbeStrategy(opts.defaultUserEnvProbe)
 	if cfg.UserEnvProbe != "" {
@@ -1577,7 +1577,7 @@ func (r *upRunner) runLifecycleForUp(containerID string, cfg *config.DevContaine
 
 	// Probe user environment inside the container (with caching)
 	if probeStrategy != lifecycle.ProbeNone {
-		probeServer, probeErr := lifecycle.NewShellServer(dockerClient.DockerPath, containerID, remoteUser, logger)
+		probeServer, probeErr := lifecycle.NewShellServer(ctx, engine, containerID, remoteUser, logger)
 		if probeErr == nil {
 			probedEnv, _ := lifecycle.ProbeRemoteEnv(logger, probeServer, probeStrategy, remoteUser, opts.containerSessionDataFolder)
 			probeServer.Close()
@@ -1605,7 +1605,7 @@ func (r *upRunner) runLifecycleForUp(containerID string, cfg *config.DevContaine
 		}
 	}
 
-	shellServer, err := lifecycle.NewShellServer(dockerClient.DockerPath, containerID, remoteUser, logger, mergedRemoteEnv...)
+	shellServer, err := lifecycle.NewShellServer(ctx, engine, containerID, remoteUser, logger, mergedRemoteEnv...)
 	if err != nil {
 		logger.Write(fmt.Sprintf("Could not start shell server for lifecycle hooks: %v", err), log.LevelWarning)
 		return nil

internal/docker/engine.go

@@ -29,6 +29,9 @@ type DockerAPI interface {
 	ContainerRemove(ctx context.Context, containerID string, options mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error)
 	ContainerStart(ctx context.Context, containerID string, options mobyclient.ContainerStartOptions) (mobyclient.ContainerStartResult, error)
 	Events(ctx context.Context, options mobyclient.EventsListOptions) mobyclient.EventsResult
+	ExecCreate(ctx context.Context, containerID string, options mobyclient.ExecCreateOptions) (mobyclient.ExecCreateResult, error)
+	ExecAttach(ctx context.Context, execID string, options mobyclient.ExecAttachOptions) (mobyclient.ExecAttachResult, error)
+	ExecInspect(ctx context.Context, execID string, options mobyclient.ExecInspectOptions) (mobyclient.ExecInspectResult, error)
 	ImageInspect(ctx context.Context, imageID string, options ...mobyclient.ImageInspectOption) (mobyclient.ImageInspectResult, error)
 	ImagePull(ctx context.Context, refStr string, options mobyclient.ImagePullOptions) (mobyclient.ImagePullResponse, error)
 	ImageTag(ctx context.Context, options mobyclient.ImageTagOptions) (mobyclient.ImageTagResult, error)

internal/docker/engine_test.go

@@ -30,6 +30,30 @@ type mockAPI struct {
 	imagePullFn        func(ctx context.Context, ref string, opts mobyclient.ImagePullOptions) (mobyclient.ImagePullResponse, error)
 	imageTagFn         func(ctx context.Context, opts mobyclient.ImageTagOptions) error
 	serverVersionFn    func(ctx context.Context, opts mobyclient.ServerVersionOptions) (mobyclient.ServerVersionResult, error)
+	execCreateFn       func(ctx context.Context, containerID string, opts mobyclient.ExecCreateOptions) (mobyclient.ExecCreateResult, error)
+	execAttachFn       func(ctx context.Context, execID string, opts mobyclient.ExecAttachOptions) (mobyclient.ExecAttachResult, error)
+	execInspectFn      func(ctx context.Context, execID string) (mobyclient.ExecInspectResult, error)
+}
+
+func (m *mockAPI) ExecInspect(ctx context.Context, execID string, _ mobyclient.ExecInspectOptions) (mobyclient.ExecInspectResult, error) {
+	if m.execInspectFn != nil {
+		return m.execInspectFn(ctx, execID)
+	}
+	return mobyclient.ExecInspectResult{}, errors.New("not implemented")
+}
+
+func (m *mockAPI) ExecCreate(ctx context.Context, containerID string, opts mobyclient.ExecCreateOptions) (mobyclient.ExecCreateResult, error) {
+	if m.execCreateFn != nil {
+		return m.execCreateFn(ctx, containerID, opts)
+	}
+	return mobyclient.ExecCreateResult{}, errors.New("not implemented")
+}
+
+func (m *mockAPI) ExecAttach(ctx context.Context, execID string, opts mobyclient.ExecAttachOptions) (mobyclient.ExecAttachResult, error) {
+	if m.execAttachFn != nil {
+		return m.execAttachFn(ctx, execID, opts)
+	}
+	return mobyclient.ExecAttachResult{}, errors.New("not implemented")
 }
 
 func (m *mockAPI) Close() error { return nil }

internal/docker/exec.go

@@ -0,0 +1,47 @@
+package docker
+
+import (
+	"bytes"
+	"context"
+
+	"github.com/moby/moby/api/pkg/stdcopy"
+	mobyclient "github.com/moby/moby/client"
+)
+
+// ExecInContainer runs `/bin/sh -c command` inside a container via the Docker
+// exec API (no `docker exec` subprocess) and returns its stdout, stderr and exit
+// code. It runs non-interactively (no TTY): stdout/stderr are demultiplexed with
+// stdcopy, and the exit code is read from the exec inspect after the stream ends.
+//
+// user (docker exec -u) and env ("KEY=VALUE", docker exec -e) mirror how the
+// lifecycle shell runs hooks and the userEnvProbe as the remote user.
+func (e *EngineClient) ExecInContainer(ctx context.Context, containerID, user string, env []string, command string) (stdout, stderr string, exitCode int, err error) {
+	created, err := e.API.ExecCreate(ctx, containerID, mobyclient.ExecCreateOptions{
+		User:         user,
+		Env:          env,
+		Cmd:          []string{"/bin/sh", "-c", command},
+		AttachStdout: true,
+		AttachStderr: true,
+	})
+	if err != nil {
+		return "", "", -1, err
+	}
+
+	attach, err := e.API.ExecAttach(ctx, created.ID, mobyclient.ExecAttachOptions{})
+	if err != nil {
+		return "", "", -1, err
+	}
+	defer attach.Close()
+
+	var outBuf, errBuf bytes.Buffer
+	// Non-TTY exec output is a multiplexed stdout/stderr frame stream.
+	if _, copyErr := stdcopy.StdCopy(&outBuf, &errBuf, attach.Reader); copyErr != nil {
+		return outBuf.String(), errBuf.String(), -1, copyErr
+	}
+
+	inspect, err := e.API.ExecInspect(ctx, created.ID, mobyclient.ExecInspectOptions{})
+	if err != nil {
+		return outBuf.String(), errBuf.String(), -1, err
+	}
+	return outBuf.String(), errBuf.String(), inspect.ExitCode, nil
+}

internal/docker/exec_test.go

@@ -0,0 +1,89 @@
+package docker
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/binary"
+	"net"
+	"testing"
+
+	mobyclient "github.com/moby/moby/client"
+
+	"github.com/devcontainers/cli/internal/log"
+)
+
+// nopConn is a net.Conn whose only real method is Close (HijackedResponse.Close
+// calls Conn.Close); the embedded nil Conn is never otherwise used in the test.
+type nopConn struct{ net.Conn }
+
+func (nopConn) Close() error { return nil }
+
+// stdcopyFrame writes one stdcopy frame (8-byte header: stream type + big-endian
+// size, then payload) — the multiplexed format the non-TTY exec stream uses.
+func stdcopyFrame(buf *bytes.Buffer, streamType byte, payload string) {
+	var hdr [8]byte
+	hdr[0] = streamType // 1=stdout, 2=stderr
+	binary.BigEndian.PutUint32(hdr[4:], uint32(len(payload)))
+	buf.Write(hdr[:])
+	buf.WriteString(payload)
+}
+
+// framedExecStream builds a stdcopy-multiplexed stream carrying the given stdout
+// and stderr payloads, as the non-TTY exec attach endpoint would return.
+func framedExecStream(_ *testing.T, out, errText string) *bufio.Reader {
+	var buf bytes.Buffer
+	stdcopyFrame(&buf, 1, out)
+	stdcopyFrame(&buf, 2, errText)
+	return bufio.NewReader(&buf)
+}
+
+func TestExecInContainer(t *testing.T) {
+	var gotOpts mobyclient.ExecCreateOptions
+	api := &mockAPI{
+		execCreateFn: func(_ context.Context, id string, opts mobyclient.ExecCreateOptions) (mobyclient.ExecCreateResult, error) {
+			if id != "cid" {
+				t.Errorf("containerID = %q", id)
+			}
+			gotOpts = opts
+			return mobyclient.ExecCreateResult{ID: "exec1"}, nil
+		},
+		execAttachFn: func(_ context.Context, execID string, _ mobyclient.ExecAttachOptions) (mobyclient.ExecAttachResult, error) {
+			if execID != "exec1" {
+				t.Errorf("execID = %q", execID)
+			}
+			return mobyclient.ExecAttachResult{HijackedResponse: mobyclient.HijackedResponse{
+				Conn:   nopConn{},
+				Reader: framedExecStream(t, "hello\n", "oops\n"),
+			}}, nil
+		},
+		execInspectFn: func(_ context.Context, _ string) (mobyclient.ExecInspectResult, error) {
+			return mobyclient.ExecInspectResult{ExitCode: 3}, nil
+		},
+	}
+	e := &EngineClient{API: api, Log: log.Null}
+
+	stdout, stderr, code, err := e.ExecInContainer(context.Background(), "cid", "vscode", []string{"FOO=bar"}, "echo hello")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if stdout != "hello\n" {
+		t.Errorf("stdout = %q, want hello", stdout)
+	}
+	if stderr != "oops\n" {
+		t.Errorf("stderr = %q, want oops", stderr)
+	}
+	if code != 3 {
+		t.Errorf("exitCode = %d, want 3", code)
+	}
+	// The command is run as `/bin/sh -c <command>` with the given user/env.
+	if gotOpts.User != "vscode" || len(gotOpts.Env) != 1 || gotOpts.Env[0] != "FOO=bar" {
+		t.Errorf("opts user/env = %q/%v", gotOpts.User, gotOpts.Env)
+	}
+	if len(gotOpts.Cmd) != 3 || gotOpts.Cmd[0] != "/bin/sh" || gotOpts.Cmd[1] != "-c" || gotOpts.Cmd[2] != "echo hello" {
+		t.Errorf("cmd = %v", gotOpts.Cmd)
+	}
+	if gotOpts.TTY {
+		t.Error("exec must be non-TTY for stdcopy demux")
+	}
+}