fix(podman): restore rootless workload SIGTERM shutdown

.agents/skills/debug-openshell-cluster/SKILL.md

@@ -262,6 +262,10 @@ When `userns` is configured (e.g. `userns = "auto"` or `userns = "keep-id"`):
   rootful Podman uses absolute host IDs (e.g. `uidmap = ["0:1000:1", "1:100000:65536"]`).
 - `nomap` (without hyphen) is accepted as input but canonicalized to `no-map`
   for Podman's API.
+- A workload remains in `stopping` until Podman resorts to `SIGKILL`: inspect
+  supervisor logs for `failed to signal entrypoint process group`. The
+  supervisor must retain `CAP_KILL` so its root process can forward `SIGTERM`
+  to a workload that runs as the sandbox user.
 
 ### Step 6: Check Kubernetes Helm Gateways
 

crates/openshell-driver-podman/README.md

@@ -81,7 +81,7 @@ The container spec in `container.rs` sets these security-critical fields:
 |---|---|---|
 | `user` | `0:0` | The supervisor needs root inside the container for namespace creation, proxy setup, Landlock, seccomp, and filesystem preparation. |
 | `cap_drop` | Selected unneeded defaults | Podman's default capability set is already restricted. The driver drops capabilities the supervisor does not need. |
-| `cap_add` | `SYS_ADMIN`, `NET_ADMIN`, `SYS_PTRACE`, `SYSLOG`, `DAC_READ_SEARCH`, `SETPCAP` | Grants supervisor-only capabilities required for namespace setup, process identity, bypass diagnostics, and child bounding-set cleanup. Policy DNS binds an unprivileged supervisor port and does not require `NET_BIND_SERVICE`. |
+| `cap_add` | `SYS_ADMIN`, `NET_ADMIN`, `SYS_PTRACE`, `SYSLOG`, `DAC_READ_SEARCH`, `SETPCAP`, `KILL` | Grants supervisor-only capabilities required for namespace setup, process identity, bypass diagnostics, child bounding-set cleanup, and forwarding shutdown signals to a workload that runs as the sandbox user. Policy DNS binds an unprivileged supervisor port and does not require `NET_BIND_SERVICE`. |
 | `no_new_privileges` | `true` | Prevents privilege escalation after exec. |
 | `seccomp_profile_path` | `unconfined` | The supervisor installs its own policy-aware BPF filter. A container-level profile can block Landlock/seccomp syscalls during setup. |
 | `mounts` | Private tmpfs at `/run/netns` | Lets the supervisor create named network namespaces in rootless Podman. |

crates/openshell-driver-podman/src/container.rs

@@ -1114,8 +1114,6 @@ pub fn build_container_spec_for_image(
             "DAC_OVERRIDE".into(),
             // Not needed: the supervisor does not create setuid/setgid executables.
             "FSETID".into(),
-            // Not needed: the supervisor does not send signals to arbitrary processes.
-            "KILL".into(),
             // Not needed: the supervisor does not bind privileged ports (<1024).
             "NET_BIND_SERVICE".into(),
             // Not in Podman's default set but explicitly denied in case the image
@@ -1146,6 +1144,9 @@ pub fn build_container_spec_for_image(
             // Child setup clears the capability bounding set before exec, which
             // requires CAP_SETPCAP in the supervisor until drop_privileges().
             "SETPCAP".into(),
+            // Forwarding shutdown signals to the canonical workload process
+            // group after it drops to the sandbox UID requires CAP_KILL.
+            "KILL".into(),
         ],
         // SETUID, SETGID, SETPCAP, CHOWN, and FOWNER are intentionally kept from
         // Podman's default set and not dropped:
@@ -1885,6 +1886,7 @@ mod tests {
             "missing DAC_READ_SEARCH"
         );
         assert!(added.contains(&"SETPCAP"), "missing SETPCAP");
+        assert!(added.contains(&"KILL"), "missing KILL");
 
         // SETUID and SETGID are NOT in cap_add — they remain available from the
         // default bounding set because we no longer use cap_drop:ALL. Verify they
@@ -1916,6 +1918,10 @@ mod tests {
             !dropped.contains(&"SETPCAP"),
             "SETPCAP must not be dropped (needed for child bounding-set clear)"
         );
+        assert!(
+            !dropped.contains(&"KILL"),
+            "KILL must not be dropped (needed to signal the sandbox workload on shutdown)"
+        );
         assert!(
             !dropped.contains(&"ALL"),
             "must not use cap_drop:ALL in rootless Podman"

crates/openshell-server/src/compute/mod.rs

@@ -1418,11 +1418,15 @@ impl ComputeRuntime {
                     || driver_snapshot_confirms_stopped(&snapshot);
                 let suspension_progressing =
                     expected_stopped && driver_snapshot_confirms_stopping(&snapshot);
-                if suspension_progressing {
+                let runtime_restart_during_stop =
+                    expected_stopped && driver_snapshot_reports_runtime_restart(&snapshot);
+                if suspension_progressing || runtime_restart_during_stop {
                     // The Kubernetes controller has accepted the stop and
-                    // is waiting for its pod to terminate. Preserve the
-                    // durable transition so a later watch event can complete
-                    // it instead of claiming the sandbox is running again.
+                    // is waiting for its pod to terminate. A container
+                    // runtime restart can likewise be the expected SIGTERM
+                    // exit from an in-flight stop. Preserve the durable
+                    // transition so completion or recovery, rather than the
+                    // watcher, determines its terminal state.
                     debug!(sandbox_id, "Sandbox stop is still progressing");
                 } else if backend_phase == SandboxPhase::Error
                     || observed_stopped == expected_stopped
@@ -4020,8 +4024,11 @@ fn apply_driver_snapshot(sandbox: &mut Sandbox, incoming: &DriverSandbox, sessio
 
     // Infrastructure errors and successful main-process completions are
     // sticky until an explicit lifecycle operation changes desired state. A
-    // late backend snapshot must not revive either result.
-    if matches!(old_phase, SandboxPhase::Error | SandboxPhase::Completed) {
+    // late signal-exit snapshot must also not overwrite the terminal reason
+    // recorded by a completed explicit stop.
+    if matches!(old_phase, SandboxPhase::Error | SandboxPhase::Completed)
+        || (old_phase == SandboxPhase::Stopped && driver_snapshot_reports_runtime_restart(incoming))
+    {
         if let Some(metadata) = sandbox.metadata.as_mut() {
             metadata.name.clone_from(sandbox_name);
         }
@@ -4058,6 +4065,13 @@ fn apply_driver_snapshot(sandbox: &mut Sandbox, incoming: &DriverSandbox, sessio
     );
 
     phase = match old_phase {
+        // SIGTERM-driven runtime exits are reported as a runtime restart by
+        // Docker and Podman. While an explicit stop owns this durable
+        // transition, preserve Stopping so the stop result chooses whether
+        // the sandbox actually reached Stopped or needs recovery.
+        SandboxPhase::Stopping if driver_snapshot_reports_runtime_restart(incoming) => {
+            SandboxPhase::Stopping
+        }
         SandboxPhase::Stopping
             if phase == SandboxPhase::Stopped || driver_snapshot_confirms_stopped(incoming) =>
         {
@@ -4139,6 +4153,17 @@ fn driver_snapshot_confirms_stopped(incoming: &DriverSandbox) -> bool {
     })
 }
 
+fn driver_snapshot_reports_runtime_restart(incoming: &DriverSandbox) -> bool {
+    incoming.status.as_ref().is_some_and(|status| {
+        status.conditions.iter().any(|condition| {
+            condition.status.eq_ignore_ascii_case("false")
+                && condition
+                    .reason
+                    .eq_ignore_ascii_case("ContainerRuntimeRestart")
+        })
+    })
+}
+
 fn driver_snapshot_confirms_stopping(incoming: &DriverSandbox) -> bool {
     incoming.status.as_ref().is_some_and(|status| {
         status.conditions.iter().any(|condition| {
@@ -6770,6 +6795,103 @@ mod tests {
         assert_eq!(driver.stop_calls(), 1);
     }
 
+    #[tokio::test]
+    async fn explicit_stop_completes_after_term_runtime_restart() {
+        let driver = ControlledDriver::new();
+        driver.block_stop();
+        let runtime = test_runtime(driver.clone()).await;
+        let sandbox = sandbox_record("sb-term-stop", "sandbox-term-stop", SandboxPhase::Ready);
+        runtime.store.put_message(&sandbox).await.unwrap();
+
+        let stop_runtime = runtime.clone();
+        let stop = tokio::spawn(async move {
+            stop_runtime
+                .stop_sandbox("default", "sandbox-term-stop")
+                .await
+        });
+        tokio::time::timeout(Duration::from_secs(1), driver.stop_started.notified())
+            .await
+            .expect("stop did not reach the driver");
+
+        let mut runtime_restart = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name());
+        runtime_restart.status = Some(make_driver_status(make_driver_condition(
+            "ContainerRuntimeRestart",
+            "container exited with status 143",
+        )));
+        runtime.apply_sandbox_update(runtime_restart).await.unwrap();
+
+        let stopping = runtime
+            .store
+            .get_message::<Sandbox>(sandbox.object_id())
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(stopping.phase(), SandboxPhase::Stopping as i32);
+        assert_eq!(
+            stopping.status.unwrap().conditions[0].reason,
+            "ContainerRuntimeRestart"
+        );
+
+        driver.release_stop();
+        let stopped = tokio::time::timeout(Duration::from_secs(1), stop)
+            .await
+            .expect("stop did not finish")
+            .unwrap()
+            .unwrap();
+        assert_eq!(stopped.phase(), SandboxPhase::Stopped as i32);
+        assert_eq!(stopped.status.unwrap().conditions[0].reason, "Stopped");
+    }
+
+    #[tokio::test]
+    async fn failed_stop_does_not_report_term_runtime_restart_as_stopped() {
+        let driver = ControlledDriver::new();
+        driver.block_stop();
+        driver.set_stop_outcome(ControlledLifecycleOutcome::Error("stop timed out"));
+        let runtime = test_runtime(driver.clone()).await;
+        let sandbox = sandbox_record("sb-term-fail", "sandbox-term-fail", SandboxPhase::Ready);
+        runtime.store.put_message(&sandbox).await.unwrap();
+
+        let stop_runtime = runtime.clone();
+        let stop = tokio::spawn(async move {
+            stop_runtime
+                .stop_sandbox("default", "sandbox-term-fail")
+                .await
+        });
+        tokio::time::timeout(Duration::from_secs(1), driver.stop_started.notified())
+            .await
+            .expect("stop did not reach the driver");
+
+        let mut runtime_restart = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name());
+        runtime_restart.status = Some(make_driver_status(make_driver_condition(
+            "ContainerRuntimeRestart",
+            "container exited with status 143",
+        )));
+        driver.set_get_outcome(ControlledGetOutcome::Sandbox(Box::new(
+            runtime_restart.clone(),
+        )));
+        runtime.apply_sandbox_update(runtime_restart).await.unwrap();
+
+        driver.release_stop();
+        let err = tokio::time::timeout(Duration::from_secs(1), stop)
+            .await
+            .expect("stop did not finish")
+            .unwrap()
+            .unwrap_err();
+        assert!(err.message().contains("stop timed out"));
+
+        let stored = runtime
+            .store
+            .get_message::<Sandbox>(sandbox.object_id())
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(stored.phase(), SandboxPhase::Stopping as i32);
+        assert_eq!(
+            stored.status.unwrap().conditions[0].reason,
+            "ContainerRuntimeRestart"
+        );
+    }
+
     #[tokio::test]
     async fn request_cancellation_does_not_cancel_start_worker() {
         let driver = ControlledDriver::new();
@@ -8360,6 +8482,73 @@ mod tests {
         assert_eq!(status.exit_code, None);
     }
 
+    #[tokio::test]
+    async fn unexpected_term_runtime_restart_transitions_to_error() {
+        let runtime = test_runtime(Arc::new(TestDriver::default())).await;
+        let sandbox = sandbox_record("sb-term-exit", "sandbox-term-exit", SandboxPhase::Ready);
+        runtime.store.put_message(&sandbox).await.unwrap();
+        let mut runtime_restart = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name());
+        runtime_restart.status = Some(make_driver_status(make_driver_condition(
+            "ContainerRuntimeRestart",
+            "container exited with status 143",
+        )));
+
+        runtime.apply_sandbox_update(runtime_restart).await.unwrap();
+
+        let stored = runtime
+            .store
+            .get_message::<Sandbox>(sandbox.object_id())
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(stored.phase(), SandboxPhase::Error as i32);
+        assert_eq!(
+            stored.status.unwrap().conditions[0].reason,
+            "ContainerRuntimeRestart"
+        );
+    }
+
+    #[tokio::test]
+    async fn late_term_runtime_restart_preserves_intentional_stop_status() {
+        let runtime = test_runtime(Arc::new(TestDriver::default())).await;
+        let mut sandbox = sandbox_record(
+            "sb-term-stopped",
+            "sandbox-term-stopped",
+            SandboxPhase::Stopped,
+        );
+        sandbox.status = Some(SandboxStatus {
+            sandbox_name: sandbox.object_name().to_string(),
+            phase: SandboxPhase::Stopped as i32,
+            conditions: vec![SandboxCondition {
+                r#type: "Ready".to_string(),
+                status: "False".to_string(),
+                reason: "Stopped".to_string(),
+                message: "Sandbox compute is stopped".to_string(),
+                last_transition_time: String::new(),
+            }],
+            ..Default::default()
+        });
+        runtime.store.put_message(&sandbox).await.unwrap();
+        let mut runtime_restart = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name());
+        runtime_restart.status = Some(make_driver_status(make_driver_condition(
+            "ContainerRuntimeRestart",
+            "container exited with status 143",
+        )));
+
+        runtime.apply_sandbox_update(runtime_restart).await.unwrap();
+
+        let stored = runtime
+            .store
+            .get_message::<Sandbox>(sandbox.object_id())
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(stored.phase(), SandboxPhase::Stopped as i32);
+        let ready = &stored.status.unwrap().conditions[0];
+        assert_eq!(ready.reason, "Stopped");
+        assert_eq!(ready.message, "Sandbox compute is stopped");
+    }
+
     #[tokio::test]
     async fn late_driver_exit_preserves_completed_main_result() {
         let runtime = test_runtime(Arc::new(TestDriver::default())).await;

crates/openshell-supervisor-process/src/run.rs

@@ -453,24 +453,34 @@ pub async fn run_process(
             .build()
     );
 
-    if let Some(tx) = sidecar_exit_tx.as_ref() {
-        report_sidecar_main_process_exit(tx, &main_instance_id, rendered_code).await?;
-    } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) {
-        report_main_process_exit_until_ack(endpoint, id, &main_instance_id, rendered_code).await;
-        info!(instance_id = %main_instance_id, "main-process exit acknowledged");
+    if outcome.should_report_main_process_exit() {
+        if let Some(tx) = sidecar_exit_tx.as_ref() {
+            report_sidecar_main_process_exit(tx, &main_instance_id, rendered_code).await?;
+        } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) {
+            report_main_process_exit_until_ack(endpoint, id, &main_instance_id, rendered_code)
+                .await;
+            info!(instance_id = %main_instance_id, "main-process exit acknowledged");
+        }
+    } else {
+        info!(
+            instance_id = %main_instance_id,
+            "skipping main-process exit report during supervisor shutdown"
+        );
     }
     main_session.mark_terminal_reported();
-    if drain_terminal && terminal_delivery_pending {
+    if outcome.should_report_main_process_exit() && drain_terminal && terminal_delivery_pending {
         // The peer's SSH channel-close confirms that the terminal frames sent
         // above traversed russh and the relay. Detached commands have no active
         // attachment and never enter this wait.
         main_session.wait_for_terminal_attachments().await;
     }
-    if let Some(tx) = sidecar_exit_tx.as_ref() {
-        finalize_sidecar_main_process_exit(tx, &main_instance_id).await?;
-    } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) {
-        finalize_main_process_exit_until_ack(endpoint, id, &main_instance_id).await;
-        info!(instance_id = %main_instance_id, "main-process terminal delivery finalized");
+    if outcome.should_report_main_process_exit() {
+        if let Some(tx) = sidecar_exit_tx.as_ref() {
+            finalize_sidecar_main_process_exit(tx, &main_instance_id).await?;
+        } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) {
+            finalize_main_process_exit_until_ack(endpoint, id, &main_instance_id).await;
+            info!(instance_id = %main_instance_id, "main-process terminal delivery finalized");
+        }
     }
 
     supervisor_terminating.store(true, Ordering::Release);
@@ -572,6 +582,16 @@ enum ProcessWaitOutcome {
     },
 }
 
+impl ProcessWaitOutcome {
+    /// A gateway acknowledgement is required for ordinary canonical-process
+    /// completion, but cannot be awaited after the supervisor itself has been
+    /// asked to terminate. At that point the gateway may already be shutting
+    /// down and no longer able to acknowledge the report.
+    fn should_report_main_process_exit(&self) -> bool {
+        !matches!(self, Self::ShutdownSignal { .. })
+    }
+}
+
 async fn wait_for_process_exit_or_shutdown(
     handle: &mut ProcessHandle,
     timeout_secs: u64,
@@ -789,4 +809,22 @@ mod tests {
 
         assert_eq!(ssh_proxy_url_for_policy(&policy, None), None);
     }
+
+    #[cfg(unix)]
+    #[test]
+    fn supervisor_shutdown_exit_skips_gateway_acknowledgement() {
+        use std::os::unix::process::ExitStatusExt;
+
+        let status = ProcessStatus::from(std::process::ExitStatus::from_raw(libc::SIGTERM));
+
+        assert!(ProcessWaitOutcome::Exited(status).should_report_main_process_exit());
+        assert!(ProcessWaitOutcome::TimedOut.should_report_main_process_exit());
+        assert!(
+            !ProcessWaitOutcome::ShutdownSignal {
+                signal: "SIGTERM",
+                status,
+            }
+            .should_report_main_process_exit()
+        );
+    }
 }