From 4e7fe721429f4b88b0db0ed18bbb1a982e310216 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 5 Aug 2026 15:54:54 -0700 Subject: [PATCH 1/4] fix(e2e): separate Podman Machine loopback listeners Signed-off-by: Matthew Grossman --- .../skills/debug-openshell-cluster/SKILL.md | 10 +++-- crates/openshell-core/src/forward.rs | 17 +++++--- docs/reference/sandbox-compute-drivers.mdx | 6 ++- e2e/support/gateway-common.sh | 31 +++++++++++++++ e2e/with-podman-gateway.sh | 25 +++++++----- tasks/scripts/test-e2e-gateway-harness.sh | 39 +++++++++++++++++++ tasks/test.toml | 7 +++- 7 files changed, 114 insertions(+), 21 deletions(-) create mode 100755 tasks/scripts/test-e2e-gateway-harness.sh diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 319031b1d1..81cc679690 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -208,9 +208,13 @@ Common findings: host's IPv4 default route. Rootless pasta uses the private source address selected by that route; rootful Podman uses the bridge gateway address. - Callback discovery reports that the requested address equals the primary - listener: configure a distinct primary address. For Podman Machine, keep the - IPv4 loopback callback separate by using an IPv6-loopback primary such as - `[::1]:17670`. + listener: configure a distinct primary address. For Podman Machine, bind the + primary listener to IPv6 loopback, for example + `bind_address = "[::1]:17670"`, and register the CLI endpoint as + `https://localhost:17670`. The generated certificate includes `localhost`, + while a raw `https://[::1]:17670` endpoint can fail TLS setup with + `invalid dns name`. This leaves `127.0.0.1:17670` available for the + callback-only listener. - Rootless slirp4netns, another named helper, or missing helper metadata requires an explicitly remote `grpc_endpoint`. An explicit `host_gateway_ip` cannot bypass slirp4netns host-loopback isolation. Do not work around diff --git a/crates/openshell-core/src/forward.rs b/crates/openshell-core/src/forward.rs index 1d97174d99..3b9527bcc6 100644 --- a/crates/openshell-core/src/forward.rs +++ b/crates/openshell-core/src/forward.rs @@ -745,11 +745,11 @@ pub fn resolve_ssh_gateway( // Remote cluster: use the remote host but keep the cluster URL port. return (host.to_string(), cluster_port); } - // Both endpoints loopback. The unspecified addresses (0.0.0.0 / ::) - // are bind-only — they aren't valid connect targets and aren't in TLS - // cert SANs, so fall back to the cluster URL's host (which the CLI - // is already using to reach the gateway). - if gateway_host == "0.0.0.0" || gateway_host == "::" { + // Unspecified addresses are bind-only, and tonic cannot use an IPv6 + // literal as a TLS DNS name. In those cases, keep the cluster URL's + // already-reachable authority. Other loopback addresses retain the + // gateway-reported host. + if matches!(gateway_host, "0.0.0.0" | "::" | "::1") { return (host.to_string(), cluster_port); } return (gateway_host.to_string(), cluster_port); @@ -1026,6 +1026,13 @@ mod tests { assert_eq!(port, 443); } + #[test] + fn resolve_ssh_gateway_preserves_loopback_tls_authority() { + let (host, port) = resolve_ssh_gateway("::1", 8080, "https://localhost:8443"); + assert_eq!(host, "localhost"); + assert_eq!(port, 8443); + } + #[test] fn resolve_ssh_gateway_swaps_zeros_for_loopback_cluster_host() { // The gateway binds 0.0.0.0 but advertises that bind address via the diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 2132f3360e..ea4c1a37b0 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -114,8 +114,10 @@ reflection, inference-route management, and HTTP requests. A is expected for those requests. The gateway fails startup if a callback requirement resolves to the exact primary listener address because one socket cannot preserve both authorization scopes. For the IPv4-loopback callback used -by Podman Machine, bind the primary listener to a distinct address such as -`[::1]:17670`. +by Podman Machine, set `bind_address = "[::1]:17670"` for the primary listener +and register `https://localhost:17670` as the CLI endpoint. The hostname matches +the generated certificate and avoids the TLS transport error produced by a raw +IPv6-literal endpoint. Do not broaden the primary listener to `0.0.0.0`. ## Docker Driver diff --git a/e2e/support/gateway-common.sh b/e2e/support/gateway-common.sh index 6e25b30e0b..e5efeb709a 100644 --- a/e2e/support/gateway-common.sh +++ b/e2e/support/gateway-common.sh @@ -40,6 +40,37 @@ e2e_pick_port() { python3 -c 'import socket; s=socket.socket(); s.bind(("",0)); print(s.getsockname()[1]); s.close()' } +# Podman Machine forwards host.containers.internal callbacks to IPv4 loopback. +# Keep the primary listener on IPv6 loopback so the callback listener can retain +# its narrower authorization scope. Use a DNS authority for the CLI because +# tonic does not accept a bracketed IPv6 literal as a TLS server name. +e2e_podman_primary_bind_ip() { + local kernel_name=${1:-$(uname -s)} + + case "${kernel_name}" in + Darwin) printf '%s\n' '::1' ;; + *) printf '%s\n' '127.0.0.1' ;; + esac +} + +e2e_podman_cli_endpoint_host() { + local kernel_name=${1:-$(uname -s)} + + case "${kernel_name}" in + Darwin) printf '%s\n' 'localhost' ;; + *) printf '%s\n' '127.0.0.1' ;; + esac +} + +e2e_url_host_for_ip() { + local ip=$1 + + case "${ip}" in + *:*) printf '[%s]\n' "${ip}" ;; + *) printf '%s\n' "${ip}" ;; + esac +} + e2e_generate_pki() { local gateway_bin=$1 local pki_dir=$2 diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 8c598c88f8..91eff496bc 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -386,6 +386,10 @@ export OPENSHELL_E2E_GATEWAY_CA_CERT="${PKI_DIR}/ca.crt" HOST_PORT=$(e2e_pick_port) HEALTH_PORT=$(e2e_pick_port) +KERNEL_NAME=$(uname -s) +PRIMARY_BIND_IP=$(e2e_podman_primary_bind_ip "${KERNEL_NAME}") +CLI_ENDPOINT_HOST=$(e2e_podman_cli_endpoint_host "${KERNEL_NAME}") +HEALTH_ENDPOINT_HOST=$(e2e_url_host_for_ip "${PRIMARY_BIND_IP}") STATE_DIR="${WORKDIR}/state" mkdir -p "${STATE_DIR}" export XDG_STATE_HOME="${STATE_DIR}" @@ -415,11 +419,11 @@ toml_string() { GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" -# Start from the RPM default template so this e2e test exercises the same -# TOML config path that RPM users get on first start. The template leaves -# bind_address unset and sets compute_drivers = ["podman"], so this test -# exercises the built-in loopback listener plus the callback listener -# requested by the Podman driver. +# Start from the RPM default template so this e2e test exercises the same TOML +# config path that RPM users get on first start. The template leaves +# bind_address unset and sets compute_drivers = ["podman"]. On Podman Machine, +# the driver reserves IPv4 loopback for its callback-only listener, so the +# primary listener uses IPv6 loopback. Native Linux keeps the IPv4 default. # # We append the driver-specific table and override the port via CLI flag # (CLI > TOML in the merge precedence) so the test can use an ephemeral port. @@ -458,8 +462,9 @@ cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}" GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" - # compute_drivers comes from the RPM template, while bind_address uses the - # built-in loopback default. Override only the port for ephemeral selection. + # compute_drivers comes from the RPM template. Override the loopback address + # and port so Podman Machine can keep its IPv4 callback listener distinct. + --bind-address "${PRIMARY_BIND_IP}" --port "${HOST_PORT}" --health-port "${HEALTH_PORT}" --tls-cert "${PKI_DIR}/server/tls.crt" @@ -495,10 +500,10 @@ printf '%s\n' "${GATEWAY_PID}" >"${GATEWAY_PID_FILE}" GATEWAY_NAME="openshell-e2e-podman-${HOST_PORT}" if [ "${OIDC_MODE}" = "1" ]; then - CLI_GATEWAY_ENDPOINT="https://127.0.0.1:${HOST_PORT}" + CLI_GATEWAY_ENDPOINT="https://${CLI_ENDPOINT_HOST}:${HOST_PORT}" export OPENSHELL_E2E_OIDC_GATEWAY_ENDPOINT="${CLI_GATEWAY_ENDPOINT}" else - CLI_GATEWAY_ENDPOINT="https://127.0.0.1:${HOST_PORT}" + CLI_GATEWAY_ENDPOINT="https://${CLI_ENDPOINT_HOST}:${HOST_PORT}" e2e_register_mtls_gateway \ "${XDG_CONFIG_HOME}" \ "${GATEWAY_NAME}" \ @@ -524,7 +529,7 @@ while [ "${elapsed}" -lt "${timeout}" ]; do echo "ERROR: openshell-gateway exited before becoming healthy" exit 1 fi - if curl -sf "http://127.0.0.1:${HEALTH_PORT}/healthz" >/dev/null 2>&1; then + if curl --noproxy '*' -sf "http://${HEALTH_ENDPOINT_HOST}:${HEALTH_PORT}/healthz" >/dev/null 2>&1; then echo "Gateway healthy after ${elapsed}s." break fi diff --git a/tasks/scripts/test-e2e-gateway-harness.sh b/tasks/scripts/test-e2e-gateway-harness.sh new file mode 100755 index 0000000000..eefca05d21 --- /dev/null +++ b/tasks/scripts/test-e2e-gateway-harness.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +# shellcheck source=e2e/support/gateway-common.sh +source "${ROOT}/e2e/support/gateway-common.sh" + +assert_eq() { + local expected=$1 + local actual=$2 + local description=$3 + + if [ "${actual}" != "${expected}" ]; then + echo "FAIL: ${description}: expected '${expected}', got '${actual}'" >&2 + exit 1 + fi +} + +darwin_bind=$(e2e_podman_primary_bind_ip Darwin) +darwin_cli_host=$(e2e_podman_cli_endpoint_host Darwin) +darwin_url_host=$(e2e_url_host_for_ip "${darwin_bind}") + +assert_eq '::1' "${darwin_bind}" "Podman Machine primary bind" +assert_eq 'localhost' "${darwin_cli_host}" "Podman Machine TLS authority" +assert_eq '[::1]' "${darwin_url_host}" "Podman Machine health URL host" + +linux_bind=$(e2e_podman_primary_bind_ip Linux) +linux_cli_host=$(e2e_podman_cli_endpoint_host Linux) +linux_url_host=$(e2e_url_host_for_ip "${linux_bind}") + +assert_eq '127.0.0.1' "${linux_bind}" "native Linux primary bind" +assert_eq '127.0.0.1' "${linux_cli_host}" "native Linux TLS authority" +assert_eq '127.0.0.1' "${linux_url_host}" "native Linux health URL host" + +echo "e2e gateway harness tests passed" diff --git a/tasks/test.toml b/tasks/test.toml index ed0d17d7af..2a086d9293 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -5,7 +5,7 @@ [test] description = "Run all tests (Rust + Python)" -depends = ["test:rust", "test:python", "test:sbom", "test:install-sh", "test:build-env", "test:packaging-assets", "test:docs-website"] +depends = ["test:rust", "test:python", "test:sbom", "test:install-sh", "test:build-env", "test:packaging-assets", "test:e2e-gateway-harness", "test:docs-website"] ["test:docs-website"] description = "Test the docs-website sync script" @@ -33,6 +33,11 @@ description = "Run static packaging asset tests" run = "tasks/scripts/test-packaging-assets.sh" hide = true +["test:e2e-gateway-harness"] +description = "Run focused local gateway E2E harness tests" +run = "tasks/scripts/test-e2e-gateway-harness.sh" +hide = true + [e2e] description = "Run all end-to-end tests (Rust + Python + MCP)" depends = ["e2e:rust", "e2e:python", "e2e:mcp"] From eb77ee64c2ebc5df36e1726b4a096f563260025a Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 5 Aug 2026 17:02:21 -0700 Subject: [PATCH 2/4] test(e2e): remove shallow harness checks Signed-off-by: Matthew Grossman --- tasks/scripts/test-e2e-gateway-harness.sh | 39 ----------------------- tasks/test.toml | 7 +--- 2 files changed, 1 insertion(+), 45 deletions(-) delete mode 100755 tasks/scripts/test-e2e-gateway-harness.sh diff --git a/tasks/scripts/test-e2e-gateway-harness.sh b/tasks/scripts/test-e2e-gateway-harness.sh deleted file mode 100755 index eefca05d21..0000000000 --- a/tasks/scripts/test-e2e-gateway-harness.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" - -# shellcheck source=e2e/support/gateway-common.sh -source "${ROOT}/e2e/support/gateway-common.sh" - -assert_eq() { - local expected=$1 - local actual=$2 - local description=$3 - - if [ "${actual}" != "${expected}" ]; then - echo "FAIL: ${description}: expected '${expected}', got '${actual}'" >&2 - exit 1 - fi -} - -darwin_bind=$(e2e_podman_primary_bind_ip Darwin) -darwin_cli_host=$(e2e_podman_cli_endpoint_host Darwin) -darwin_url_host=$(e2e_url_host_for_ip "${darwin_bind}") - -assert_eq '::1' "${darwin_bind}" "Podman Machine primary bind" -assert_eq 'localhost' "${darwin_cli_host}" "Podman Machine TLS authority" -assert_eq '[::1]' "${darwin_url_host}" "Podman Machine health URL host" - -linux_bind=$(e2e_podman_primary_bind_ip Linux) -linux_cli_host=$(e2e_podman_cli_endpoint_host Linux) -linux_url_host=$(e2e_url_host_for_ip "${linux_bind}") - -assert_eq '127.0.0.1' "${linux_bind}" "native Linux primary bind" -assert_eq '127.0.0.1' "${linux_cli_host}" "native Linux TLS authority" -assert_eq '127.0.0.1' "${linux_url_host}" "native Linux health URL host" - -echo "e2e gateway harness tests passed" diff --git a/tasks/test.toml b/tasks/test.toml index 2a086d9293..ed0d17d7af 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -5,7 +5,7 @@ [test] description = "Run all tests (Rust + Python)" -depends = ["test:rust", "test:python", "test:sbom", "test:install-sh", "test:build-env", "test:packaging-assets", "test:e2e-gateway-harness", "test:docs-website"] +depends = ["test:rust", "test:python", "test:sbom", "test:install-sh", "test:build-env", "test:packaging-assets", "test:docs-website"] ["test:docs-website"] description = "Test the docs-website sync script" @@ -33,11 +33,6 @@ description = "Run static packaging asset tests" run = "tasks/scripts/test-packaging-assets.sh" hide = true -["test:e2e-gateway-harness"] -description = "Run focused local gateway E2E harness tests" -run = "tasks/scripts/test-e2e-gateway-harness.sh" -hide = true - [e2e] description = "Run all end-to-end tests (Rust + Python + MCP)" depends = ["e2e:rust", "e2e:python", "e2e:mcp"] From 7debf050bffd19d810b520867ba4f7c09ea28880 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 6 Aug 2026 09:06:10 -0700 Subject: [PATCH 3/4] refactor(e2e): trim Podman listener workaround Signed-off-by: Matthew Grossman --- e2e/support/gateway-common.sh | 31 ------------------------------- e2e/with-podman-gateway.sh | 16 +++++++++++----- 2 files changed, 11 insertions(+), 36 deletions(-) diff --git a/e2e/support/gateway-common.sh b/e2e/support/gateway-common.sh index e5efeb709a..6e25b30e0b 100644 --- a/e2e/support/gateway-common.sh +++ b/e2e/support/gateway-common.sh @@ -40,37 +40,6 @@ e2e_pick_port() { python3 -c 'import socket; s=socket.socket(); s.bind(("",0)); print(s.getsockname()[1]); s.close()' } -# Podman Machine forwards host.containers.internal callbacks to IPv4 loopback. -# Keep the primary listener on IPv6 loopback so the callback listener can retain -# its narrower authorization scope. Use a DNS authority for the CLI because -# tonic does not accept a bracketed IPv6 literal as a TLS server name. -e2e_podman_primary_bind_ip() { - local kernel_name=${1:-$(uname -s)} - - case "${kernel_name}" in - Darwin) printf '%s\n' '::1' ;; - *) printf '%s\n' '127.0.0.1' ;; - esac -} - -e2e_podman_cli_endpoint_host() { - local kernel_name=${1:-$(uname -s)} - - case "${kernel_name}" in - Darwin) printf '%s\n' 'localhost' ;; - *) printf '%s\n' '127.0.0.1' ;; - esac -} - -e2e_url_host_for_ip() { - local ip=$1 - - case "${ip}" in - *:*) printf '[%s]\n' "${ip}" ;; - *) printf '%s\n' "${ip}" ;; - esac -} - e2e_generate_pki() { local gateway_bin=$1 local pki_dir=$2 diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 91eff496bc..fe3429c18a 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -386,10 +386,16 @@ export OPENSHELL_E2E_GATEWAY_CA_CERT="${PKI_DIR}/ca.crt" HOST_PORT=$(e2e_pick_port) HEALTH_PORT=$(e2e_pick_port) -KERNEL_NAME=$(uname -s) -PRIMARY_BIND_IP=$(e2e_podman_primary_bind_ip "${KERNEL_NAME}") -CLI_ENDPOINT_HOST=$(e2e_podman_cli_endpoint_host "${KERNEL_NAME}") -HEALTH_ENDPOINT_HOST=$(e2e_url_host_for_ip "${PRIMARY_BIND_IP}") +if [ "$(uname -s)" = "Darwin" ]; then + # Podman Machine reserves IPv4 loopback for its callback-only listener. + PRIMARY_BIND_IP="::1" + CLI_ENDPOINT_HOST="localhost" + HEALTH_ENDPOINT_HOST="[::1]" +else + PRIMARY_BIND_IP="127.0.0.1" + CLI_ENDPOINT_HOST="127.0.0.1" + HEALTH_ENDPOINT_HOST="127.0.0.1" +fi STATE_DIR="${WORKDIR}/state" mkdir -p "${STATE_DIR}" export XDG_STATE_HOME="${STATE_DIR}" @@ -529,7 +535,7 @@ while [ "${elapsed}" -lt "${timeout}" ]; do echo "ERROR: openshell-gateway exited before becoming healthy" exit 1 fi - if curl --noproxy '*' -sf "http://${HEALTH_ENDPOINT_HOST}:${HEALTH_PORT}/healthz" >/dev/null 2>&1; then + if curl -sf "http://${HEALTH_ENDPOINT_HOST}:${HEALTH_PORT}/healthz" >/dev/null 2>&1; then echo "Gateway healthy after ${elapsed}s." break fi From 4962e225e7834fb88a47ee0d82a125824cc92be4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 6 Aug 2026 09:32:13 -0700 Subject: [PATCH 4/4] fix(e2e): bypass proxies for Podman health probe Signed-off-by: Matthew Grossman --- e2e/with-podman-gateway.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index fe3429c18a..cd52e007ab 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -535,7 +535,8 @@ while [ "${elapsed}" -lt "${timeout}" ]; do echo "ERROR: openshell-gateway exited before becoming healthy" exit 1 fi - if curl -sf "http://${HEALTH_ENDPOINT_HOST}:${HEALTH_PORT}/healthz" >/dev/null 2>&1; then + # Keep this loopback probe direct even when ::1 is absent from NO_PROXY. + if curl --noproxy '*' -sf "http://${HEALTH_ENDPOINT_HOST}:${HEALTH_PORT}/healthz" >/dev/null 2>&1; then echo "Gateway healthy after ${elapsed}s." break fi