Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions .agents/skills/debug-openshell-cluster/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,26 @@ helm -n openshell get values openshell | grep sandboxNamespace

Then inspect sandbox resources in that namespace.

For a split release, the gateway values should have
`workspaceResources.enabled=false`, and the target namespace should contain a
separate `openshell-workspace` release:

```bash
helm -n openshell get values openshell | grep -A2 workspaceResources
helm -n <sandbox-namespace> status openshell-workspace
kubectl -n <sandbox-namespace> get serviceaccount,role,rolebinding,networkpolicy \
-l app.kubernetes.io/instance=openshell-workspace
kubectl auth can-i create sandboxes.agents.x-k8s.io \
--namespace <sandbox-namespace> \
--as system:serviceaccount:openshell:openshell
```

If the gateway cannot create or watch sandboxes, verify the workspace
RoleBinding subject matches the gateway ServiceAccount name and namespace.
If SSH relay connections fail, verify the workspace NetworkPolicy selects the
gateway's actual `app.kubernetes.io/name` and
`app.kubernetes.io/instance` labels.

Check the configured sandbox service account when TokenReview bootstrap or
sandbox registration fails. Helm creates a dedicated sandbox service account by
default and writes it to `[openshell.drivers.kubernetes].service_account_name`;
Expand Down
63 changes: 41 additions & 22 deletions .github/actions/release-helm-oci/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
name: Release Helm OCI
description: >
Patch chart version/appVersion, refuse duplicate OCI versions on public
releases, package the chart, and push to GHCR OCI.
releases, package the gateway and workspace charts, and push them to GHCR OCI.

inputs:
chart-version:
Expand Down Expand Up @@ -51,11 +51,16 @@ runs:
shell: bash
run: |
set -euo pipefail
CHART_DIR="${RUNNER_TEMP}/chart-build"
cp -a deploy/helm/openshell/. "${CHART_DIR}"
sed -i "s/^version:.*/version: ${CHART_VERSION}/" "${CHART_DIR}/Chart.yaml"
sed -i "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/" "${CHART_DIR}/Chart.yaml"
echo "chart_dir=${CHART_DIR}" >> "$GITHUB_OUTPUT"
GATEWAY_CHART_DIR="${RUNNER_TEMP}/gateway-chart-build"
WORKSPACE_CHART_DIR="${RUNNER_TEMP}/workspace-chart-build"
cp -a deploy/helm/openshell/. "${GATEWAY_CHART_DIR}"
cp -a deploy/helm/openshell-workspace/. "${WORKSPACE_CHART_DIR}"
for chart_dir in "${GATEWAY_CHART_DIR}" "${WORKSPACE_CHART_DIR}"; do
sed -i "s/^version:.*/version: ${CHART_VERSION}/" "${chart_dir}/Chart.yaml"
sed -i "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/" "${chart_dir}/Chart.yaml"
done
echo "gateway_chart_dir=${GATEWAY_CHART_DIR}" >> "$GITHUB_OUTPUT"
echo "workspace_chart_dir=${WORKSPACE_CHART_DIR}" >> "$GITHUB_OUTPUT"
echo "chart_version=${CHART_VERSION}" >> "$GITHUB_OUTPUT"

- name: Refuse duplicate chart version
Expand All @@ -65,38 +70,52 @@ runs:
shell: bash
run: |
set -euo pipefail
OCI_CHART="oci://ghcr.io/nvidia/openshell/helm-chart"
if helm show chart "${OCI_CHART}" --version "${CHART_VERSION}" >/dev/null 2>&1; then
echo "::error::Chart ${CHART_VERSION} is already published. Use a new tag or delete the existing package first."
exit 1
fi
for chart in helm-chart openshell-workspace; do
OCI_CHART="oci://ghcr.io/nvidia/openshell/${chart}"
if helm show chart "${OCI_CHART}" --version "${CHART_VERSION}" >/dev/null 2>&1; then
echo "::error::Chart ${chart}:${CHART_VERSION} is already published. Use a new tag or delete the existing package first."
exit 1
fi
done

- name: Package Helm chart
env:
CHART_DIR: ${{ steps.prep.outputs.chart_dir }}
GATEWAY_CHART_DIR: ${{ steps.prep.outputs.gateway_chart_dir }}
WORKSPACE_CHART_DIR: ${{ steps.prep.outputs.workspace_chart_dir }}
shell: bash
run: |
set -euo pipefail
helm package "${CHART_DIR}" --destination /tmp
ls /tmp/helm-chart-*.tgz
mkdir -p /tmp/helm-charts
helm package "${GATEWAY_CHART_DIR}" --destination /tmp/helm-charts
helm package "${WORKSPACE_CHART_DIR}" --destination /tmp/helm-charts
ls /tmp/helm-charts/*.tgz

- name: Push Helm chart to GHCR OCI
shell: bash
run: |
set -euo pipefail
helm push /tmp/helm-chart-*.tgz oci://ghcr.io/nvidia/openshell
for archive in /tmp/helm-charts/*.tgz; do
helm push "${archive}" oci://ghcr.io/nvidia/openshell
done

- name: Push SHA-pinned chart
if: inputs.pin-sha != ''
env:
PIN_SHA: ${{ inputs.pin-sha }}
CHART_DIR: ${{ steps.prep.outputs.chart_dir }}
GATEWAY_CHART_DIR: ${{ steps.prep.outputs.gateway_chart_dir }}
WORKSPACE_CHART_DIR: ${{ steps.prep.outputs.workspace_chart_dir }}
shell: bash
run: |
set -euo pipefail
SHA_CHART_DIR="${RUNNER_TEMP}/chart-build-sha"
cp -a "${CHART_DIR}/." "${SHA_CHART_DIR}"
sed -i "s/^version:.*/version: 0.0.0-dev.${PIN_SHA}/" "${SHA_CHART_DIR}/Chart.yaml"
sed -i "s/^appVersion:.*/appVersion: \"${PIN_SHA}\"/" "${SHA_CHART_DIR}/Chart.yaml"
helm package "${SHA_CHART_DIR}" --destination /tmp/sha-pin
helm push /tmp/sha-pin/helm-chart-*.tgz oci://ghcr.io/nvidia/openshell
mkdir -p /tmp/sha-pin
for source_dir in "${GATEWAY_CHART_DIR}" "${WORKSPACE_CHART_DIR}"; do
chart_name="$(basename "${source_dir}")"
sha_chart_dir="${RUNNER_TEMP}/${chart_name}-sha"
cp -a "${source_dir}/." "${sha_chart_dir}"
sed -i "s/^version:.*/version: 0.0.0-dev.${PIN_SHA}/" "${sha_chart_dir}/Chart.yaml"
sed -i "s/^appVersion:.*/appVersion: \"${PIN_SHA}\"/" "${sha_chart_dir}/Chart.yaml"
helm package "${sha_chart_dir}" --destination /tmp/sha-pin
done
for archive in /tmp/sha-pin/*.tgz; do
helm push "${archive}" oci://ghcr.io/nvidia/openshell
done
9 changes: 9 additions & 0 deletions architecture/compute-runtimes.md
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,15 @@ through the driver configuration. The Helm chart defaults sandbox agents to
`Unconfined` so runtime/default AppArmor profiles do not block supervisor
network namespace setup on AppArmor-enabled nodes.

The Kubernetes deployment packaging has two ownership boundaries. The gateway
chart owns the gateway workload, configuration, Services, PKI, and
cluster-scoped gateway resources. It can retain the legacy combined behavior,
or omit workspace resources. The workspace chart is installed into a
pre-provisioned sandbox namespace and owns only the sandbox ServiceAccount,
namespaced RBAC, and sandbox ingress NetworkPolicy. Its RoleBinding names the
gateway ServiceAccount and namespace explicitly, so the two releases have
disjoint lifecycle ownership.

Resource requirements enter the driver layer through `SandboxSpec.resource_requirements`. This includes a set of GPU requirements, where a user
can request a specific number of GPUs or the driver-specific default behaviour.
For all in-tree drivers, this is equivalent to selecting a single GPU.
Expand Down
9 changes: 9 additions & 0 deletions deploy/helm/openshell-workspace/Chart.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

apiVersion: v2
name: openshell-workspace
description: Namespace-scoped prerequisites for OpenShell Kubernetes sandboxes
type: application
version: 0.0.0
appVersion: "0.0.0"
41 changes: 41 additions & 0 deletions deploy/helm/openshell-workspace/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
# OpenShell Workspace Helm Chart

<!--
This file is generated by helm-docs.
Edit README.md.gotmpl and values.yaml, then run `mise run helm:docs`.
-->

> **Experimental** - the shared-gateway, multi-namespace deployment path is
> under active design.

This chart installs the namespace-scoped ServiceAccount, RBAC, and NetworkPolicy
needed for OpenShell Kubernetes sandboxes. Install it once in every
platform-managed workspace namespace. It does not create a namespace or deploy
an OpenShell gateway.

Install the gateway chart with `workspaceResources.enabled=false`, then install
this chart with the gateway ServiceAccount identity:

```shell
helm install openshell-workspace ./deploy/helm/openshell-workspace \
--namespace app-a \
--set gateway.serviceAccount.name=openshell \
--set gateway.serviceAccount.namespace=openshell
```

## Values

| Key | Type | Default | Description |
|-----|------|---------|-------------|
| fullnameOverride | string | `""` | Override the full generated resource name. |
| gateway.networkPolicy.podSelector | object | `{"app.kubernetes.io/instance":"openshell","app.kubernetes.io/name":"openshell"}` | Labels selecting gateway pods allowed to reach sandbox SSH. |
| gateway.serviceAccount.name | string | `"openshell"` | Name of the shared gateway ServiceAccount. |
| gateway.serviceAccount.namespace | string | `"openshell"` | Namespace containing the shared gateway ServiceAccount. |
| nameOverride | string | `""` | Override the chart name used in generated resource names. |
| networkPolicy.enabled | bool | `true` | Restrict sandbox SSH ingress to the shared gateway pods. |
| sandboxServiceAccount.annotations | object | `{}` | Annotations added to the generated sandbox ServiceAccount. |
| sandboxServiceAccount.create | bool | `true` | Create the ServiceAccount assigned to sandbox pods. |
| sandboxServiceAccount.name | string | `"openshell-sandbox"` | Sandbox ServiceAccount name. |

----------------------------------------------
Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2)
27 changes: 27 additions & 0 deletions deploy/helm/openshell-workspace/README.md.gotmpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# OpenShell Workspace Helm Chart

<!--
This file is generated by helm-docs.
Edit README.md.gotmpl and values.yaml, then run `mise run helm:docs`.
-->

> **Experimental** - the shared-gateway, multi-namespace deployment path is
> under active design.

This chart installs the namespace-scoped ServiceAccount, RBAC, and NetworkPolicy
needed for OpenShell Kubernetes sandboxes. Install it once in every
platform-managed workspace namespace. It does not create a namespace or deploy
an OpenShell gateway.

Install the gateway chart with `workspaceResources.enabled=false`, then install
this chart with the gateway ServiceAccount identity:

```shell
helm install openshell-workspace ./deploy/helm/openshell-workspace \
--namespace app-a \
--set gateway.serviceAccount.name=openshell \
--set gateway.serviceAccount.namespace=openshell
```

{{ template "chart.valuesSection" . }}
{{ template "helm-docs.versionFooter" . }}
44 changes: 44 additions & 0 deletions deploy/helm/openshell-workspace/templates/_helpers.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "openshell-workspace.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}

{{/*
Create a default fully qualified app name.
*/}}
{{- define "openshell-workspace.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}

{{/*
Common labels.
*/}}
{{- define "openshell-workspace.labels" -}}
helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
app.kubernetes.io/name: {{ include "openshell-workspace.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}

{{/*
Validate required cross-namespace gateway identity values.
*/}}
{{- define "openshell-workspace.validateValues" -}}
{{- required "gateway.serviceAccount.name is required" .Values.gateway.serviceAccount.name -}}
{{- required "gateway.serviceAccount.namespace is required" .Values.gateway.serviceAccount.namespace -}}
{{- required "sandboxServiceAccount.name is required" .Values.sandboxServiceAccount.name -}}
{{- end }}
30 changes: 30 additions & 0 deletions deploy/helm/openshell-workspace/templates/networkpolicy.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

{{- include "openshell-workspace.validateValues" . -}}
{{- if .Values.networkPolicy.enabled }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ include "openshell-workspace.fullname" . }}-sandbox-ssh
namespace: {{ .Release.Namespace }}
labels:
{{- include "openshell-workspace.labels" . | nindent 4 }}
spec:
podSelector:
matchLabels:
openshell.ai/managed-by: openshell
policyTypes:
- Ingress
ingress:
- from:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: {{ .Values.gateway.serviceAccount.namespace }}
podSelector:
matchLabels:
{{- toYaml .Values.gateway.networkPolicy.podSelector | nindent 14 }}
ports:
- protocol: TCP
port: 2222
{{- end }}
39 changes: 39 additions & 0 deletions deploy/helm/openshell-workspace/templates/role.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

{{- include "openshell-workspace.validateValues" . -}}
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ include "openshell-workspace.fullname" . }}-sandbox
namespace: {{ .Release.Namespace }}
labels:
{{- include "openshell-workspace.labels" . | nindent 4 }}
rules:
- apiGroups:
- agents.x-k8s.io
resources:
- sandboxes
- sandboxes/status
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- pods
verbs:
- get
19 changes: 19 additions & 0 deletions deploy/helm/openshell-workspace/templates/rolebinding.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

{{- include "openshell-workspace.validateValues" . -}}
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ include "openshell-workspace.fullname" . }}-sandbox
namespace: {{ .Release.Namespace }}
labels:
{{- include "openshell-workspace.labels" . | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: {{ include "openshell-workspace.fullname" . }}-sandbox
subjects:
- kind: ServiceAccount
name: {{ .Values.gateway.serviceAccount.name }}
namespace: {{ .Values.gateway.serviceAccount.namespace }}
17 changes: 17 additions & 0 deletions deploy/helm/openshell-workspace/templates/serviceaccount.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

{{- include "openshell-workspace.validateValues" . -}}
{{- if .Values.sandboxServiceAccount.create }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ .Values.sandboxServiceAccount.name }}
namespace: {{ .Release.Namespace }}
labels:
{{- include "openshell-workspace.labels" . | nindent 4 }}
{{- with .Values.sandboxServiceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
Loading