diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 319031b1d1..4f9af12206 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -355,6 +355,26 @@ helm -n openshell get values openshell | grep sandboxNamespace Then inspect sandbox resources in that namespace. +For a split release, the gateway values should have +`workspaceResources.enabled=false`, and the target namespace should contain a +separate `openshell-workspace` release: + +```bash +helm -n openshell get values openshell | grep -A2 workspaceResources +helm -n status openshell-workspace +kubectl -n get serviceaccount,role,rolebinding,networkpolicy \ + -l app.kubernetes.io/instance=openshell-workspace +kubectl auth can-i create sandboxes.agents.x-k8s.io \ + --namespace \ + --as system:serviceaccount:openshell:openshell +``` + +If the gateway cannot create or watch sandboxes, verify the workspace +RoleBinding subject matches the gateway ServiceAccount name and namespace. +If SSH relay connections fail, verify the workspace NetworkPolicy selects the +gateway's actual `app.kubernetes.io/name` and +`app.kubernetes.io/instance` labels. + Check the configured sandbox service account when TokenReview bootstrap or sandbox registration fails. Helm creates a dedicated sandbox service account by default and writes it to `[openshell.drivers.kubernetes].service_account_name`; diff --git a/.github/actions/release-helm-oci/action.yml b/.github/actions/release-helm-oci/action.yml index d20691ad0f..46649e0cb0 100644 --- a/.github/actions/release-helm-oci/action.yml +++ b/.github/actions/release-helm-oci/action.yml @@ -4,7 +4,7 @@ name: Release Helm OCI description: > Patch chart version/appVersion, refuse duplicate OCI versions on public - releases, package the chart, and push to GHCR OCI. + releases, package the gateway and workspace charts, and push them to GHCR OCI. inputs: chart-version: @@ -51,11 +51,16 @@ runs: shell: bash run: | set -euo pipefail - CHART_DIR="${RUNNER_TEMP}/chart-build" - cp -a deploy/helm/openshell/. "${CHART_DIR}" - sed -i "s/^version:.*/version: ${CHART_VERSION}/" "${CHART_DIR}/Chart.yaml" - sed -i "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/" "${CHART_DIR}/Chart.yaml" - echo "chart_dir=${CHART_DIR}" >> "$GITHUB_OUTPUT" + GATEWAY_CHART_DIR="${RUNNER_TEMP}/gateway-chart-build" + WORKSPACE_CHART_DIR="${RUNNER_TEMP}/workspace-chart-build" + cp -a deploy/helm/openshell/. "${GATEWAY_CHART_DIR}" + cp -a deploy/helm/openshell-workspace/. "${WORKSPACE_CHART_DIR}" + for chart_dir in "${GATEWAY_CHART_DIR}" "${WORKSPACE_CHART_DIR}"; do + sed -i "s/^version:.*/version: ${CHART_VERSION}/" "${chart_dir}/Chart.yaml" + sed -i "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/" "${chart_dir}/Chart.yaml" + done + echo "gateway_chart_dir=${GATEWAY_CHART_DIR}" >> "$GITHUB_OUTPUT" + echo "workspace_chart_dir=${WORKSPACE_CHART_DIR}" >> "$GITHUB_OUTPUT" echo "chart_version=${CHART_VERSION}" >> "$GITHUB_OUTPUT" - name: Refuse duplicate chart version @@ -65,38 +70,52 @@ runs: shell: bash run: | set -euo pipefail - OCI_CHART="oci://ghcr.io/nvidia/openshell/helm-chart" - if helm show chart "${OCI_CHART}" --version "${CHART_VERSION}" >/dev/null 2>&1; then - echo "::error::Chart ${CHART_VERSION} is already published. Use a new tag or delete the existing package first." - exit 1 - fi + for chart in helm-chart openshell-workspace; do + OCI_CHART="oci://ghcr.io/nvidia/openshell/${chart}" + if helm show chart "${OCI_CHART}" --version "${CHART_VERSION}" >/dev/null 2>&1; then + echo "::error::Chart ${chart}:${CHART_VERSION} is already published. Use a new tag or delete the existing package first." + exit 1 + fi + done - name: Package Helm chart env: - CHART_DIR: ${{ steps.prep.outputs.chart_dir }} + GATEWAY_CHART_DIR: ${{ steps.prep.outputs.gateway_chart_dir }} + WORKSPACE_CHART_DIR: ${{ steps.prep.outputs.workspace_chart_dir }} shell: bash run: | set -euo pipefail - helm package "${CHART_DIR}" --destination /tmp - ls /tmp/helm-chart-*.tgz + mkdir -p /tmp/helm-charts + helm package "${GATEWAY_CHART_DIR}" --destination /tmp/helm-charts + helm package "${WORKSPACE_CHART_DIR}" --destination /tmp/helm-charts + ls /tmp/helm-charts/*.tgz - name: Push Helm chart to GHCR OCI shell: bash run: | set -euo pipefail - helm push /tmp/helm-chart-*.tgz oci://ghcr.io/nvidia/openshell + for archive in /tmp/helm-charts/*.tgz; do + helm push "${archive}" oci://ghcr.io/nvidia/openshell + done - name: Push SHA-pinned chart if: inputs.pin-sha != '' env: PIN_SHA: ${{ inputs.pin-sha }} - CHART_DIR: ${{ steps.prep.outputs.chart_dir }} + GATEWAY_CHART_DIR: ${{ steps.prep.outputs.gateway_chart_dir }} + WORKSPACE_CHART_DIR: ${{ steps.prep.outputs.workspace_chart_dir }} shell: bash run: | set -euo pipefail - SHA_CHART_DIR="${RUNNER_TEMP}/chart-build-sha" - cp -a "${CHART_DIR}/." "${SHA_CHART_DIR}" - sed -i "s/^version:.*/version: 0.0.0-dev.${PIN_SHA}/" "${SHA_CHART_DIR}/Chart.yaml" - sed -i "s/^appVersion:.*/appVersion: \"${PIN_SHA}\"/" "${SHA_CHART_DIR}/Chart.yaml" - helm package "${SHA_CHART_DIR}" --destination /tmp/sha-pin - helm push /tmp/sha-pin/helm-chart-*.tgz oci://ghcr.io/nvidia/openshell + mkdir -p /tmp/sha-pin + for source_dir in "${GATEWAY_CHART_DIR}" "${WORKSPACE_CHART_DIR}"; do + chart_name="$(basename "${source_dir}")" + sha_chart_dir="${RUNNER_TEMP}/${chart_name}-sha" + cp -a "${source_dir}/." "${sha_chart_dir}" + sed -i "s/^version:.*/version: 0.0.0-dev.${PIN_SHA}/" "${sha_chart_dir}/Chart.yaml" + sed -i "s/^appVersion:.*/appVersion: \"${PIN_SHA}\"/" "${sha_chart_dir}/Chart.yaml" + helm package "${sha_chart_dir}" --destination /tmp/sha-pin + done + for archive in /tmp/sha-pin/*.tgz; do + helm push "${archive}" oci://ghcr.io/nvidia/openshell + done diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 646b6320bd..699616e08e 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -142,6 +142,15 @@ through the driver configuration. The Helm chart defaults sandbox agents to `Unconfined` so runtime/default AppArmor profiles do not block supervisor network namespace setup on AppArmor-enabled nodes. +The Kubernetes deployment packaging has two ownership boundaries. The gateway +chart owns the gateway workload, configuration, Services, PKI, and +cluster-scoped gateway resources. It can retain the legacy combined behavior, +or omit workspace resources. The workspace chart is installed into a +pre-provisioned sandbox namespace and owns only the sandbox ServiceAccount, +namespaced RBAC, and sandbox ingress NetworkPolicy. Its RoleBinding names the +gateway ServiceAccount and namespace explicitly, so the two releases have +disjoint lifecycle ownership. + Resource requirements enter the driver layer through `SandboxSpec.resource_requirements`. This includes a set of GPU requirements, where a user can request a specific number of GPUs or the driver-specific default behaviour. For all in-tree drivers, this is equivalent to selecting a single GPU. diff --git a/deploy/helm/openshell-workspace/Chart.yaml b/deploy/helm/openshell-workspace/Chart.yaml new file mode 100644 index 0000000000..03a3919d46 --- /dev/null +++ b/deploy/helm/openshell-workspace/Chart.yaml @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v2 +name: openshell-workspace +description: Namespace-scoped prerequisites for OpenShell Kubernetes sandboxes +type: application +version: 0.0.0 +appVersion: "0.0.0" diff --git a/deploy/helm/openshell-workspace/README.md b/deploy/helm/openshell-workspace/README.md new file mode 100644 index 0000000000..7d724db5b7 --- /dev/null +++ b/deploy/helm/openshell-workspace/README.md @@ -0,0 +1,41 @@ +# OpenShell Workspace Helm Chart + + + +> **Experimental** - the shared-gateway, multi-namespace deployment path is +> under active design. + +This chart installs the namespace-scoped ServiceAccount, RBAC, and NetworkPolicy +needed for OpenShell Kubernetes sandboxes. Install it once in every +platform-managed workspace namespace. It does not create a namespace or deploy +an OpenShell gateway. + +Install the gateway chart with `workspaceResources.enabled=false`, then install +this chart with the gateway ServiceAccount identity: + +```shell +helm install openshell-workspace ./deploy/helm/openshell-workspace \ + --namespace app-a \ + --set gateway.serviceAccount.name=openshell \ + --set gateway.serviceAccount.namespace=openshell +``` + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| fullnameOverride | string | `""` | Override the full generated resource name. | +| gateway.networkPolicy.podSelector | object | `{"app.kubernetes.io/instance":"openshell","app.kubernetes.io/name":"openshell"}` | Labels selecting gateway pods allowed to reach sandbox SSH. | +| gateway.serviceAccount.name | string | `"openshell"` | Name of the shared gateway ServiceAccount. | +| gateway.serviceAccount.namespace | string | `"openshell"` | Namespace containing the shared gateway ServiceAccount. | +| nameOverride | string | `""` | Override the chart name used in generated resource names. | +| networkPolicy.enabled | bool | `true` | Restrict sandbox SSH ingress to the shared gateway pods. | +| sandboxServiceAccount.annotations | object | `{}` | Annotations added to the generated sandbox ServiceAccount. | +| sandboxServiceAccount.create | bool | `true` | Create the ServiceAccount assigned to sandbox pods. | +| sandboxServiceAccount.name | string | `"openshell-sandbox"` | Sandbox ServiceAccount name. | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2) diff --git a/deploy/helm/openshell-workspace/README.md.gotmpl b/deploy/helm/openshell-workspace/README.md.gotmpl new file mode 100644 index 0000000000..7e9cafced4 --- /dev/null +++ b/deploy/helm/openshell-workspace/README.md.gotmpl @@ -0,0 +1,27 @@ +# OpenShell Workspace Helm Chart + + + +> **Experimental** - the shared-gateway, multi-namespace deployment path is +> under active design. + +This chart installs the namespace-scoped ServiceAccount, RBAC, and NetworkPolicy +needed for OpenShell Kubernetes sandboxes. Install it once in every +platform-managed workspace namespace. It does not create a namespace or deploy +an OpenShell gateway. + +Install the gateway chart with `workspaceResources.enabled=false`, then install +this chart with the gateway ServiceAccount identity: + +```shell +helm install openshell-workspace ./deploy/helm/openshell-workspace \ + --namespace app-a \ + --set gateway.serviceAccount.name=openshell \ + --set gateway.serviceAccount.namespace=openshell +``` + +{{ template "chart.valuesSection" . }} +{{ template "helm-docs.versionFooter" . }} diff --git a/deploy/helm/openshell-workspace/templates/_helpers.tpl b/deploy/helm/openshell-workspace/templates/_helpers.tpl new file mode 100644 index 0000000000..8eaaaab2b7 --- /dev/null +++ b/deploy/helm/openshell-workspace/templates/_helpers.tpl @@ -0,0 +1,44 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "openshell-workspace.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "openshell-workspace.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Common labels. +*/}} +{{- define "openshell-workspace.labels" -}} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +app.kubernetes.io/name: {{ include "openshell-workspace.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Validate required cross-namespace gateway identity values. +*/}} +{{- define "openshell-workspace.validateValues" -}} +{{- required "gateway.serviceAccount.name is required" .Values.gateway.serviceAccount.name -}} +{{- required "gateway.serviceAccount.namespace is required" .Values.gateway.serviceAccount.namespace -}} +{{- required "sandboxServiceAccount.name is required" .Values.sandboxServiceAccount.name -}} +{{- end }} diff --git a/deploy/helm/openshell-workspace/templates/networkpolicy.yaml b/deploy/helm/openshell-workspace/templates/networkpolicy.yaml new file mode 100644 index 0000000000..4a5e2c2293 --- /dev/null +++ b/deploy/helm/openshell-workspace/templates/networkpolicy.yaml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- include "openshell-workspace.validateValues" . -}} +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "openshell-workspace.fullname" . }}-sandbox-ssh + namespace: {{ .Release.Namespace }} + labels: + {{- include "openshell-workspace.labels" . | nindent 4 }} +spec: + podSelector: + matchLabels: + openshell.ai/managed-by: openshell + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.gateway.serviceAccount.namespace }} + podSelector: + matchLabels: + {{- toYaml .Values.gateway.networkPolicy.podSelector | nindent 14 }} + ports: + - protocol: TCP + port: 2222 +{{- end }} diff --git a/deploy/helm/openshell-workspace/templates/role.yaml b/deploy/helm/openshell-workspace/templates/role.yaml new file mode 100644 index 0000000000..e0d6d9624e --- /dev/null +++ b/deploy/helm/openshell-workspace/templates/role.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- include "openshell-workspace.validateValues" . -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "openshell-workspace.fullname" . }}-sandbox + namespace: {{ .Release.Namespace }} + labels: + {{- include "openshell-workspace.labels" . | nindent 4 }} +rules: + - apiGroups: + - agents.x-k8s.io + resources: + - sandboxes + - sandboxes/status + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - get diff --git a/deploy/helm/openshell-workspace/templates/rolebinding.yaml b/deploy/helm/openshell-workspace/templates/rolebinding.yaml new file mode 100644 index 0000000000..6b71d1f25b --- /dev/null +++ b/deploy/helm/openshell-workspace/templates/rolebinding.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- include "openshell-workspace.validateValues" . -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "openshell-workspace.fullname" . }}-sandbox + namespace: {{ .Release.Namespace }} + labels: + {{- include "openshell-workspace.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "openshell-workspace.fullname" . }}-sandbox +subjects: + - kind: ServiceAccount + name: {{ .Values.gateway.serviceAccount.name }} + namespace: {{ .Values.gateway.serviceAccount.namespace }} diff --git a/deploy/helm/openshell-workspace/templates/serviceaccount.yaml b/deploy/helm/openshell-workspace/templates/serviceaccount.yaml new file mode 100644 index 0000000000..f30663caa4 --- /dev/null +++ b/deploy/helm/openshell-workspace/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- include "openshell-workspace.validateValues" . -}} +{{- if .Values.sandboxServiceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.sandboxServiceAccount.name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "openshell-workspace.labels" . | nindent 4 }} + {{- with .Values.sandboxServiceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/deploy/helm/openshell-workspace/tests/workspace_test.yaml b/deploy/helm/openshell-workspace/tests/workspace_test.yaml new file mode 100644 index 0000000000..2f71920eaf --- /dev/null +++ b/deploy/helm/openshell-workspace/tests/workspace_test.yaml @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: workspace namespace resources +templates: + - templates/serviceaccount.yaml + - templates/role.yaml + - templates/rolebinding.yaml + - templates/networkpolicy.yaml +release: + name: tenant-a + namespace: app-a + +tests: + - it: creates the workspace role in the release namespace + template: templates/role.yaml + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.namespace + value: app-a + + - it: binds the shared gateway service account + template: templates/rolebinding.yaml + set: + gateway.serviceAccount.name: shared-gateway + gateway.serviceAccount.namespace: openshell-system + asserts: + - equal: + path: metadata.namespace + value: app-a + - equal: + path: subjects[0].name + value: shared-gateway + - equal: + path: subjects[0].namespace + value: openshell-system + + - it: selects gateway pods in the gateway namespace + template: templates/networkpolicy.yaml + set: + gateway.serviceAccount.namespace: openshell-system + gateway.networkPolicy.podSelector: + app.kubernetes.io/name: openshell + app.kubernetes.io/instance: central + asserts: + - equal: + path: spec.ingress[0].from[0].namespaceSelector.matchLabels["kubernetes.io/metadata.name"] + value: openshell-system + - equal: + path: spec.ingress[0].from[0].podSelector.matchLabels["app.kubernetes.io/instance"] + value: central + + - it: supports a pre-existing sandbox service account + template: templates/serviceaccount.yaml + set: + sandboxServiceAccount.create: false + asserts: + - hasDocuments: + count: 0 diff --git a/deploy/helm/openshell-workspace/values.yaml b/deploy/helm/openshell-workspace/values.yaml new file mode 100644 index 0000000000..2c52b0460d --- /dev/null +++ b/deploy/helm/openshell-workspace/values.yaml @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# -- Override the chart name used in generated resource names. +nameOverride: "" +# -- Override the full generated resource name. +fullnameOverride: "" + +gateway: + serviceAccount: + # -- Name of the shared gateway ServiceAccount. + name: openshell + # -- Namespace containing the shared gateway ServiceAccount. + namespace: openshell + networkPolicy: + # -- Labels selecting gateway pods allowed to reach sandbox SSH. + podSelector: + app.kubernetes.io/name: openshell + app.kubernetes.io/instance: openshell + +sandboxServiceAccount: + # -- Create the ServiceAccount assigned to sandbox pods. + create: true + # -- Sandbox ServiceAccount name. + name: openshell-sandbox + # -- Annotations added to the generated sandbox ServiceAccount. + annotations: {} + +networkPolicy: + # -- Restrict sandbox SSH ingress to the shared gateway pods. + enabled: true diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 7096a8ca74..6c85464493 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -9,6 +9,13 @@ Edit README.md.gotmpl and values.yaml, then run `mise run helm:docs`. This chart deploys the OpenShell gateway into a Kubernetes cluster. It is published as an OCI artifact to GHCR at `oci://ghcr.io/nvidia/openshell/helm-chart`. +By default, this chart also creates the namespace-scoped resources needed by +sandboxes. For a shared-gateway deployment, install it with +`workspaceResources.enabled=false`, then install the +`deploy/helm/openshell-workspace` chart in every pre-provisioned workspace +namespace. The gateway and workspace releases can then be upgraded and removed +independently. + ## Prerequisites The Kubernetes Agent Sandbox CRDs and controller must be installed on the cluster before deploying OpenShell. Install them with: @@ -275,6 +282,7 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | tolerations | list | `[]` | Tolerations for the gateway pod. | | workload.allowMultiReplicaStatefulSet | bool | `false` | Allow replicaCount > 1 while rendering a StatefulSet. Prefer workload.kind=deployment for external database-backed multi-replica gateways; this override exists for operators who explicitly require StatefulSet identity or storage semantics. | | workload.kind | string | `"statefulset"` | Gateway workload controller kind. Use `statefulset` for the default SQLite database, or `deployment` when server.externalDbSecret points at an external database. | +| workspaceResources.enabled | bool | `true` | Create the sandbox ServiceAccount, Role, RoleBinding, and NetworkPolicy from this chart. Disable for a gateway-only release. | ---------------------------------------------- Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2) diff --git a/deploy/helm/openshell/README.md.gotmpl b/deploy/helm/openshell/README.md.gotmpl index 0242d8118c..bc34a47e85 100644 --- a/deploy/helm/openshell/README.md.gotmpl +++ b/deploy/helm/openshell/README.md.gotmpl @@ -9,6 +9,13 @@ Edit README.md.gotmpl and values.yaml, then run `mise run helm:docs`. This chart deploys the OpenShell gateway into a Kubernetes cluster. It is published as an OCI artifact to GHCR at `oci://ghcr.io/nvidia/openshell/helm-chart`. +By default, this chart also creates the namespace-scoped resources needed by +sandboxes. For a shared-gateway deployment, install it with +`workspaceResources.enabled=false`, then install the +`deploy/helm/openshell-workspace` chart in every pre-provisioned workspace +namespace. The gateway and workspace releases can then be upgraded and removed +independently. + ## Prerequisites The Kubernetes Agent Sandbox CRDs and controller must be installed on the cluster before deploying OpenShell. Install them with: diff --git a/deploy/helm/openshell/templates/networkpolicy.yaml b/deploy/helm/openshell/templates/networkpolicy.yaml index e85571e5f5..fb72604e60 100644 --- a/deploy/helm/openshell/templates/networkpolicy.yaml +++ b/deploy/helm/openshell/templates/networkpolicy.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- if .Values.networkPolicy.enabled }} +{{- if and .Values.workspaceResources.enabled .Values.networkPolicy.enabled }} # NetworkPolicy restricting SSH ingress on sandbox pods to the gateway pod. # Sandbox pods are dynamically created by the server and labelled with # openshell.ai/managed-by=openshell. This policy ensures only the gateway diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index 5ecc4428ad..72b29dfb3d 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +{{- if .Values.workspaceResources.enabled }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -42,3 +43,4 @@ rules: - pods verbs: - get +{{- end }} diff --git a/deploy/helm/openshell/templates/rolebinding.yaml b/deploy/helm/openshell/templates/rolebinding.yaml index e5233f753c..d556a58170 100644 --- a/deploy/helm/openshell/templates/rolebinding.yaml +++ b/deploy/helm/openshell/templates/rolebinding.yaml @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +{{- if .Values.workspaceResources.enabled }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -16,3 +17,4 @@ subjects: - kind: ServiceAccount name: {{ include "openshell.serviceAccountName" . }} namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/deploy/helm/openshell/templates/serviceaccount.yaml b/deploy/helm/openshell/templates/serviceaccount.yaml index a98ad5363e..8346ecfc2a 100644 --- a/deploy/helm/openshell/templates/serviceaccount.yaml +++ b/deploy/helm/openshell/templates/serviceaccount.yaml @@ -13,10 +13,10 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} {{- end }} -{{- if and .Values.serviceAccount.create .Values.sandboxServiceAccount.create }} +{{- if and .Values.serviceAccount.create .Values.workspaceResources.enabled .Values.sandboxServiceAccount.create }} --- {{- end }} -{{- if .Values.sandboxServiceAccount.create }} +{{- if and .Values.workspaceResources.enabled .Values.sandboxServiceAccount.create }} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml index ee89fce53d..864e3a8512 100644 --- a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml +++ b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml @@ -75,3 +75,21 @@ tests: path: metadata.namespace value: other-ns documentIndex: 1 + + - it: omits workspace resources in gateway-only mode + set: + workspaceResources.enabled: false + networkPolicy.enabled: true + asserts: + - hasDocuments: + count: 1 + template: templates/gateway-config.yaml + - hasDocuments: + count: 0 + template: templates/networkpolicy.yaml + - hasDocuments: + count: 0 + template: templates/role.yaml + - hasDocuments: + count: 0 + template: templates/rolebinding.yaml diff --git a/deploy/helm/openshell/tests/sandbox_service_account_test.yaml b/deploy/helm/openshell/tests/sandbox_service_account_test.yaml index c426415823..c9f10868fe 100644 --- a/deploy/helm/openshell/tests/sandbox_service_account_test.yaml +++ b/deploy/helm/openshell/tests/sandbox_service_account_test.yaml @@ -29,3 +29,13 @@ tests: asserts: - hasDocuments: count: 1 + + - it: renders only the gateway service account in gateway-only mode + set: + workspaceResources.enabled: false + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: openshell diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 39205df1bf..8c264c202d 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -85,6 +85,14 @@ sandboxServiceAccount: # -- Existing service account name for sandbox pods when sandboxServiceAccount.create is false. name: "" +# Namespace-scoped resources needed to run sandboxes. Disable this when the +# gateway and workspace prerequisites are managed as separate Helm releases +# using the openshell-workspace chart. +workspaceResources: + # -- Create the sandbox ServiceAccount, Role, RoleBinding, and NetworkPolicy + # from this chart. Disable for a gateway-only release. + enabled: true + # -- Extra annotations to add to the gateway pod. podAnnotations: {} # -- Extra labels to add to the gateway pod. diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index c2fca827f1..7a15f52534 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -86,6 +86,33 @@ helm upgrade --install openshell \ The chart automatically generates PKI secrets on first install using pre-install Helm hooks. No manual secret creation is required. +### Split gateway and workspace releases + +For a platform-managed namespace, install the gateway without namespace-scoped +sandbox resources, then install the workspace chart in the sandbox namespace: + +```shell +helm upgrade --install openshell \ + oci://ghcr.io/nvidia/openshell/helm-chart \ + --version \ + --namespace openshell \ + --set workspaceResources.enabled=false \ + --set server.sandboxNamespace=app-a + +helm upgrade --install openshell-workspace \ + oci://ghcr.io/nvidia/openshell/openshell-workspace \ + --version \ + --namespace app-a \ + --set gateway.serviceAccount.name=openshell \ + --set gateway.serviceAccount.namespace=openshell +``` + +The workspace chart does not create the namespace or deploy a gateway. It owns +only the sandbox ServiceAccount, Role, RoleBinding, and NetworkPolicy in its +release namespace. The current gateway runtime still selects one +`server.sandboxNamespace`; mapping workspaces to several namespaces requires +the separate multi-namespace runtime feature. + ## Wait for the gateway to be ready ```shell @@ -150,6 +177,7 @@ The most commonly changed values are: | `workload.kind` | Gateway workload controller. Use `statefulset` for SQLite or `deployment` with `server.externalDbSecret`. | | `workload.allowMultiReplicaStatefulSet` | Allow `replicaCount > 1` with `workload.kind=statefulset`. Prefer Deployment for external database-backed multi-replica gateways. | | `server.sandboxNamespace` | Namespace where sandbox pods are created. Defaults to the Helm release namespace when left empty. | +| `workspaceResources.enabled` | Create namespace-scoped sandbox prerequisites from the gateway chart. Disable when installing the workspace chart separately. | | `server.externalDbSecret` | Secret containing a PostgreSQL connection URI in the `uri` key. Use when the database is managed outside the chart. | | `server.sandboxImage` | Default sandbox image used when a sandbox does not specify one. | | `server.sandboxImagePullSecrets` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | diff --git a/tasks/helm.toml b/tasks/helm.toml index 24b6667b1d..707b2ad7f9 100644 --- a/tasks/helm.toml +++ b/tasks/helm.toml @@ -4,26 +4,35 @@ # Helm chart tasks ["helm:docs"] -description = "Generate the openshell Helm chart README from Chart.yaml, values.yaml, and README.md.gotmpl" -run = "helm-docs --chart-search-root deploy/helm/openshell" +description = "Generate the OpenShell gateway and workspace Helm chart READMEs" +run = """ + helm-docs --chart-search-root deploy/helm/openshell + helm-docs --chart-search-root deploy/helm/openshell-workspace +""" ["helm:docs:check"] description = "Verify the openshell Helm chart README is generated and up to date" run = """ set -e - tmp="$(mktemp)" - trap 'rm -f "$tmp"' EXIT + gateway_tmp="$(mktemp)" + workspace_tmp="$(mktemp)" + trap 'rm -f "$gateway_tmp" "$workspace_tmp"' EXIT - helm-docs --chart-search-root deploy/helm/openshell --dry-run > "$tmp" - if ! diff -u deploy/helm/openshell/README.md "$tmp"; then + helm-docs --chart-search-root deploy/helm/openshell --dry-run > "$gateway_tmp" + if ! diff -u deploy/helm/openshell/README.md "$gateway_tmp"; then echo "Helm chart README is out of sync. Run: mise run helm:docs" >&2 exit 1 fi + helm-docs --chart-search-root deploy/helm/openshell-workspace --dry-run > "$workspace_tmp" + if ! diff -u deploy/helm/openshell-workspace/README.md "$workspace_tmp"; then + echo "Workspace Helm chart README is out of sync. Run: mise run helm:docs" >&2 + exit 1 + fi """ hide = true ["helm:lint"] -description = "Lint the openshell Helm chart (defaults + all CI configuration variants)" +description = "Lint the OpenShell gateway and workspace Helm charts" run = """ set -e helm dependency build deploy/helm/openshell @@ -36,11 +45,13 @@ run = """ echo "values files: deploy/helm/openshell/values.yaml, $f" helm lint deploy/helm/openshell -f "$f" done + echo "--- helm lint: workspace defaults ---" + helm lint deploy/helm/openshell-workspace echo "All variants passed." """ ["helm:test"] -description = "Run Helm chart unit tests" +description = "Run gateway and workspace Helm chart unit tests" run = """ set -e if ! helm plugin list | grep -q unittest; then @@ -48,6 +59,7 @@ run = """ fi helm dependency build deploy/helm/openshell helm unittest deploy/helm/openshell + helm unittest deploy/helm/openshell-workspace """ ["helm:skaffold:dev"]