NVIDIA · mikeyrcamp · Sep 2, 2026
@@ -98,9 +98,36 @@ jobs:
       - name: Lint and render every chart with a CI values file
         run: ./tools/ci/check-helm-charts
 
+      - name: Run control-plane isolation render regressions
+        run: |
+          sh migrations/cassandra/tests/test-execute-sqls.sh
+          bash deploy/helm/cassandra/helm/scripts/test-bitnami-upgrade-identity.sh
+          bash migrations/openbao/tests/namespace-isolation-test.sh
+          bash deploy/helm/gateway-routes/scripts/test-render-routes.sh
+          bash deploy/helm/admin-token-issuer-proxy/scripts/test-gateway-namespace-isolation.sh
+          bash deploy/helm/cloud-functions/nvcf-api/scripts/test-account-bootstrap-render.sh
+          bash deploy/helm/openbao/helm/scripts/test-control-plane-isolation.sh
+          bash deploy/helm/icms/scripts/test-control-plane-isolation.sh
+          make -C deploy/helm/nvca-operator test-control-plane-isolation
+          make -C deploy/helm/llm-request-router check-pki-render
+
       - name: Run self-managed Helmfile render tests
         run: make -C deploy/stacks/self-managed test
 
+      - name: Run compute-plane Helmfile render and lifecycle tests
+        run: make -C deploy/stacks/nvcf-compute-plane test-local
+
+      - name: Run compute-plane Make safety regression on Alpine 3.20
+        run: |
+          docker run --rm \
+            --volume "${GITHUB_WORKSPACE}:/workspace:ro" \
+            --workdir /workspace \
+            alpine:3.20 sh -ec '
+              apk add --no-cache bash make yq >/dev/null
+              make --version | sed -n "1p"
+              make -C deploy/stacks/nvcf-compute-plane test-control-plane-id-safety
+            '
+
       - name: Check for uncommitted helm dependency artifacts
         run: |
           set -euo pipefail
@@ -134,6 +161,7 @@ jobs:
         run: |
           make -C tools/ncp-local-cluster test-cluster-lifecycle-make
           make -C tools/ncp-local-cluster test-multicluster-make
+          make -C tools/ncp-local-cluster test-isolated-control-plane-gateways
 
       # The repo tooling modules carry tests that no workflow ran, so a pull
       # request could break them and still go green. tools/docs-version-sync

@@ -45,3 +45,6 @@ jobs:
 
       - name: Run kv write retry test
         run: migrations/openbao/tests/kv-write-retry-test.sh
+
+      - name: Run control-plane namespace isolation test
+        run: migrations/openbao/tests/namespace-isolation-test.sh
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 {{- if .Values.adminIssuerProxy.gateway.enabled }}
+{{- $routeNamespace := .Values.adminIssuerProxy.gateway.routeNamespace | default .Values.adminIssuerProxy.gateway.namespace }}
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -33,7 +34,7 @@ apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
 metadata:
   name: {{ include "admin-issuer-proxy.fullname" . }}
-  namespace: {{ .Values.adminIssuerProxy.gateway.namespace }}
+  namespace: {{ $routeNamespace }}
 spec:
   parentRefs:
   - name: {{ .Values.adminIssuerProxy.gateway.gatewayRef.name }}

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 {{- if .Values.adminIssuerProxy.gateway.enabled }}
+{{- $routeNamespace := .Values.adminIssuerProxy.gateway.routeNamespace | default .Values.adminIssuerProxy.gateway.namespace }}
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -38,7 +39,7 @@ spec:
   from:
     - group: gateway.networking.k8s.io
       kind: HTTPRoute
-      namespace: {{ .Values.adminIssuerProxy.gateway.namespace }}
+      namespace: {{ $routeNamespace }}
   to:
     - group: ""
       kind: Service

@@ -150,6 +150,10 @@ adminIssuerProxy:
     # Namespace where the Gateway resource is located
     namespace: envoy-gateway-system
 
+    # Namespace where the HTTPRoute is created. Empty preserves the legacy
+    # behavior by using the Gateway namespace.
+    routeNamespace: ""
+
     # Reference to the Gateway resource to attach this HTTPRoute to
     gatewayRef:
       name: shared-gw

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+chart_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../chart" && pwd)"
+tmpdir="$(mktemp -d)"
+trap 'rm -rf "$tmpdir"' EXIT
+
+render() {
+  local output="$1"
+  shift
+  helm template admin-token-issuer-proxy "$chart_dir" \
+    --namespace plane-a-api-keys \
+    --set-string adminIssuerProxy.image.registry=example.invalid \
+    --set-string adminIssuerProxy.image.repository=admin-token-issuer-proxy \
+    --set-string adminIssuerProxy.gateway.namespace=envoy-gateway-system \
+    --set-string adminIssuerProxy.gateway.gatewayRef.name=plane-a-shared-gw \
+    "$@" >"$output"
+}
+
+render "$tmpdir/legacy.yaml"
+render "$tmpdir/isolated.yaml" \
+  --set-string adminIssuerProxy.gateway.routeNamespace=plane-a-ingress
+
+ruby -ryaml -e '
+  file, expected_route_namespace = ARGV
+  docs = YAML.load_stream(File.read(file)).compact
+  route = docs.find { |doc| doc["kind"] == "HTTPRoute" }
+  grant = docs.find { |doc| doc["kind"] == "ReferenceGrant" }
+  abort "missing HTTPRoute or ReferenceGrant" unless route && grant
+  abort "wrong route namespace" unless route.dig("metadata", "namespace") == expected_route_namespace
+  abort "wrong Gateway parent namespace" unless route.dig("spec", "parentRefs", 0, "namespace") == "envoy-gateway-system"
+  abort "wrong ReferenceGrant source namespace" unless grant.dig("spec", "from", 0, "namespace") == expected_route_namespace
+  abort "wrong backend namespace" unless route.dig("spec", "rules", 0, "backendRefs", 0, "namespace") == "plane-a-api-keys"
+' "$tmpdir/legacy.yaml" envoy-gateway-system
+
+ruby -ryaml -e '
+  file, expected_route_namespace = ARGV
+  docs = YAML.load_stream(File.read(file)).compact
+  route = docs.find { |doc| doc["kind"] == "HTTPRoute" }
+  grant = docs.find { |doc| doc["kind"] == "ReferenceGrant" }
+  abort "missing HTTPRoute or ReferenceGrant" unless route && grant
+  abort "wrong isolated route namespace" unless route.dig("metadata", "namespace") == expected_route_namespace
+  abort "wrong isolated Gateway parent namespace" unless route.dig("spec", "parentRefs", 0, "namespace") == "envoy-gateway-system"
+  abort "wrong isolated ReferenceGrant source namespace" unless grant.dig("spec", "from", 0, "namespace") == expected_route_namespace
+' "$tmpdir/isolated.yaml" plane-a-ingress
+
+echo "Admin token issuer route namespace isolation checks passed."
@@ -68,6 +68,7 @@ Important settings to review before deployment:
 
 - `cassandra.image.*` for the main Cassandra image
 - `cassandra.migrations.image.*` for the migrations job image
+- `cassandra.migrations.controlPlaneID` for a named, namespace-isolated control plane; empty preserves legacy single-plane authorization URLs
 - `cassandra.global.imagePullSecrets` for private registry access
 - `cassandra.replicaCount`, `cassandra.cluster.*`, and storage settings for your environment
 - `cassandra.dbUser.*` and `cassandra.serviceRolePassword` for database credentials

@@ -19,16 +19,13 @@ Phase 4 tested an in-place over-the-top upgrade on k3d (old Bitnami stack with a
 known dataset, then upgrade to the new chart/image). Three concrete obstacles
 surfaced, each with evidence:
 
-1. StatefulSet immutability. A direct `helm upgrade` from the Bitnami-subchart
-   release to the in-house chart fails:
-   `StatefulSet.apps "cassandra" is invalid: spec: Forbidden: updates to
-   statefulset spec for fields other than 'replicas', 'ordinals', 'template',
-   'updateStrategy', 'persistentVolumeClaimRetentionPolicy' and
-   'minReadySeconds' are forbidden`. The new chart intentionally removes the
-   existing `app.kubernetes.io/name` and `app.kubernetes.io/instance` labels
-   from `volumeClaimTemplates.metadata.labels`. Kubernetes treats the entire
-   volume claim template as immutable, so the StatefulSet must be recreated
-   instead of updated in place.
+1. StatefulSet immutability. Early versions of the in-house chart removed the
+   `apiVersion`, `kind`, and the existing `app.kubernetes.io/name` and
+   `app.kubernetes.io/instance` labels from
+   `volumeClaimTemplates.metadata`. Kubernetes treats the entire volume claim
+   template as immutable, so the API server rejected a normal Helm upgrade.
+   The chart now preserves the exact identity emitted by published chart
+   0.15.5, and CI guards that contract. Do not remove or rename those fields.
 
 2. Data-layout nesting. Bitnami stored data nested under the mount:
    `<pvc>/data/{data,commitlog,hints,saved_caches}` with
@@ -49,7 +46,7 @@ the old and new charts, so PVC adoption is mechanically possible. The UID
 difference (old 1001, new 999) is handled by the pod `fsGroup: 999`; in testing
 the new image read the old files without a permission error.
 
-## StatefulSet immutability and the recreate step
+## StatefulSet immutability and the data-safe upgrade
 
 This is the mechanical heart of an in-place migration, so it is worth spelling
 out.
@@ -61,59 +58,38 @@ fields you may change on an existing StatefulSet are `replicas`, `ordinals`,
 `serviceName`, `podManagementPolicy`, and `volumeClaimTemplates`.
 
 The old and new StatefulSets share the name, selector, service name,
-`podManagementPolicy`, and the `volumeClaimTemplates.spec` fields. The only
-immutable-field difference is in `volumeClaimTemplates.metadata.labels`. The
-new chart intentionally removes the existing `app.kubernetes.io/name` and
-`app.kubernetes.io/instance` labels.
-
-So `helm upgrade` applies the new chart onto the existing `cassandra`
-StatefulSet, and the API server rejects it with the Forbidden error above. You
-cannot reshape one StatefulSet into a structurally different one in place. This
-is a Kubernetes constraint, not a chart defect.
-
-The recreate step works because the data lives in a PersistentVolumeClaim
-(`data-cassandra-0`) whose lifecycle is independent of the StatefulSet
-controller object. Deleting the StatefulSet does not delete its PVCs;
-StatefulSet-managed PVCs are retained by default (this is what
-`persistentVolumeClaimRetentionPolicy` governs, and its default is Retain). The
-runbook:
-
-1. `kubectl delete statefulset cassandra --cascade=orphan` removes only the
-   StatefulSet controller object. `--cascade=orphan` also leaves the running
-   pods; the PVC is kept regardless of cascade mode.
-2. Delete the old pod so the new controller starts a fresh one on the official
-   image. The PVC stays.
-3. `helm upgrade` creates a brand-new StatefulSet named `cassandra`. A create is
-   not an update, so there is no immutability check.
-4. Because the new StatefulSet has the same name and the same
-   volumeClaimTemplate name (`data`), it re-adopts the existing
-   `data-cassandra-0` PVC by name instead of provisioning a new empty one.
-   StatefulSets bind to a matching existing PVC and never recreate one that is
-   already present.
-
-Net effect: the controller object is swapped, the data volume is untouched and
-re-adopted, and the new pod comes up on the old data. In the Phase 4 test the
-PVC stayed `Bound` throughout and the new UID-999 pod mounted and read it.
+`podManagementPolicy`, and the `volumeClaimTemplates` identity and spec fields.
+The in-house chart must keep that immutable subset byte-for-byte equivalent to
+the Kubernetes-normalized 0.15.5 object. The regression test at
+`helm/scripts/test-bitnami-upgrade-identity.sh` enforces the fields that caused
+the original rejection; live upgrade validation must additionally confirm that
+the StatefulSet UID and PVC UID remain unchanged.
+
+With that immutable identity preserved, a normal `helm upgrade` updates the
+existing StatefulSet in place. Set `cassandra.persistence.subPath: data` for
+this one-time transition so the new UID-999 container sees the Bitnami layout,
+and retain the compatibility keys in `cassandra.config`. Do not delete or
+recreate the StatefulSet or PVC as part of the normal path.
+
+The published-0.15.5-to-source validation kept the StatefulSet, PVC, and PV
+UIDs unchanged, retained a pre-upgrade sentinel row, and reached Ready on the
+official image. The pod was replaced, as expected for an image and pod-template
+change; the controller and storage objects were not.
 
 Two caveats:
-- This is a one-time migration hop (Bitnami-shaped StatefulSet to
-  in-house-shaped StatefulSet). Ordinary upgrades within the in-house chart
-  later do not hit this, because the StatefulSet spec shape stays stable, unless
-  a future change edits a `volumeClaimTemplate` field (which would trip the same
-  rule).
-- The orphan-delete is a manual, operator-error-prone step (a wrong flag or
-  target can delete more than intended). That risk is one of the reasons Option
-  C (backup and restore) is the safer path for production.
+- This is a one-time data-layout compatibility setting. Keep `subPath: data`
+  for that release after the transition; changing it later changes where the
+  node looks for its files.
+- Any future edit to a `volumeClaimTemplate` field will trip the same immutable
+  StatefulSet rule and must be rejected by upgrade testing.
 
 ## Options
 
 ### Option A: in-place adopt (legacy layout + config compat)
 
 Reuse the existing PVC in place. Mechanics:
-- Recreate the StatefulSet: `kubectl delete statefulset cassandra
-  --cascade=orphan` (keeps the pod and PVC), delete the old pod (the PVC
-  persists), then `helm upgrade` so the new StatefulSet adopts
-  `data-cassandra-0`.
+- Use a normal `helm upgrade`; the chart preserves the published 0.15.5
+  StatefulSet and volume-claim-template identities.
 - Align the layout: set `persistence.subPath: data` so the old nested
   `data/{data,commitlog,...}` surfaces at the official defaults
   `/var/lib/cassandra/*` with no cassandra.yaml directory edits. This is
@@ -124,15 +100,10 @@ Reuse the existing PVC in place. Mechanics:
   is not yet enumerated.
 
 
-The orphan-delete-and-recreate runbook is scripted with safety checks at
-`upgrade/migrate-from-bitnami.sh` (dry-run by default). It is provisional until
-the config-compat set and strategy below are settled.
-
 Pros: no downtime beyond the pod restart, no data copy, keeps the existing PVC.
-Cons: relies on the orphan-delete runbook (operator error prone), and on
-enumerating every cassandra.yaml setting the old fleet used. Config drift risk:
-if an old setting is missed, the node fails to start on real data. Leaves the
-data physically in the Bitnami nesting.
+Cons: relies on enumerating every cassandra.yaml setting the old fleet used.
+Config drift risk: if an old setting is missed, the node fails to start on real
+data. Leaves the data physically in the Bitnami nesting.
 
 ### Option B: data relocation on first upgrade
 
@@ -162,8 +133,8 @@ cutover; more operator steps; larger data means longer restore.
 Given we are pre-1.0.0 and want a clean result:
 - Ship Option A as the convenience path for environments that want in-place
   adoption, but only after the full cassandra.yaml config-compat set is
-  enumerated and encoded, and with the orphan-delete-and-recreate documented as
-  a supported runbook.
+  enumerated and encoded, with the published-to-source UID and data-retention
+  regression kept as a release gate.
 - Document Option C (backup/restore) as the recommended, safest path,
   especially for production, and as the fallback when in-place adoption is not
   acceptable.
@@ -190,7 +161,7 @@ Open questions for Brad:
 ## Phase 4 evidence
 
 - Fresh install (single and multi node) on the new stack: validated.
-- In-place upgrade: PVC adoption mechanically works; `subPath: data` surfaces
-  the old data and the UID-999 image reads the UID-1001 files; blocked by the
-  `uuid_sstable_identifiers_enabled` config mismatch and gated behind the
-  StatefulSet recreate runbook.
+- In-place upgrade from published 0.15.5: validated with unchanged StatefulSet,
+  PVC, and PV UIDs and a retained sentinel row. `subPath: data` surfaces the old
+  data and the chart's compatibility config lets the UID-999 image read the
+  UID-1001 files.
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Guard the immutable StatefulSet identity used by the published 0.15.5 chart.
+# Kubernetes rejects a Helm upgrade when any volumeClaimTemplate metadata is
+# removed, even though the PVC name and storage request remain unchanged.
+
+set -euo pipefail
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+chart_dir="$(cd "${script_dir}/.." && pwd)"
+render="$(mktemp)"
+trap 'rm -f "${render}"' EXIT
+
+release="cassandra"
+helm template "${release}" "${chart_dir}" \
+  --namespace plane-a-cassandra-system \
+  --show-only templates/statefulset.yaml >"${render}"
+
+if ! grep -Fqx -- '      serviceAccountName: default' "${render}"; then
+  echo "upgraded StatefulSet must explicitly leave the removed Bitnami ServiceAccount" >&2
+  exit 1
+fi
+
+vct="$({ sed -n '/^  volumeClaimTemplates:/,$p' "${render}"; } )"
+
+assert_contains() {
+  local expected="$1"
+  if ! grep -Fqx -- "${expected}" <<<"${vct}"; then
+    echo "missing published-0.15.5 volumeClaimTemplate identity: ${expected}" >&2
+    sed -n '/^  volumeClaimTemplates:/,$p' "${render}" >&2
+    exit 1
+  fi
+}
+
+assert_contains '    - apiVersion: v1'
+assert_contains '      kind: PersistentVolumeClaim'
+assert_contains '          app.kubernetes.io/instance: cassandra'
+assert_contains '          app.kubernetes.io/name: cassandra'
+assert_contains '        name: data'
+
+echo "Cassandra StatefulSet preserves the published 0.15.5 immutable PVC-template identity."
@@ -55,6 +55,8 @@ spec:
               value: {{ .Values.cassandra.serviceRolePassword | quote }}
             - name: REPLICA_COUNT
               value: {{ .Values.cassandra.replicaCount | quote }}
+            - name: CONTROL_PLANE_ID
+              value: {{ .Values.cassandra.migrations.controlPlaneID | quote }}
           resources:
             {{- toYaml .Values.cassandra.hooks.migrations.resources | nindent 12 }}
 {{- end }}
@@ -32,6 +32,12 @@ spec:
         # Roll pods when the auth config changes so the initContainer re-patches.
         checksum/cassandra-conf: {{ include (print $.Template.BasePath "/configmap-cassandra-conf.yaml") . | sha256sum }}
     spec:
+      # The published Bitnami-backed chart used a release-owned ServiceAccount
+      # named cassandra. The in-house chart does not need Kubernetes API
+      # permissions and removes that object. Set the replacement explicitly so
+      # an upgrade does not retain the deleted ServiceAccount through the
+      # three-way patch and leave the replacement pod forbidden at admission.
+      serviceAccountName: default
       {{- include "cassandra.imagePullSecrets" . | nindent 6 }}
       securityContext:
         fsGroup: {{ .Values.cassandra.podSecurityContext.fsGroup }}
@@ -204,8 +210,15 @@ spec:
             name: cassandra-init-cql
             defaultMode: 0500
   volumeClaimTemplates:
-    - metadata:
+    # Preserve the object shape emitted by the Bitnami subchart used through
+    # helm-nvcf-cassandra 0.15.5. These fields are persisted inside the
+    # StatefulSet's immutable volumeClaimTemplates, so removing them makes a
+    # normal Helm upgrade fail before the pod can roll to the in-house chart.
+    - apiVersion: v1
+      kind: PersistentVolumeClaim
+      metadata:
         name: data
+        labels: {{- include "cassandra.selectorLabels" . | nindent 10 }}
       spec:
         accessModes: ["ReadWriteOnce"]
         resources: