diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index e887a197b..0241af532 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -98,9 +98,36 @@ jobs: - name: Lint and render every chart with a CI values file run: ./tools/ci/check-helm-charts + - name: Run control-plane isolation render regressions + run: | + sh migrations/cassandra/tests/test-execute-sqls.sh + bash deploy/helm/cassandra/helm/scripts/test-bitnami-upgrade-identity.sh + bash migrations/openbao/tests/namespace-isolation-test.sh + bash deploy/helm/gateway-routes/scripts/test-render-routes.sh + bash deploy/helm/admin-token-issuer-proxy/scripts/test-gateway-namespace-isolation.sh + bash deploy/helm/cloud-functions/nvcf-api/scripts/test-account-bootstrap-render.sh + bash deploy/helm/openbao/helm/scripts/test-control-plane-isolation.sh + bash deploy/helm/icms/scripts/test-control-plane-isolation.sh + make -C deploy/helm/nvca-operator test-control-plane-isolation + make -C deploy/helm/llm-request-router check-pki-render + - name: Run self-managed Helmfile render tests run: make -C deploy/stacks/self-managed test + - name: Run compute-plane Helmfile render and lifecycle tests + run: make -C deploy/stacks/nvcf-compute-plane test-local + + - name: Run compute-plane Make safety regression on Alpine 3.20 + run: | + docker run --rm \ + --volume "${GITHUB_WORKSPACE}:/workspace:ro" \ + --workdir /workspace \ + alpine:3.20 sh -ec ' + apk add --no-cache bash make yq >/dev/null + make --version | sed -n "1p" + make -C deploy/stacks/nvcf-compute-plane test-control-plane-id-safety + ' + - name: Check for uncommitted helm dependency artifacts run: | set -euo pipefail @@ -134,6 +161,7 @@ jobs: run: | make -C tools/ncp-local-cluster test-cluster-lifecycle-make make -C tools/ncp-local-cluster test-multicluster-make + make -C tools/ncp-local-cluster test-isolated-control-plane-gateways # The repo tooling modules carry tests that no workflow ran, so a pull # request could break them and still go green. tools/docs-version-sync diff --git a/.github/workflows/openbao-migrations.yml b/.github/workflows/openbao-migrations.yml index 584412043..24a34740d 100644 --- a/.github/workflows/openbao-migrations.yml +++ b/.github/workflows/openbao-migrations.yml @@ -45,3 +45,6 @@ jobs: - name: Run kv write retry test run: migrations/openbao/tests/kv-write-retry-test.sh + + - name: Run control-plane namespace isolation test + run: migrations/openbao/tests/namespace-isolation-test.sh diff --git a/deploy/helm/admin-token-issuer-proxy/chart/templates/httproute.yaml b/deploy/helm/admin-token-issuer-proxy/chart/templates/httproute.yaml index 71053bdf1..241894ef8 100644 --- a/deploy/helm/admin-token-issuer-proxy/chart/templates/httproute.yaml +++ b/deploy/helm/admin-token-issuer-proxy/chart/templates/httproute.yaml @@ -14,6 +14,7 @@ # limitations under the License. {{- if .Values.adminIssuerProxy.gateway.enabled }} +{{- $routeNamespace := .Values.adminIssuerProxy.gateway.routeNamespace | default .Values.adminIssuerProxy.gateway.namespace }} # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # @@ -33,7 +34,7 @@ apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: {{ include "admin-issuer-proxy.fullname" . }} - namespace: {{ .Values.adminIssuerProxy.gateway.namespace }} + namespace: {{ $routeNamespace }} spec: parentRefs: - name: {{ .Values.adminIssuerProxy.gateway.gatewayRef.name }} diff --git a/deploy/helm/admin-token-issuer-proxy/chart/templates/referencegrant.yaml b/deploy/helm/admin-token-issuer-proxy/chart/templates/referencegrant.yaml index e0e4ea023..4bef07b5e 100644 --- a/deploy/helm/admin-token-issuer-proxy/chart/templates/referencegrant.yaml +++ b/deploy/helm/admin-token-issuer-proxy/chart/templates/referencegrant.yaml @@ -14,6 +14,7 @@ # limitations under the License. {{- if .Values.adminIssuerProxy.gateway.enabled }} +{{- $routeNamespace := .Values.adminIssuerProxy.gateway.routeNamespace | default .Values.adminIssuerProxy.gateway.namespace }} # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # @@ -38,7 +39,7 @@ spec: from: - group: gateway.networking.k8s.io kind: HTTPRoute - namespace: {{ .Values.adminIssuerProxy.gateway.namespace }} + namespace: {{ $routeNamespace }} to: - group: "" kind: Service diff --git a/deploy/helm/admin-token-issuer-proxy/chart/values.yaml b/deploy/helm/admin-token-issuer-proxy/chart/values.yaml index 23ac92965..fe427d10f 100644 --- a/deploy/helm/admin-token-issuer-proxy/chart/values.yaml +++ b/deploy/helm/admin-token-issuer-proxy/chart/values.yaml @@ -150,6 +150,10 @@ adminIssuerProxy: # Namespace where the Gateway resource is located namespace: envoy-gateway-system + # Namespace where the HTTPRoute is created. Empty preserves the legacy + # behavior by using the Gateway namespace. + routeNamespace: "" + # Reference to the Gateway resource to attach this HTTPRoute to gatewayRef: name: shared-gw diff --git a/deploy/helm/admin-token-issuer-proxy/scripts/test-gateway-namespace-isolation.sh b/deploy/helm/admin-token-issuer-proxy/scripts/test-gateway-namespace-isolation.sh new file mode 100644 index 000000000..197a09def --- /dev/null +++ b/deploy/helm/admin-token-issuer-proxy/scripts/test-gateway-namespace-isolation.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +chart_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../chart" && pwd)" +tmpdir="$(mktemp -d)" +trap 'rm -rf "$tmpdir"' EXIT + +render() { + local output="$1" + shift + helm template admin-token-issuer-proxy "$chart_dir" \ + --namespace plane-a-api-keys \ + --set-string adminIssuerProxy.image.registry=example.invalid \ + --set-string adminIssuerProxy.image.repository=admin-token-issuer-proxy \ + --set-string adminIssuerProxy.gateway.namespace=envoy-gateway-system \ + --set-string adminIssuerProxy.gateway.gatewayRef.name=plane-a-shared-gw \ + "$@" >"$output" +} + +render "$tmpdir/legacy.yaml" +render "$tmpdir/isolated.yaml" \ + --set-string adminIssuerProxy.gateway.routeNamespace=plane-a-ingress + +ruby -ryaml -e ' + file, expected_route_namespace = ARGV + docs = YAML.load_stream(File.read(file)).compact + route = docs.find { |doc| doc["kind"] == "HTTPRoute" } + grant = docs.find { |doc| doc["kind"] == "ReferenceGrant" } + abort "missing HTTPRoute or ReferenceGrant" unless route && grant + abort "wrong route namespace" unless route.dig("metadata", "namespace") == expected_route_namespace + abort "wrong Gateway parent namespace" unless route.dig("spec", "parentRefs", 0, "namespace") == "envoy-gateway-system" + abort "wrong ReferenceGrant source namespace" unless grant.dig("spec", "from", 0, "namespace") == expected_route_namespace + abort "wrong backend namespace" unless route.dig("spec", "rules", 0, "backendRefs", 0, "namespace") == "plane-a-api-keys" +' "$tmpdir/legacy.yaml" envoy-gateway-system + +ruby -ryaml -e ' + file, expected_route_namespace = ARGV + docs = YAML.load_stream(File.read(file)).compact + route = docs.find { |doc| doc["kind"] == "HTTPRoute" } + grant = docs.find { |doc| doc["kind"] == "ReferenceGrant" } + abort "missing HTTPRoute or ReferenceGrant" unless route && grant + abort "wrong isolated route namespace" unless route.dig("metadata", "namespace") == expected_route_namespace + abort "wrong isolated Gateway parent namespace" unless route.dig("spec", "parentRefs", 0, "namespace") == "envoy-gateway-system" + abort "wrong isolated ReferenceGrant source namespace" unless grant.dig("spec", "from", 0, "namespace") == expected_route_namespace +' "$tmpdir/isolated.yaml" plane-a-ingress + +echo "Admin token issuer route namespace isolation checks passed." diff --git a/deploy/helm/cassandra/README.md b/deploy/helm/cassandra/README.md index f6076cfcd..1b8e0d949 100644 --- a/deploy/helm/cassandra/README.md +++ b/deploy/helm/cassandra/README.md @@ -68,6 +68,7 @@ Important settings to review before deployment: - `cassandra.image.*` for the main Cassandra image - `cassandra.migrations.image.*` for the migrations job image +- `cassandra.migrations.controlPlaneID` for a named, namespace-isolated control plane; empty preserves legacy single-plane authorization URLs - `cassandra.global.imagePullSecrets` for private registry access - `cassandra.replicaCount`, `cassandra.cluster.*`, and storage settings for your environment - `cassandra.dbUser.*` and `cassandra.serviceRolePassword` for database credentials diff --git a/deploy/helm/cassandra/docs/upgrade-from-bitnami.md b/deploy/helm/cassandra/docs/upgrade-from-bitnami.md index 33d64ef19..b256983de 100644 --- a/deploy/helm/cassandra/docs/upgrade-from-bitnami.md +++ b/deploy/helm/cassandra/docs/upgrade-from-bitnami.md @@ -19,16 +19,13 @@ Phase 4 tested an in-place over-the-top upgrade on k3d (old Bitnami stack with a known dataset, then upgrade to the new chart/image). Three concrete obstacles surfaced, each with evidence: -1. StatefulSet immutability. A direct `helm upgrade` from the Bitnami-subchart - release to the in-house chart fails: - `StatefulSet.apps "cassandra" is invalid: spec: Forbidden: updates to - statefulset spec for fields other than 'replicas', 'ordinals', 'template', - 'updateStrategy', 'persistentVolumeClaimRetentionPolicy' and - 'minReadySeconds' are forbidden`. The new chart intentionally removes the - existing `app.kubernetes.io/name` and `app.kubernetes.io/instance` labels - from `volumeClaimTemplates.metadata.labels`. Kubernetes treats the entire - volume claim template as immutable, so the StatefulSet must be recreated - instead of updated in place. +1. StatefulSet immutability. Early versions of the in-house chart removed the + `apiVersion`, `kind`, and the existing `app.kubernetes.io/name` and + `app.kubernetes.io/instance` labels from + `volumeClaimTemplates.metadata`. Kubernetes treats the entire volume claim + template as immutable, so the API server rejected a normal Helm upgrade. + The chart now preserves the exact identity emitted by published chart + 0.15.5, and CI guards that contract. Do not remove or rename those fields. 2. Data-layout nesting. Bitnami stored data nested under the mount: `/data/{data,commitlog,hints,saved_caches}` with @@ -49,7 +46,7 @@ the old and new charts, so PVC adoption is mechanically possible. The UID difference (old 1001, new 999) is handled by the pod `fsGroup: 999`; in testing the new image read the old files without a permission error. -## StatefulSet immutability and the recreate step +## StatefulSet immutability and the data-safe upgrade This is the mechanical heart of an in-place migration, so it is worth spelling out. @@ -61,59 +58,38 @@ fields you may change on an existing StatefulSet are `replicas`, `ordinals`, `serviceName`, `podManagementPolicy`, and `volumeClaimTemplates`. The old and new StatefulSets share the name, selector, service name, -`podManagementPolicy`, and the `volumeClaimTemplates.spec` fields. The only -immutable-field difference is in `volumeClaimTemplates.metadata.labels`. The -new chart intentionally removes the existing `app.kubernetes.io/name` and -`app.kubernetes.io/instance` labels. - -So `helm upgrade` applies the new chart onto the existing `cassandra` -StatefulSet, and the API server rejects it with the Forbidden error above. You -cannot reshape one StatefulSet into a structurally different one in place. This -is a Kubernetes constraint, not a chart defect. - -The recreate step works because the data lives in a PersistentVolumeClaim -(`data-cassandra-0`) whose lifecycle is independent of the StatefulSet -controller object. Deleting the StatefulSet does not delete its PVCs; -StatefulSet-managed PVCs are retained by default (this is what -`persistentVolumeClaimRetentionPolicy` governs, and its default is Retain). The -runbook: - -1. `kubectl delete statefulset cassandra --cascade=orphan` removes only the - StatefulSet controller object. `--cascade=orphan` also leaves the running - pods; the PVC is kept regardless of cascade mode. -2. Delete the old pod so the new controller starts a fresh one on the official - image. The PVC stays. -3. `helm upgrade` creates a brand-new StatefulSet named `cassandra`. A create is - not an update, so there is no immutability check. -4. Because the new StatefulSet has the same name and the same - volumeClaimTemplate name (`data`), it re-adopts the existing - `data-cassandra-0` PVC by name instead of provisioning a new empty one. - StatefulSets bind to a matching existing PVC and never recreate one that is - already present. - -Net effect: the controller object is swapped, the data volume is untouched and -re-adopted, and the new pod comes up on the old data. In the Phase 4 test the -PVC stayed `Bound` throughout and the new UID-999 pod mounted and read it. +`podManagementPolicy`, and the `volumeClaimTemplates` identity and spec fields. +The in-house chart must keep that immutable subset byte-for-byte equivalent to +the Kubernetes-normalized 0.15.5 object. The regression test at +`helm/scripts/test-bitnami-upgrade-identity.sh` enforces the fields that caused +the original rejection; live upgrade validation must additionally confirm that +the StatefulSet UID and PVC UID remain unchanged. + +With that immutable identity preserved, a normal `helm upgrade` updates the +existing StatefulSet in place. Set `cassandra.persistence.subPath: data` for +this one-time transition so the new UID-999 container sees the Bitnami layout, +and retain the compatibility keys in `cassandra.config`. Do not delete or +recreate the StatefulSet or PVC as part of the normal path. + +The published-0.15.5-to-source validation kept the StatefulSet, PVC, and PV +UIDs unchanged, retained a pre-upgrade sentinel row, and reached Ready on the +official image. The pod was replaced, as expected for an image and pod-template +change; the controller and storage objects were not. Two caveats: -- This is a one-time migration hop (Bitnami-shaped StatefulSet to - in-house-shaped StatefulSet). Ordinary upgrades within the in-house chart - later do not hit this, because the StatefulSet spec shape stays stable, unless - a future change edits a `volumeClaimTemplate` field (which would trip the same - rule). -- The orphan-delete is a manual, operator-error-prone step (a wrong flag or - target can delete more than intended). That risk is one of the reasons Option - C (backup and restore) is the safer path for production. +- This is a one-time data-layout compatibility setting. Keep `subPath: data` + for that release after the transition; changing it later changes where the + node looks for its files. +- Any future edit to a `volumeClaimTemplate` field will trip the same immutable + StatefulSet rule and must be rejected by upgrade testing. ## Options ### Option A: in-place adopt (legacy layout + config compat) Reuse the existing PVC in place. Mechanics: -- Recreate the StatefulSet: `kubectl delete statefulset cassandra - --cascade=orphan` (keeps the pod and PVC), delete the old pod (the PVC - persists), then `helm upgrade` so the new StatefulSet adopts - `data-cassandra-0`. +- Use a normal `helm upgrade`; the chart preserves the published 0.15.5 + StatefulSet and volume-claim-template identities. - Align the layout: set `persistence.subPath: data` so the old nested `data/{data,commitlog,...}` surfaces at the official defaults `/var/lib/cassandra/*` with no cassandra.yaml directory edits. This is @@ -124,15 +100,10 @@ Reuse the existing PVC in place. Mechanics: is not yet enumerated. -The orphan-delete-and-recreate runbook is scripted with safety checks at -`upgrade/migrate-from-bitnami.sh` (dry-run by default). It is provisional until -the config-compat set and strategy below are settled. - Pros: no downtime beyond the pod restart, no data copy, keeps the existing PVC. -Cons: relies on the orphan-delete runbook (operator error prone), and on -enumerating every cassandra.yaml setting the old fleet used. Config drift risk: -if an old setting is missed, the node fails to start on real data. Leaves the -data physically in the Bitnami nesting. +Cons: relies on enumerating every cassandra.yaml setting the old fleet used. +Config drift risk: if an old setting is missed, the node fails to start on real +data. Leaves the data physically in the Bitnami nesting. ### Option B: data relocation on first upgrade @@ -162,8 +133,8 @@ cutover; more operator steps; larger data means longer restore. Given we are pre-1.0.0 and want a clean result: - Ship Option A as the convenience path for environments that want in-place adoption, but only after the full cassandra.yaml config-compat set is - enumerated and encoded, and with the orphan-delete-and-recreate documented as - a supported runbook. + enumerated and encoded, with the published-to-source UID and data-retention + regression kept as a release gate. - Document Option C (backup/restore) as the recommended, safest path, especially for production, and as the fallback when in-place adoption is not acceptable. @@ -190,7 +161,7 @@ Open questions for Brad: ## Phase 4 evidence - Fresh install (single and multi node) on the new stack: validated. -- In-place upgrade: PVC adoption mechanically works; `subPath: data` surfaces - the old data and the UID-999 image reads the UID-1001 files; blocked by the - `uuid_sstable_identifiers_enabled` config mismatch and gated behind the - StatefulSet recreate runbook. +- In-place upgrade from published 0.15.5: validated with unchanged StatefulSet, + PVC, and PV UIDs and a retained sentinel row. `subPath: data` surfaces the old + data and the chart's compatibility config lets the UID-999 image read the + UID-1001 files. diff --git a/deploy/helm/cassandra/helm/scripts/test-bitnami-upgrade-identity.sh b/deploy/helm/cassandra/helm/scripts/test-bitnami-upgrade-identity.sh new file mode 100644 index 000000000..ccb13f07e --- /dev/null +++ b/deploy/helm/cassandra/helm/scripts/test-bitnami-upgrade-identity.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Guard the immutable StatefulSet identity used by the published 0.15.5 chart. +# Kubernetes rejects a Helm upgrade when any volumeClaimTemplate metadata is +# removed, even though the PVC name and storage request remain unchanged. + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +chart_dir="$(cd "${script_dir}/.." && pwd)" +render="$(mktemp)" +trap 'rm -f "${render}"' EXIT + +release="cassandra" +helm template "${release}" "${chart_dir}" \ + --namespace plane-a-cassandra-system \ + --show-only templates/statefulset.yaml >"${render}" + +if ! grep -Fqx -- ' serviceAccountName: default' "${render}"; then + echo "upgraded StatefulSet must explicitly leave the removed Bitnami ServiceAccount" >&2 + exit 1 +fi + +vct="$({ sed -n '/^ volumeClaimTemplates:/,$p' "${render}"; } )" + +assert_contains() { + local expected="$1" + if ! grep -Fqx -- "${expected}" <<<"${vct}"; then + echo "missing published-0.15.5 volumeClaimTemplate identity: ${expected}" >&2 + sed -n '/^ volumeClaimTemplates:/,$p' "${render}" >&2 + exit 1 + fi +} + +assert_contains ' - apiVersion: v1' +assert_contains ' kind: PersistentVolumeClaim' +assert_contains ' app.kubernetes.io/instance: cassandra' +assert_contains ' app.kubernetes.io/name: cassandra' +assert_contains ' name: data' + +echo "Cassandra StatefulSet preserves the published 0.15.5 immutable PVC-template identity." diff --git a/deploy/helm/cassandra/helm/templates/hook-post-02-migrations.yaml b/deploy/helm/cassandra/helm/templates/hook-post-02-migrations.yaml index 0946d763a..f40ccdd88 100644 --- a/deploy/helm/cassandra/helm/templates/hook-post-02-migrations.yaml +++ b/deploy/helm/cassandra/helm/templates/hook-post-02-migrations.yaml @@ -55,6 +55,8 @@ spec: value: {{ .Values.cassandra.serviceRolePassword | quote }} - name: REPLICA_COUNT value: {{ .Values.cassandra.replicaCount | quote }} + - name: CONTROL_PLANE_ID + value: {{ .Values.cassandra.migrations.controlPlaneID | quote }} resources: {{- toYaml .Values.cassandra.hooks.migrations.resources | nindent 12 }} {{- end }} diff --git a/deploy/helm/cassandra/helm/templates/statefulset.yaml b/deploy/helm/cassandra/helm/templates/statefulset.yaml index f451842d7..00946d8e8 100644 --- a/deploy/helm/cassandra/helm/templates/statefulset.yaml +++ b/deploy/helm/cassandra/helm/templates/statefulset.yaml @@ -32,6 +32,12 @@ spec: # Roll pods when the auth config changes so the initContainer re-patches. checksum/cassandra-conf: {{ include (print $.Template.BasePath "/configmap-cassandra-conf.yaml") . | sha256sum }} spec: + # The published Bitnami-backed chart used a release-owned ServiceAccount + # named cassandra. The in-house chart does not need Kubernetes API + # permissions and removes that object. Set the replacement explicitly so + # an upgrade does not retain the deleted ServiceAccount through the + # three-way patch and leave the replacement pod forbidden at admission. + serviceAccountName: default {{- include "cassandra.imagePullSecrets" . | nindent 6 }} securityContext: fsGroup: {{ .Values.cassandra.podSecurityContext.fsGroup }} @@ -204,8 +210,15 @@ spec: name: cassandra-init-cql defaultMode: 0500 volumeClaimTemplates: - - metadata: + # Preserve the object shape emitted by the Bitnami subchart used through + # helm-nvcf-cassandra 0.15.5. These fields are persisted inside the + # StatefulSet's immutable volumeClaimTemplates, so removing them makes a + # normal Helm upgrade fail before the pod can roll to the in-house chart. + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: name: data + labels: {{- include "cassandra.selectorLabels" . | nindent 10 }} spec: accessModes: ["ReadWriteOnce"] resources: diff --git a/deploy/helm/cassandra/helm/values.yaml b/deploy/helm/cassandra/helm/values.yaml index caa0d8c1b..d46a376d8 100644 --- a/deploy/helm/cassandra/helm/values.yaml +++ b/deploy/helm/cassandra/helm/values.yaml @@ -191,6 +191,9 @@ cassandra: memory: 256Mi migrations: + # Stable identity used by the migration image to derive plane-scoped ESS + # authorization endpoints. Empty preserves legacy single-plane URLs. + controlPlaneID: "" image: # registry: must be supplied in additional values # repository: must be supplied in additional values diff --git a/deploy/helm/cassandra/upgrade/README.md b/deploy/helm/cassandra/upgrade/README.md index c49dae05e..a162c0457 100644 --- a/deploy/helm/cassandra/upgrade/README.md +++ b/deploy/helm/cassandra/upgrade/README.md @@ -5,11 +5,20 @@ in-house StatefulSet chart. Read `../docs/upgrade-from-bitnami.md` first for the full context, the three migration options, and the open questions still under review. -## migrate-from-bitnami.sh +## Normal upgrade -Automates Option A (in-place adopt) from the migration doc: it recreates the -StatefulSet and adopts the existing data PVC, with safety checks. It is dry-run -by default and refuses managed/cloud contexts. +Current charts preserve the immutable StatefulSet/PVC-template identity from +published 0.15.5. Use a normal Helm upgrade with +`cassandra.persistence.subPath: data` and the documented config-compat values. +Verify the StatefulSet, PVC, and PV UIDs and application data before and after. + +## Legacy migrate-from-bitnami.sh fallback + +This provisional script predates the immutable-identity fix. It recreates the +StatefulSet and adopts the existing data PVC, with safety checks. It is not the +normal upgrade path; retain it only as a recovery tool for an early source +chart that omitted the 0.15.5 PVC-template identity. It is dry-run by default +and refuses managed/cloud contexts. Status: provisional. It does not resolve the open items from the migration doc (the full cassandra.yaml config-compat set, and whether in-place adopt or diff --git a/deploy/helm/cloud-functions/Makefile b/deploy/helm/cloud-functions/Makefile index 8a5096e03..0f33d5334 100644 --- a/deploy/helm/cloud-functions/Makefile +++ b/deploy/helm/cloud-functions/Makefile @@ -84,6 +84,7 @@ validate: template test: @echo "Running chart tests..." @./tests/sidecar_release_artifacts_test.sh + @sh ./nvcf-api/scripts/test-account-bootstrap-render.sh # Publish Chart # NOTE: this is manual until the CI pipeline is updated to push the chart to the NVCR OCI registry diff --git a/deploy/helm/cloud-functions/nvcf-api/scripts/account-bootstrap.sh b/deploy/helm/cloud-functions/nvcf-api/scripts/account-bootstrap.sh index dd35dc8db..8312ad7fa 100755 --- a/deploy/helm/cloud-functions/nvcf-api/scripts/account-bootstrap.sh +++ b/deploy/helm/cloud-functions/nvcf-api/scripts/account-bootstrap.sh @@ -37,7 +37,7 @@ readonly API_READINESS_PERIOD_SECONDS="${API_READINESS_PERIOD_SECONDS:-10}" readonly API_READINESS_FAILURE_THRESHOLD="${API_READINESS_FAILURE_THRESHOLD:-60}" # failureThreshold (period * threshold = total timeout) # Service endpoints -readonly OPENBAO_SERVICE_ADDR="openbao-server.vault-system.svc.cluster.local:8200" +readonly OPENBAO_SERVICE_ADDR="{{ required "api.accountBootstrap.openbaoServiceAddress is required" .Values.api.accountBootstrap.openbaoServiceAddress }}" # Vault JWT Auth configuration # Role used for the /v1/auth/jwt/login endpoint (overridable via env var) readonly OPENBAO_JWT_AUTH_ROLE="${OPENBAO_JWT_AUTH_ROLE:-nvcf-api-account-bootstrap}" diff --git a/deploy/helm/cloud-functions/nvcf-api/scripts/test-account-bootstrap-render.sh b/deploy/helm/cloud-functions/nvcf-api/scripts/test-account-bootstrap-render.sh new file mode 100755 index 000000000..84f7ed867 --- /dev/null +++ b/deploy/helm/cloud-functions/nvcf-api/scripts/test-account-bootstrap-render.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eu + +chart_dir="$(cd "$(dirname "$0")/.." && pwd)" +render="$(mktemp)" +trap 'rm -f "$render"' EXIT + +openbao_address="plane-a-openbao-server.plane-a-vault-system.svc.cluster.local:8200" + +helm template plane-a-api "$chart_dir" \ + --set api.image.registry=example.invalid \ + --set api.image.repository=nvcf-api \ + --set api.accountBootstrap.image.registry=example.invalid \ + --set api.accountBootstrap.image.repository=bootstrap \ + --set-string "api.accountBootstrap.openbaoServiceAddress=${openbao_address}" \ + >"$render" + +if ! grep -Fq "readonly OPENBAO_SERVICE_ADDR=\"${openbao_address}\"" "$render"; then + echo "FAIL: account bootstrap script did not use the configured OpenBao service address" >&2 + exit 1 +fi + +if ! grep -Fq 'audience: http://openbao-server.vault-system.svc.cluster.local:8200' "$render"; then + echo "FAIL: account bootstrap changed the shared OpenBao token audience" >&2 + exit 1 +fi + +if grep -Fq 'readonly OPENBAO_SERVICE_ADDR="openbao-server.vault-system.svc.cluster.local:8200"' "$render"; then + echo "FAIL: account bootstrap script retained the legacy OpenBao service address" >&2 + exit 1 +fi + +echo "NVCF API account-bootstrap render checks passed." diff --git a/deploy/helm/cloud-functions/nvcf-api/templates/account-bootstrap-hook-job.yaml b/deploy/helm/cloud-functions/nvcf-api/templates/account-bootstrap-hook-job.yaml index 210bd8061..3483c84f2 100644 --- a/deploy/helm/cloud-functions/nvcf-api/templates/account-bootstrap-hook-job.yaml +++ b/deploy/helm/cloud-functions/nvcf-api/templates/account-bootstrap-hook-job.yaml @@ -51,6 +51,8 @@ spec: - serviceAccountToken: path: token expirationSeconds: 3600 + # Audience is an opaque shared trust-domain value. OpenBao roles + # enforce control-plane isolation with the bound SA namespace. audience: http://openbao-server.vault-system.svc.cluster.local:8200 - name: account-bootstrap-script configMap: diff --git a/deploy/helm/cloud-functions/nvcf-api/values.yaml b/deploy/helm/cloud-functions/nvcf-api/values.yaml index 97c8ab5c2..789865420 100644 --- a/deploy/helm/cloud-functions/nvcf-api/values.yaml +++ b/deploy/helm/cloud-functions/nvcf-api/values.yaml @@ -43,6 +43,9 @@ api: accountBootstrap: # Set false to skip the account-bootstrap hook (runs on post-install and post-upgrade; e.g. when accounts are provisioned via the API). enabled: true + # In-cluster OpenBao endpoint used by the bootstrap script. The projected + # token audience remains the shared trust-domain value used by all roles. + openbaoServiceAddress: openbao-server.vault-system.svc.cluster.local:8200 image: registry: "" repository: "" diff --git a/deploy/helm/gateway-routes/chart/templates/_helpers.tpl b/deploy/helm/gateway-routes/chart/templates/_helpers.tpl index 1e9f357d5..dac43a28f 100644 --- a/deploy/helm/gateway-routes/chart/templates/_helpers.tpl +++ b/deploy/helm/gateway-routes/chart/templates/_helpers.tpl @@ -53,6 +53,14 @@ app.kubernetes.io/name: {{ include "nvcf-gateway.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} +{{/* +Namespace containing per-control-plane Route and route-policy resources. +Defaulting to the shared Gateway namespace preserves the legacy layout. +*/}} +{{- define "nvcf-gateway.routeNamespace" -}} +{{- .Values.nvcfGatewayRoutes.routeNamespace | default .Values.nvcfGatewayRoutes.gateways.shared.namespace -}} +{{- end }} + {{- define "nvcf-gateway.llmWorkerBackendNamespace" -}} {{- required "nvcfGatewayRoutes.routes.llmWorker.backend.namespace is required when llmWorker.enabled is true" .Values.nvcfGatewayRoutes.routes.llmWorker.backend.namespace -}} {{- end }} diff --git a/deploy/helm/gateway-routes/chart/templates/backendtrafficpolicy-llm-worker-grpc.yaml b/deploy/helm/gateway-routes/chart/templates/backendtrafficpolicy-llm-worker-grpc.yaml index 36bb01524..3133b55ef 100644 --- a/deploy/helm/gateway-routes/chart/templates/backendtrafficpolicy-llm-worker-grpc.yaml +++ b/deploy/helm/gateway-routes/chart/templates/backendtrafficpolicy-llm-worker-grpc.yaml @@ -7,7 +7,7 @@ apiVersion: gateway.envoyproxy.io/v1alpha1 kind: BackendTrafficPolicy metadata: name: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.name }}-grpc-streams - namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: llm-worker-grpc-stream-policy diff --git a/deploy/helm/gateway-routes/chart/templates/certificate-llm-worker-grpc.yaml b/deploy/helm/gateway-routes/chart/templates/certificate-llm-worker-grpc.yaml index 8fb20f273..77aa474bb 100644 --- a/deploy/helm/gateway-routes/chart/templates/certificate-llm-worker-grpc.yaml +++ b/deploy/helm/gateway-routes/chart/templates/certificate-llm-worker-grpc.yaml @@ -9,6 +9,8 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: {{ dig "secretName" "" $grpcTls | quote }} + # Gateway listener Secrets are namespace-local to the Gateway. Routes may + # live in a separate namespace, but the generated TLS Secret may not. namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.namespace }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} diff --git a/deploy/helm/gateway-routes/chart/templates/grpcroute-nvcf-api.yaml b/deploy/helm/gateway-routes/chart/templates/grpcroute-nvcf-api.yaml index ab17b91e9..afa4d0f0c 100644 --- a/deploy/helm/gateway-routes/chart/templates/grpcroute-nvcf-api.yaml +++ b/deploy/helm/gateway-routes/chart/templates/grpcroute-nvcf-api.yaml @@ -19,7 +19,7 @@ apiVersion: gateway.networking.k8s.io/v1 kind: GRPCRoute metadata: name: {{ .Values.nvcfGatewayRoutes.routes.nvcfApi.grpc.name }} - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: nvcf-api-grpc-route diff --git a/deploy/helm/gateway-routes/chart/templates/grpcroute-nvct-api.yaml b/deploy/helm/gateway-routes/chart/templates/grpcroute-nvct-api.yaml index e91a6e232..b4ef4d180 100644 --- a/deploy/helm/gateway-routes/chart/templates/grpcroute-nvct-api.yaml +++ b/deploy/helm/gateway-routes/chart/templates/grpcroute-nvct-api.yaml @@ -19,7 +19,7 @@ apiVersion: gateway.networking.k8s.io/v1 kind: GRPCRoute metadata: name: {{ .Values.nvcfGatewayRoutes.routes.nvctApi.grpc.name }} - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: nvct-api-grpc-route diff --git a/deploy/helm/gateway-routes/chart/templates/httproute-api-keys.yaml b/deploy/helm/gateway-routes/chart/templates/httproute-api-keys.yaml index b75eab844..d6ebd7ae2 100644 --- a/deploy/helm/gateway-routes/chart/templates/httproute-api-keys.yaml +++ b/deploy/helm/gateway-routes/chart/templates/httproute-api-keys.yaml @@ -19,7 +19,7 @@ apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: {{ .Values.nvcfGatewayRoutes.routes.apiKeys.name }} - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: api-keys-route @@ -49,4 +49,3 @@ spec: namespace: {{ .Values.nvcfGatewayRoutes.routes.apiKeys.backend.namespace }} port: {{ .Values.nvcfGatewayRoutes.routes.apiKeys.backend.port }} {{- end }} - diff --git a/deploy/helm/gateway-routes/chart/templates/httproute-ess.yaml b/deploy/helm/gateway-routes/chart/templates/httproute-ess.yaml index ca45435e7..47899878d 100644 --- a/deploy/helm/gateway-routes/chart/templates/httproute-ess.yaml +++ b/deploy/helm/gateway-routes/chart/templates/httproute-ess.yaml @@ -19,7 +19,7 @@ apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: {{ .Values.nvcfGatewayRoutes.routes.ess.name }} - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: ess-route diff --git a/deploy/helm/gateway-routes/chart/templates/httproute-event-ledger.yaml b/deploy/helm/gateway-routes/chart/templates/httproute-event-ledger.yaml index 4bebb1195..5b2168c0b 100644 --- a/deploy/helm/gateway-routes/chart/templates/httproute-event-ledger.yaml +++ b/deploy/helm/gateway-routes/chart/templates/httproute-event-ledger.yaml @@ -19,7 +19,7 @@ apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: {{ .Values.nvcfGatewayRoutes.routes.eventLedger.name }} - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: event-ledger-route diff --git a/deploy/helm/gateway-routes/chart/templates/httproute-invocation.yaml b/deploy/helm/gateway-routes/chart/templates/httproute-invocation.yaml index a7fbe9179..58a745025 100644 --- a/deploy/helm/gateway-routes/chart/templates/httproute-invocation.yaml +++ b/deploy/helm/gateway-routes/chart/templates/httproute-invocation.yaml @@ -19,7 +19,7 @@ apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: {{ .Values.nvcfGatewayRoutes.routes.invocation.name }} - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: invocation-service-route @@ -49,4 +49,3 @@ spec: namespace: {{ .Values.nvcfGatewayRoutes.routes.invocation.backend.namespace }} port: {{ .Values.nvcfGatewayRoutes.routes.invocation.backend.port }} {{- end }} - diff --git a/deploy/helm/gateway-routes/chart/templates/httproute-llm-api-gateway.yaml b/deploy/helm/gateway-routes/chart/templates/httproute-llm-api-gateway.yaml index 94c4b6a87..d9d940f26 100644 --- a/deploy/helm/gateway-routes/chart/templates/httproute-llm-api-gateway.yaml +++ b/deploy/helm/gateway-routes/chart/templates/httproute-llm-api-gateway.yaml @@ -19,7 +19,7 @@ apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: {{ .Values.nvcfGatewayRoutes.routes.llmApiGateway.name }} - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: llm-api-gateway-route diff --git a/deploy/helm/gateway-routes/chart/templates/httproute-llm-invocation.yaml b/deploy/helm/gateway-routes/chart/templates/httproute-llm-invocation.yaml index da5128141..12a490369 100644 --- a/deploy/helm/gateway-routes/chart/templates/httproute-llm-invocation.yaml +++ b/deploy/helm/gateway-routes/chart/templates/httproute-llm-invocation.yaml @@ -19,7 +19,7 @@ apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: {{ .Values.nvcfGatewayRoutes.routes.llmInvocation.name }} - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: llm-invocation-route diff --git a/deploy/helm/gateway-routes/chart/templates/httproute-nvcf-api.yaml b/deploy/helm/gateway-routes/chart/templates/httproute-nvcf-api.yaml index 33769b2b4..22a91ab15 100644 --- a/deploy/helm/gateway-routes/chart/templates/httproute-nvcf-api.yaml +++ b/deploy/helm/gateway-routes/chart/templates/httproute-nvcf-api.yaml @@ -19,7 +19,7 @@ apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: {{ .Values.nvcfGatewayRoutes.routes.nvcfApi.name }} - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: nvcf-api-route @@ -49,4 +49,3 @@ spec: namespace: {{ .Values.nvcfGatewayRoutes.routes.nvcfApi.backend.namespace }} port: {{ .Values.nvcfGatewayRoutes.routes.nvcfApi.backend.port }} {{- end }} - diff --git a/deploy/helm/gateway-routes/chart/templates/httproute-nvcf-ui.yaml b/deploy/helm/gateway-routes/chart/templates/httproute-nvcf-ui.yaml index ccbd54c4e..45a055316 100644 --- a/deploy/helm/gateway-routes/chart/templates/httproute-nvcf-ui.yaml +++ b/deploy/helm/gateway-routes/chart/templates/httproute-nvcf-ui.yaml @@ -19,7 +19,7 @@ apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: {{ .Values.nvcfGatewayRoutes.routes.nvcfUi.name }} - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: nvcf-ui-route diff --git a/deploy/helm/gateway-routes/chart/templates/httproute-nvct-api.yaml b/deploy/helm/gateway-routes/chart/templates/httproute-nvct-api.yaml index df96c4c1c..9b42bc6bb 100644 --- a/deploy/helm/gateway-routes/chart/templates/httproute-nvct-api.yaml +++ b/deploy/helm/gateway-routes/chart/templates/httproute-nvct-api.yaml @@ -19,7 +19,7 @@ apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: {{ .Values.nvcfGatewayRoutes.routes.nvctApi.name }} - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: nvcf-api-route @@ -49,4 +49,3 @@ spec: namespace: {{ .Values.nvcfGatewayRoutes.routes.nvctApi.backend.namespace }} port: {{ .Values.nvcfGatewayRoutes.routes.nvctApi.backend.port }} {{- end }} - diff --git a/deploy/helm/gateway-routes/chart/templates/httproute-reval.yaml b/deploy/helm/gateway-routes/chart/templates/httproute-reval.yaml index 298f9da74..007d65f90 100644 --- a/deploy/helm/gateway-routes/chart/templates/httproute-reval.yaml +++ b/deploy/helm/gateway-routes/chart/templates/httproute-reval.yaml @@ -19,7 +19,7 @@ apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: {{ .Values.nvcfGatewayRoutes.routes.reval.name }} - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: reval-route diff --git a/deploy/helm/gateway-routes/chart/templates/httproute-sis.yaml b/deploy/helm/gateway-routes/chart/templates/httproute-sis.yaml index d70ed1cd9..38d167c94 100644 --- a/deploy/helm/gateway-routes/chart/templates/httproute-sis.yaml +++ b/deploy/helm/gateway-routes/chart/templates/httproute-sis.yaml @@ -19,7 +19,7 @@ apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: {{ .Values.nvcfGatewayRoutes.routes.sis.name }} - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: sis-route diff --git a/deploy/helm/gateway-routes/chart/templates/httproute-vanity-gateway.yaml b/deploy/helm/gateway-routes/chart/templates/httproute-vanity-gateway.yaml index 97ce62c18..eed8a583a 100644 --- a/deploy/helm/gateway-routes/chart/templates/httproute-vanity-gateway.yaml +++ b/deploy/helm/gateway-routes/chart/templates/httproute-vanity-gateway.yaml @@ -21,7 +21,7 @@ apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: {{ .Values.nvcfGatewayRoutes.routes.vanityGateway.name }} - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: vanity-gateway-route diff --git a/deploy/helm/gateway-routes/chart/templates/podmonitor-grpc.yaml b/deploy/helm/gateway-routes/chart/templates/podmonitor-grpc.yaml index 82289dc92..81aeaa05c 100644 --- a/deploy/helm/gateway-routes/chart/templates/podmonitor-grpc.yaml +++ b/deploy/helm/gateway-routes/chart/templates/podmonitor-grpc.yaml @@ -18,7 +18,7 @@ apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: - name: envoy-gateway-proxy-grpc + name: {{ .Values.nvcfGatewayRoutes.podMonitors.grpcName }} namespace: {{ .Values.nvcfGatewayRoutes.gateways.grpc.namespace }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} diff --git a/deploy/helm/gateway-routes/chart/templates/podmonitor-shared.yaml b/deploy/helm/gateway-routes/chart/templates/podmonitor-shared.yaml index ff62f982c..df022f004 100644 --- a/deploy/helm/gateway-routes/chart/templates/podmonitor-shared.yaml +++ b/deploy/helm/gateway-routes/chart/templates/podmonitor-shared.yaml @@ -18,7 +18,7 @@ apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: - name: envoy-gateway-proxy-shared + name: {{ .Values.nvcfGatewayRoutes.podMonitors.sharedName }} namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} diff --git a/deploy/helm/gateway-routes/chart/templates/referencegrant-api-keys.yaml b/deploy/helm/gateway-routes/chart/templates/referencegrant-api-keys.yaml index eca911a70..edf675480 100644 --- a/deploy/helm/gateway-routes/chart/templates/referencegrant-api-keys.yaml +++ b/deploy/helm/gateway-routes/chart/templates/referencegrant-api-keys.yaml @@ -15,13 +15,13 @@ {{- if and .Values.nvcfGatewayRoutes.enabled .Values.nvcfGatewayRoutes.routes.apiKeys.enabled }} --- -# ReferenceGrant allows HTTPRoute in traefik-gateway namespace to reference -# Service in api-keys namespace (cross-namespace reference) +# ReferenceGrant allows HTTPRoute in the shared Gateway namespace to reference +# the Service in the configured API Keys backend namespace. apiVersion: gateway.networking.k8s.io/v1beta1 kind: ReferenceGrant metadata: name: allow-httproute-to-api-keys - namespace: api-keys # Grant is created in the target namespace + namespace: {{ .Values.nvcfGatewayRoutes.routes.apiKeys.backend.namespace }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} spec: @@ -29,11 +29,9 @@ spec: from: - group: gateway.networking.k8s.io kind: HTTPRoute - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} - # TO Services in api-keys namespace + namespace: {{ include "nvcf-gateway.routeNamespace" . }} + # TO Services in the configured API Keys backend namespace to: - group: "" kind: Service {{- end }} - - diff --git a/deploy/helm/gateway-routes/chart/templates/referencegrant-ess.yaml b/deploy/helm/gateway-routes/chart/templates/referencegrant-ess.yaml index fb1483ded..75905c1d8 100644 --- a/deploy/helm/gateway-routes/chart/templates/referencegrant-ess.yaml +++ b/deploy/helm/gateway-routes/chart/templates/referencegrant-ess.yaml @@ -29,7 +29,7 @@ spec: from: - group: gateway.networking.k8s.io kind: HTTPRoute - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} # TO Services in the ess namespace to: - group: "" diff --git a/deploy/helm/gateway-routes/chart/templates/referencegrant-llm-worker.yaml b/deploy/helm/gateway-routes/chart/templates/referencegrant-llm-worker.yaml index b7119dd16..4e8096912 100644 --- a/deploy/helm/gateway-routes/chart/templates/referencegrant-llm-worker.yaml +++ b/deploy/helm/gateway-routes/chart/templates/referencegrant-llm-worker.yaml @@ -14,10 +14,10 @@ spec: from: - group: gateway.networking.k8s.io kind: {{ if .Values.llmRequestRouter.grpcTls.enabled }}GRPCRoute{{ else }}TCPRoute{{ end }} - namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} - group: gateway.networking.k8s.io kind: UDPRoute - namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmQuic.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} to: - group: "" kind: Service diff --git a/deploy/helm/gateway-routes/chart/templates/referencegrant-nats.yaml b/deploy/helm/gateway-routes/chart/templates/referencegrant-nats.yaml index 0ed5cb19e..4acc88dc4 100644 --- a/deploy/helm/gateway-routes/chart/templates/referencegrant-nats.yaml +++ b/deploy/helm/gateway-routes/chart/templates/referencegrant-nats.yaml @@ -28,7 +28,7 @@ spec: from: - group: gateway.networking.k8s.io kind: TCPRoute - namespace: {{ .Values.nvcfGatewayRoutes.gateways.nats.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} to: - group: "" kind: Service diff --git a/deploy/helm/gateway-routes/chart/templates/referencegrant-nvcf-ui.yaml b/deploy/helm/gateway-routes/chart/templates/referencegrant-nvcf-ui.yaml index f47523be0..a51bf1013 100644 --- a/deploy/helm/gateway-routes/chart/templates/referencegrant-nvcf-ui.yaml +++ b/deploy/helm/gateway-routes/chart/templates/referencegrant-nvcf-ui.yaml @@ -30,7 +30,7 @@ spec: from: - group: gateway.networking.k8s.io kind: HTTPRoute - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} # TO Services in the nvcf-ui namespace to: - group: "" diff --git a/deploy/helm/gateway-routes/chart/templates/referencegrant-nvcf.yaml b/deploy/helm/gateway-routes/chart/templates/referencegrant-nvcf.yaml index 3b74cbaa4..e830a9d29 100644 --- a/deploy/helm/gateway-routes/chart/templates/referencegrant-nvcf.yaml +++ b/deploy/helm/gateway-routes/chart/templates/referencegrant-nvcf.yaml @@ -15,13 +15,13 @@ {{- if .Values.nvcfGatewayRoutes.enabled }} --- -# ReferenceGrant allows HTTPRoutes and GRPCRoutes in traefik-gateway namespace -# to reference Services in nvcf namespace (for api, invocation-service, grpc) +# ReferenceGrant allows HTTPRoutes and GRPCRoutes in the shared Gateway +# namespaces to reference Services in the configured NVCF backend namespace. apiVersion: gateway.networking.k8s.io/v1beta1 kind: ReferenceGrant metadata: name: allow-routes-to-nvcf - namespace: nvcf # Grant is created in the target namespace + namespace: {{ .Values.nvcfGatewayRoutes.routes.nvcfApi.backend.namespace }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} spec: @@ -29,16 +29,15 @@ spec: from: - group: gateway.networking.k8s.io kind: HTTPRoute - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} - group: gateway.networking.k8s.io kind: TCPRoute - namespace: {{ .Values.nvcfGatewayRoutes.gateways.grpc.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} - group: gateway.networking.k8s.io kind: GRPCRoute - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} - # TO Services in nvcf namespace + namespace: {{ include "nvcf-gateway.routeNamespace" . }} + # TO Services in the configured NVCF backend namespace to: - group: "" kind: Service {{- end }} - diff --git a/deploy/helm/gateway-routes/chart/templates/referencegrant-sis.yaml b/deploy/helm/gateway-routes/chart/templates/referencegrant-sis.yaml index 385630961..4e71962a2 100644 --- a/deploy/helm/gateway-routes/chart/templates/referencegrant-sis.yaml +++ b/deploy/helm/gateway-routes/chart/templates/referencegrant-sis.yaml @@ -29,7 +29,7 @@ spec: from: - group: gateway.networking.k8s.io kind: HTTPRoute - namespace: {{ .Values.nvcfGatewayRoutes.gateways.shared.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} # TO Services in the sis namespace to: - group: "" diff --git a/deploy/helm/gateway-routes/chart/templates/tcproute-grpc-proxy.yaml b/deploy/helm/gateway-routes/chart/templates/tcproute-grpc-proxy.yaml index 33fb906ae..b22506b2a 100644 --- a/deploy/helm/gateway-routes/chart/templates/tcproute-grpc-proxy.yaml +++ b/deploy/helm/gateway-routes/chart/templates/tcproute-grpc-proxy.yaml @@ -19,7 +19,7 @@ apiVersion: gateway.networking.k8s.io/v1alpha2 kind: TCPRoute metadata: name: {{ .Values.nvcfGatewayRoutes.routes.grpc.name }} - namespace: {{ .Values.nvcfGatewayRoutes.gateways.grpc.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: grpc-route diff --git a/deploy/helm/gateway-routes/chart/templates/tcproute-grpc-worker.yaml b/deploy/helm/gateway-routes/chart/templates/tcproute-grpc-worker.yaml index 1f174eef3..84a3fea9c 100644 --- a/deploy/helm/gateway-routes/chart/templates/tcproute-grpc-worker.yaml +++ b/deploy/helm/gateway-routes/chart/templates/tcproute-grpc-worker.yaml @@ -19,7 +19,7 @@ apiVersion: gateway.networking.k8s.io/v1alpha2 kind: TCPRoute metadata: name: {{ .Values.nvcfGatewayRoutes.routes.grpcWorker.name }} - namespace: {{ .Values.nvcfGatewayRoutes.gateways.grpc.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: grpc-worker-route diff --git a/deploy/helm/gateway-routes/chart/templates/tcproute-llm-worker.yaml b/deploy/helm/gateway-routes/chart/templates/tcproute-llm-worker.yaml index 5a7070b78..5997b7b1d 100644 --- a/deploy/helm/gateway-routes/chart/templates/tcproute-llm-worker.yaml +++ b/deploy/helm/gateway-routes/chart/templates/tcproute-llm-worker.yaml @@ -9,7 +9,7 @@ apiVersion: gateway.networking.k8s.io/{{ if $tlsEnabled }}v1{{ else }}v1alpha2{{ kind: {{ if $tlsEnabled }}GRPCRoute{{ else }}TCPRoute{{ end }} metadata: name: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.name }}-grpc - namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: llm-worker-grpc-route diff --git a/deploy/helm/gateway-routes/chart/templates/tcproute-nats.yaml b/deploy/helm/gateway-routes/chart/templates/tcproute-nats.yaml index 5ab1949d3..b28dfd6fa 100644 --- a/deploy/helm/gateway-routes/chart/templates/tcproute-nats.yaml +++ b/deploy/helm/gateway-routes/chart/templates/tcproute-nats.yaml @@ -19,7 +19,7 @@ apiVersion: gateway.networking.k8s.io/v1alpha2 kind: TCPRoute metadata: name: {{ .Values.nvcfGatewayRoutes.routes.nats.name }} - namespace: {{ .Values.nvcfGatewayRoutes.gateways.nats.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: nats-route diff --git a/deploy/helm/gateway-routes/chart/templates/udproute-llm-worker.yaml b/deploy/helm/gateway-routes/chart/templates/udproute-llm-worker.yaml index cba18b322..b50e8d44e 100644 --- a/deploy/helm/gateway-routes/chart/templates/udproute-llm-worker.yaml +++ b/deploy/helm/gateway-routes/chart/templates/udproute-llm-worker.yaml @@ -7,7 +7,7 @@ apiVersion: gateway.networking.k8s.io/v1alpha2 kind: UDPRoute metadata: name: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.name }}-quic - namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmQuic.namespace }} + namespace: {{ include "nvcf-gateway.routeNamespace" . }} labels: {{- include "nvcf-gateway.labels" . | nindent 4 }} app.kubernetes.io/component: llm-worker-quic-route diff --git a/deploy/helm/gateway-routes/chart/values.yaml b/deploy/helm/gateway-routes/chart/values.yaml index 2b1dcadeb..cdbd59fac 100644 --- a/deploy/helm/gateway-routes/chart/values.yaml +++ b/deploy/helm/gateway-routes/chart/values.yaml @@ -20,6 +20,10 @@ nvcfGatewayRoutes: # Enable/disable ingress deployment enabled: true + # Namespace for Route and route-policy objects. Empty preserves the legacy + # behavior of placing them in the shared Gateway namespace. + routeNamespace: "" + # Domain for hostname routing domain: "localhost" @@ -294,6 +298,8 @@ nvcfGatewayRoutes: # PodMonitors for Envoy Gateway proxy pods. podMonitors: enabled: false + sharedName: envoy-gateway-proxy-shared + grpcName: envoy-gateway-proxy-grpc # Worker-facing gRPC TLS is configured with the request router because the # certificate and advertised dial URI form one transport contract. The diff --git a/deploy/helm/gateway-routes/scripts/check-llm-worker-routes.sh b/deploy/helm/gateway-routes/scripts/check-llm-worker-routes.sh index cf1ddf00e..511b8ea78 100755 --- a/deploy/helm/gateway-routes/scripts/check-llm-worker-routes.sh +++ b/deploy/helm/gateway-routes/scripts/check-llm-worker-routes.sh @@ -19,9 +19,10 @@ trap 'rm -f "$rendered" "$disabled" "$plaintext" "$existing_secret" "$invalid_ba helm template nvcf-gateway-routes "$chart_dir" \ --namespace gateway \ + --set nvcfGatewayRoutes.routeNamespace=plane-a-ingress \ --set nvcfGatewayRoutes.routes.llmWorker.enabled=true \ --set nvcfGatewayRoutes.gateways.llmGrpc.name=llm-grpc-gateway \ - --set nvcfGatewayRoutes.gateways.llmGrpc.namespace=gateway \ + --set nvcfGatewayRoutes.gateways.llmGrpc.namespace=plane-a-gateway \ --set nvcfGatewayRoutes.gateways.llmQuic.name=llm-quic-gateway \ --set nvcfGatewayRoutes.gateways.llmQuic.namespace=gateway \ --set nvcfGatewayRoutes.routes.llmWorker.backend.namespace=router-system \ @@ -75,6 +76,17 @@ assert_contains "sectionName: llm-grpc" \ assert_contains "sectionName: llm-quic" \ "UDPRoute must attach to the configured LLM QUIC listener" +certificate_namespace="$(awk ' + $0 == "kind: Certificate" { in_certificate = 1; in_metadata = 0; next } + in_certificate && /^---$/ { in_certificate = 0; in_metadata = 0 } + in_certificate && $0 == "metadata:" { in_metadata = 1; next } + in_certificate && in_metadata && $1 == "namespace:" { print $2; exit } +' "$rendered")" +if [[ "$certificate_namespace" != "plane-a-gateway" ]]; then + echo "FAIL: the gRPC Certificate Secret must be created in the Gateway namespace" >&2 + exit 1 +fi + reference_grant_service_name="$(awk ' $0 == "kind: ReferenceGrant" { in_grant = 1; target_grant = 0; in_to = 0 } in_grant && !target_grant && $1 == "name:" && $2 == "allow-llm-worker-routes" { target_grant = 1 } diff --git a/deploy/helm/gateway-routes/scripts/test-render-routes.sh b/deploy/helm/gateway-routes/scripts/test-render-routes.sh index a13b54372..1b1436977 100755 --- a/deploy/helm/gateway-routes/scripts/test-render-routes.sh +++ b/deploy/helm/gateway-routes/scripts/test-render-routes.sh @@ -21,7 +21,8 @@ default_render="$(mktemp)" enabled_render="$(mktemp)" annotated_render="$(mktemp)" disabled_render="$(mktemp)" -trap 'rm -f "$default_render" "$enabled_render" "$annotated_render" "$disabled_render"' EXIT +isolated_render="$(mktemp)" +trap 'rm -f "$default_render" "$enabled_render" "$annotated_render" "$disabled_render" "$isolated_render"' EXIT if ! command -v yq >/dev/null 2>&1; then echo "yq is required for render tests" >&2 @@ -169,6 +170,36 @@ assert_resource_field "$default_render" ReferenceGrant allow-httproute-to-sis si assert_resource_field "$default_render" ReferenceGrant allow-httproute-to-sis sis '.spec.from[0].namespace' gateway assert_resource_field "$default_render" ReferenceGrant allow-httproute-to-sis sis '.spec.to[0].kind' Service +# ReferenceGrants must be installed alongside their configured backends. This +# is required when multiple control planes use distinct namespaces in one +# cluster; hard-coded default namespaces silently grant the wrong plane. +helm template plane-a-gateway-routes "$repo_root/chart" \ + --set nvcfGatewayRoutes.routeNamespace=plane-a-ingress \ + --api-versions monitoring.coreos.com/v1/PodMonitor \ + --set nvcfGatewayRoutes.routes.nvcfApi.backend.namespace=plane-a-nvcf \ + --set nvcfGatewayRoutes.routes.nvcfApi.grpc.backend.namespace=plane-a-nvcf \ + --set nvcfGatewayRoutes.routes.apiKeys.backend.namespace=plane-a-api-keys \ + --set nvcfGatewayRoutes.podMonitors.enabled=true \ + --set nvcfGatewayRoutes.podMonitors.sharedName=plane-a-envoy-gateway-proxy-shared \ + --set nvcfGatewayRoutes.podMonitors.grpcName=plane-a-envoy-gateway-proxy-grpc \ + > "$isolated_render" + +assert_resource_count "$isolated_render" ReferenceGrant allow-routes-to-nvcf plane-a-nvcf 1 +assert_resource_count "$isolated_render" ReferenceGrant allow-routes-to-nvcf nvcf 0 +assert_resource_count "$isolated_render" ReferenceGrant allow-httproute-to-api-keys plane-a-api-keys 1 +assert_resource_count "$isolated_render" ReferenceGrant allow-httproute-to-api-keys api-keys 0 +assert_resource_count "$isolated_render" PodMonitor plane-a-envoy-gateway-proxy-shared gateway 1 +assert_resource_count "$isolated_render" PodMonitor plane-a-envoy-gateway-proxy-grpc gateway 1 +assert_resource_count "$isolated_render" PodMonitor envoy-gateway-proxy-shared gateway 0 +assert_resource_count "$isolated_render" PodMonitor envoy-gateway-proxy-grpc gateway 0 +assert_resource_count "$isolated_render" HTTPRoute nvcf-api plane-a-ingress 1 +assert_resource_count "$isolated_render" HTTPRoute nvcf-api gateway 0 +assert_resource_field "$isolated_render" HTTPRoute nvcf-api plane-a-ingress '.spec.parentRefs[0].namespace' gateway +assert_resource_field "$isolated_render" ReferenceGrant allow-routes-to-nvcf plane-a-nvcf '.spec.from[0].namespace' plane-a-ingress +assert_resource_field "$isolated_render" ReferenceGrant allow-routes-to-nvcf plane-a-nvcf '.spec.from[1].namespace' plane-a-ingress +assert_resource_field "$isolated_render" ReferenceGrant allow-routes-to-nvcf plane-a-nvcf '.spec.from[2].namespace' plane-a-ingress +assert_resource_field "$isolated_render" ReferenceGrant allow-httproute-to-api-keys plane-a-api-keys '.spec.from[0].namespace' plane-a-ingress + # Routes disabled by default stay absent unless explicitly enabled. assert_resource_count "$default_render" HTTPRoute llm-invocation gateway 0 assert_resource_count "$default_render" GRPCRoute nvcf-api-grpc gateway 0 diff --git a/deploy/helm/icms/Makefile b/deploy/helm/icms/Makefile index f6ad78393..e645f5c12 100644 --- a/deploy/helm/icms/Makefile +++ b/deploy/helm/icms/Makefile @@ -34,7 +34,10 @@ OCI_REGISTRY_NAMESPACE ?= 0651155215864979/ncp-dev CHART_NAME := $(shell yq -r .name $(helm_dir)/Chart.yaml) CHART_VERSION := $(shell yq -r .version $(helm_dir)/Chart.yaml) -.PHONY: deploy delete status lint template +.PHONY: deploy delete status lint template test + +test: + @sh ./scripts/test-control-plane-isolation.sh install: ifndef values diff --git a/deploy/helm/icms/icms-api/templates/hook-lls-migrations.yaml b/deploy/helm/icms/icms-api/templates/hook-lls-migrations.yaml index ff1b75d78..8404b7df4 100644 --- a/deploy/helm/icms/icms-api/templates/hook-lls-migrations.yaml +++ b/deploy/helm/icms/icms-api/templates/hook-lls-migrations.yaml @@ -53,6 +53,10 @@ spec: value: {{ (.Values.sis.lls.hmacRotation.baoService | default (printf "openbao-server.%s.svc.cluster.local" .Values.sis.lls.namespace)) | quote }} - name: BAO_PORT value: {{ .Values.sis.lls.hmacRotation.baoPort | quote }} + - name: TURN_SERVICE_ACCOUNT_NAME + value: {{ required "sis.lls.turn.serviceAccountName is required when sis.lls.enabled is true" .Values.sis.lls.turn.serviceAccountName | quote }} + - name: TURN_SERVICE_ACCOUNT_NAMESPACE + value: {{ required "sis.lls.turn.serviceAccountNamespace is required when sis.lls.enabled is true" .Values.sis.lls.turn.serviceAccountNamespace | quote }} volumeMounts: - name: root-token mountPath: /secrets/root_token diff --git a/deploy/helm/icms/icms-api/values.yaml b/deploy/helm/icms/icms-api/values.yaml index bc6462ad2..2337802e3 100644 --- a/deploy/helm/icms/icms-api/values.yaml +++ b/deploy/helm/icms/icms-api/values.yaml @@ -29,6 +29,10 @@ sis: enabled: false # Namespace for LLS resources (migrations hook, rotation CronJob) namespace: "vault-system" + # External TURN workload identity bound into this plane's OpenBao role. + turn: + serviceAccountName: turn + serviceAccountNamespace: gdn-streaming podAnnotations: {} migrations: resources: diff --git a/deploy/helm/icms/scripts/test-control-plane-isolation.sh b/deploy/helm/icms/scripts/test-control-plane-isolation.sh new file mode 100755 index 000000000..909784bd2 --- /dev/null +++ b/deploy/helm/icms/scripts/test-control-plane-isolation.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eu + +chart_dir="$(cd "$(dirname "$0")/../icms-api" && pwd)" +render="$(mktemp)" +trap 'rm -f "$render"' EXIT + +helm template plane-a-sis "$chart_dir" \ + --namespace plane-a-sis \ + --set sis.image.registry=example.invalid \ + --set sis.image.repository=sis \ + --set sis.lls.enabled=true \ + --set sis.lls.namespace=plane-a-vault-system \ + --set sis.lls.turn.serviceAccountName=turn \ + --set sis.lls.turn.serviceAccountNamespace=plane-a-gdn-streaming \ + --set sis.lls.hmacRotation.image.registry=example.invalid \ + --set sis.lls.hmacRotation.image.repository=migrations \ + --set sis.lls.hmacRotation.image.tag=test \ + --set sis.lls.hmacRotation.baoService=plane-a-openbao-server.plane-a-vault-system.svc.cluster.local \ + --set sis.lls.hmacRotation.serviceAccountName=plane-a-openbao-server-initialize-cluster \ + --set sis.lls.hmacRotation.rootTokenSecretName=plane-a-openbao-server-root-token \ + >"$render" + +for expected in \ + 'namespace: plane-a-vault-system' \ + 'serviceAccountName: plane-a-openbao-server-initialize-cluster' \ + 'value: "plane-a-openbao-server.plane-a-vault-system.svc.cluster.local"' \ + 'name: TURN_SERVICE_ACCOUNT_NAMESPACE' \ + 'value: "plane-a-gdn-streaming"' \ + 'secretName: plane-a-openbao-server-root-token'; do + if ! grep -Fq "$expected" "$render"; then + echo "FAIL: SIS LLS render missing: $expected" >&2 + exit 1 + fi +done + +echo "SIS LLS control-plane isolation render checks passed." diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/hook-llm-migrations.yaml b/deploy/helm/llm-request-router/llm-request-router/templates/hook-llm-migrations.yaml index 522989f04..a92dc6048 100644 --- a/deploy/helm/llm-request-router/llm-request-router/templates/hook-llm-migrations.yaml +++ b/deploy/helm/llm-request-router/llm-request-router/templates/hook-llm-migrations.yaml @@ -64,6 +64,12 @@ spec: value: {{ ($pki.baoService | default (printf "openbao-server.%s.svc.cluster.local" $pki.namespace)) | quote }} - name: BAO_PORT value: {{ $pki.baoPort | quote }} + - name: OPENBAO_SERVER_INTERNAL_URL + value: {{ printf "http://%s:%s" ($pki.baoService | default (printf "openbao-server.%s.svc.cluster.local" $pki.namespace)) ($pki.baoPort | toString) | quote }} + - name: OPENBAO_JWT_AUDIENCE + value: {{ required "llmRequestRouter.pki.jwtAudience is required when llmRequestRouter.pki.enabled is true" $pki.jwtAudience | quote }} + - name: SIS_SERVICE_ACCOUNT_NAMESPACE + value: {{ required "llmRequestRouter.pki.sisServiceAccountNamespace is required when llmRequestRouter.pki.enabled is true" $pki.sisServiceAccountNamespace | quote }} volumeMounts: - name: root-token mountPath: /secrets/root_token diff --git a/deploy/helm/llm-request-router/llm-request-router/values.yaml b/deploy/helm/llm-request-router/llm-request-router/values.yaml index 6ca5ea4fb..8233892c6 100644 --- a/deploy/helm/llm-request-router/llm-request-router/values.yaml +++ b/deploy/helm/llm-request-router/llm-request-router/values.yaml @@ -219,6 +219,11 @@ llmRequestRouter: # `openbao-server..svc.cluster.local`. baoService: "" baoPort: "8200" + # Opaque projected-token audience. This intentionally remains independent + # of the plane-specific OpenBao network address. + jwtAudience: http://openbao-server.vault-system.svc.cluster.local:8200 + # Namespace bound into the SIS JWT role provisioned by the PKI addon. + sisServiceAccountNamespace: sis # Service account that has read access to the openbao root-token # secret (created by the k8s-openbao chart). serviceAccountName: openbao-server-initialize-cluster diff --git a/deploy/helm/llm-request-router/scripts/check-pki-render.sh b/deploy/helm/llm-request-router/scripts/check-pki-render.sh index 34964ac1f..59ad1633e 100644 --- a/deploy/helm/llm-request-router/scripts/check-pki-render.sh +++ b/deploy/helm/llm-request-router/scripts/check-pki-render.sh @@ -104,6 +104,11 @@ helm template llm-request-router ./llm-request-router \ --set llmRequestRouter.pki.image.registry=nvcr.io \ --set 'llmRequestRouter.pki.image.repository=/nvcf-openbao-migrations' \ --set llmRequestRouter.pki.image.tag=0.12.1 \ + --set llmRequestRouter.pki.namespace=plane-a-vault-system \ + --set llmRequestRouter.pki.baoService=plane-a-openbao-server.plane-a-vault-system.svc.cluster.local \ + --set llmRequestRouter.pki.serviceAccountName=plane-a-openbao-server-initialize-cluster \ + --set llmRequestRouter.pki.rootTokenSecretName=plane-a-openbao-server-root-token \ + --set llmRequestRouter.pki.sisServiceAccountNamespace=plane-a-sis \ > "${manifest}" cert_secret="$(yq -rN 'select(.kind == "Certificate" and .metadata.name == "stargate-quic-tls") | .spec.secretName' "${manifest}")" @@ -137,6 +142,13 @@ hook_image="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-mi hook_addons_llm="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-migrations") | .spec.template.spec.containers[0].env[] | select(.name == "ADDONS_LLM_ENABLED") | .value' "${manifest}")" hook_core_off="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-migrations") | .spec.template.spec.containers[0].env[] | select(.name == "CORE_MIGRATIONS_ENABLED") | .value' "${manifest}")" hook_allowed_domains="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-migrations") | .spec.template.spec.containers[0].env[] | select(.name == "NVCF_SERVICE_PKI_ALLOWED_DOMAINS") | .value' "${manifest}")" +hook_namespace="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-migrations") | .metadata.namespace' "${manifest}")" +hook_service_account="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-migrations") | .spec.template.spec.serviceAccountName' "${manifest}")" +hook_bao_service="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-migrations") | .spec.template.spec.containers[0].env[] | select(.name == "BAO_SERVICE") | .value' "${manifest}")" +hook_internal_url="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-migrations") | .spec.template.spec.containers[0].env[] | select(.name == "OPENBAO_SERVER_INTERNAL_URL") | .value' "${manifest}")" +hook_jwt_audience="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-migrations") | .spec.template.spec.containers[0].env[] | select(.name == "OPENBAO_JWT_AUDIENCE") | .value' "${manifest}")" +hook_sis_namespace="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-migrations") | .spec.template.spec.containers[0].env[] | select(.name == "SIS_SERVICE_ACCOUNT_NAMESPACE") | .value' "${manifest}")" +hook_root_token_secret="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-migrations") | .spec.template.spec.volumes[] | select(.name == "root-token") | .secret.secretName' "${manifest}")" [ "${hook_job_name}" = "addons-llm-migrations" ] [ "${hook_helm_hook}" = "pre-install,pre-upgrade" ] @@ -144,6 +156,13 @@ hook_allowed_domains="$(yq -rN 'select(.kind == "Job" and .metadata.name == "add [ "${hook_addons_llm}" = "true" ] [ "${hook_core_off}" = "false" ] [ "${hook_allowed_domains}" = "stargate.localhost,cluster.local" ] +[ "${hook_namespace}" = "plane-a-vault-system" ] +[ "${hook_service_account}" = "plane-a-openbao-server-initialize-cluster" ] +[ "${hook_bao_service}" = "plane-a-openbao-server.plane-a-vault-system.svc.cluster.local" ] +[ "${hook_internal_url}" = "http://plane-a-openbao-server.plane-a-vault-system.svc.cluster.local:8200" ] +[ "${hook_jwt_audience}" = "http://openbao-server.vault-system.svc.cluster.local:8200" ] +[ "${hook_sis_namespace}" = "plane-a-sis" ] +[ "${hook_root_token_secret}" = "plane-a-openbao-server-root-token" ] # Exact and wildcard SANs cover a static advertised hostname. exact_manifest="${tmp_dir}/exact.yaml" diff --git a/deploy/helm/nvca-operator/Makefile b/deploy/helm/nvca-operator/Makefile index 523411f48..b5c9bf651 100644 --- a/deploy/helm/nvca-operator/Makefile +++ b/deploy/helm/nvca-operator/Makefile @@ -44,7 +44,7 @@ OCI_REGISTRY_NAMESPACE ?= CHART_NAME := $(shell yq -r .name $(helm_dir)/Chart.yaml) CHART_VERSION := $(shell yq -r .version $(helm_dir)/Chart.yaml) -.PHONY: install uninstall status lint template validate clean package push-oci sync-chart check-synced-chart render-values-from-stack install-from-stack test-render-values test-vendor-chart-image-tag test-build-release-assets test-release-image-manifest test-release-artifact-permissions test-package-release-assets test-attach-release-assets test-release-sbom-wrapper test-self-managed-nvca-image-reference test-image-pull-secret-defaults test-pod-disruption-budget +.PHONY: install uninstall status lint template validate clean package push-oci sync-chart check-synced-chart render-values-from-stack install-from-stack test-render-values test-vendor-chart-image-tag test-build-release-assets test-release-image-manifest test-release-artifact-permissions test-package-release-assets test-attach-release-assets test-release-sbom-wrapper test-self-managed-nvca-image-reference test-image-pull-secret-defaults test-pod-disruption-budget test-control-plane-isolation install: ifndef values @@ -122,6 +122,9 @@ test-image-pull-secret-defaults: test-pod-disruption-budget: @bash ./tests/pod_disruption_budget_test.sh +test-control-plane-isolation: + @bash ./tests/control_plane_isolation_test.sh + uninstall: @echo "Deleting $(release) from namespace $(namespace)..." helm uninstall $(release) --namespace $(namespace) diff --git a/deploy/helm/nvca-operator/nvca-operator/README.md b/deploy/helm/nvca-operator/nvca-operator/README.md index 19406bc7a..f0e0378ef 100644 --- a/deploy/helm/nvca-operator/nvca-operator/README.md +++ b/deploy/helm/nvca-operator/nvca-operator/README.md @@ -29,6 +29,7 @@ used in Kubernetes Clusters to run NVCF Workloads. | `generateImagePullSecret` | Use the ngcConfig.serviceKey to generate an image pull secret for nvca and nvca-operator Pods | `true` | | `imagePullSecretName` | Name of the image pull secret to use for nvca and nvca-operator Pods. | `nvca-operator-image-pull` | | `imagePullSecrets` | List of pre-existing imagePullSecret objects in the nvca-operator namespace to use for nvca and nvca-operator Pods. Each object must have a 'name' field. Example: [{name: "foo-bar"}, {name: "baz"}] | `[]` | +| `controlPlane.id` | Optional lowercase DNS label identifying an isolated control plane. Empty preserves legacy names; a value such as `plane-a` requires release and namespace `plane-a-nvca-operator`. | `""` | | `serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | | `serviceAccount.annotations` | Additional custom annotations for the ServiceAccount | `{}` | | `serviceAccount.name` | The name of the ServiceAccount to use. | `""` | @@ -77,7 +78,7 @@ used in Kubernetes Clusters to run NVCF Workloads. | `agent.cacheMountOptionsEnabled` | Enable or disable CSI volume mount options for NVCA caches | `true` | | `agent.cacheMountOptions` | Comma-separated string of CSI volume mount options (e.g., "ro,noatime,nouuid") used when cacheMountOptionsEnabled is true | `ro,norecovery,nouuid` | | `agent.workerDegradationPeriod` | Duration for determining if a worker is degraded (e.g., "90m", "1h30m") | `""` | -| `agent.secretMirrorNamespace` | Default namespace to mirror custom secrets for nvcf workloads | `nvca-operator` | +| `agent.secretMirrorNamespace` | Namespace to source mirrored workload secrets from; empty derives this control plane's operator namespace. A custom namespace is an intentional shared/alternate source and must be access-controlled. | `""` | | `agent.secretMirrorLabelSelector` | Label selector on the secrets in the sourceNamespace | `""` | | `agent.customAnnotations` | Map of custom annotations to add to the agent pod | `{}` | | `agent.gpuProfiling.functionIds` | Comma/space/newline-separated NVCF function IDs (or "*" for all) whose pods NVCA labels for NVIDIA Nsight GPU profiling. Empty disables profiling. | `""` | diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/_helpers.tpl b/deploy/helm/nvca-operator/nvca-operator/templates/_helpers.tpl index 1e093ec48..6c4ff1b77 100644 --- a/deploy/helm/nvca-operator/nvca-operator/templates/_helpers.tpl +++ b/deploy/helm/nvca-operator/nvca-operator/templates/_helpers.tpl @@ -22,13 +22,64 @@ Expand the name of the chart. {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Validated control-plane identity. Empty is the legacy compatibility mode. +*/}} +{{- define "nvcaop.controlPlaneID" -}} +{{- $controlPlane := .Values.controlPlane | default dict -}} +{{- $id := $controlPlane.id | default "" -}} +{{- if eq $id "default" -}} +{{- fail "controlPlane.id \"default\" is reserved" -}} +{{- end -}} +{{- if and $id (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $id)) -}} +{{- fail (printf "controlPlane.id must be a lowercase RFC 1123 DNS label, got %q" $id) -}} +{{- end -}} +{{- if gt (len $id) 20 -}} +{{- fail "controlPlane.id must be at most 20 characters" -}} +{{- end -}} +{{- if $id -}} +{{- $expectedOperatorName := printf "%s-nvca-operator" $id -}} +{{- if ne .Release.Name $expectedOperatorName -}} +{{- fail (printf "controlPlane.id=%q requires Helm release name %q" $id $expectedOperatorName) -}} +{{- end -}} +{{- if ne .Release.Namespace $expectedOperatorName -}} +{{- fail (printf "controlPlane.id=%q requires Helm release namespace %q" $id $expectedOperatorName) -}} +{{- end -}} +{{- end -}} +{{- $id -}} +{{- end -}} + +{{/* Namespaces derived from the control-plane identity. */}} +{{- define "nvcaop.operatorNamespace" -}} +{{- $id := include "nvcaop.controlPlaneID" . -}} +{{- if $id -}}{{ printf "%s-nvca-operator" $id }}{{- else -}}nvca-operator{{- end -}} +{{- end -}} + +{{- define "nvcaop.systemNamespace" -}} +{{- $id := include "nvcaop.controlPlaneID" . -}} +{{- if $id -}}{{ printf "%s-nvca-system" $id }}{{- else -}}nvca-system{{- end -}} +{{- end -}} + +{{- define "nvcaop.requestsNamespace" -}} +{{- $id := include "nvcaop.controlPlaneID" . -}} +{{- if $id -}}{{ printf "%s-nvcf-backend" $id }}{{- else -}}nvcf-backend{{- end -}} +{{- end -}} + +{{/* Secret mirror source defaults to this control plane's operator namespace. */}} +{{- define "nvcaop.secretMirrorNamespace" -}} +{{- .Values.agent.secretMirrorNamespace | default (include "nvcaop.operatorNamespace" .) -}} +{{- end -}} + {{/* Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} {{- define "nvcaop.fullname" -}} -{{- if .Values.fullnameOverride -}} +{{- $id := include "nvcaop.controlPlaneID" . -}} +{{- if $id -}} +{{- printf "%s-nvca-operator" $id -}} +{{- else if .Values.fullnameOverride -}} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} {{- else -}} {{- $name := default .Chart.Name .Values.nameOverride -}} @@ -54,6 +105,10 @@ Common labels helm.sh/chart: {{ include "nvcaop.chart" . }} app.kubernetes.io/name: {{ include "nvcaop.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} +{{- $controlPlaneID := include "nvcaop.controlPlaneID" . }} +{{- if $controlPlaneID }} +nvcf.nvidia.com/control-plane-id: {{ $controlPlaneID | quote }} +{{- end }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/crds/nvidia.io_nvcfbackends_crd.yaml b/deploy/helm/nvca-operator/nvca-operator/templates/crds/nvidia.io_nvcfbackends_crd.yaml index cb0d433ab..c85df2a5f 100644 --- a/deploy/helm/nvca-operator/nvca-operator/templates/crds/nvidia.io_nvcfbackends_crd.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/templates/crds/nvidia.io_nvcfbackends_crd.yaml @@ -13,10 +13,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +{{- $controlPlaneID := include "nvcaop.controlPlaneID" . -}} +{{- if not $controlPlaneID -}} +{{- $existing := lookup "apiextensions.k8s.io/v1" "CustomResourceDefinition" "" "nvcfbackends.nvcf.nvidia.io" -}} +{{- $ownedByThisRelease := false -}} +{{- with $existing -}} +{{- $annotations := .metadata.annotations | default dict -}} +{{- $ownedByThisRelease = and + (eq (get $annotations "meta.helm.sh/release-name") $.Release.Name) + (eq (get $annotations "meta.helm.sh/release-namespace") $.Release.Namespace) -}} +{{- end -}} +{{- if or (not $existing) $ownedByThisRelease }} apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: nvcfbackends.nvcf.nvidia.io + annotations: + helm.sh/resource-policy: keep labels: {{- include "nvcaop.labels" . | nindent 4 }} spec: @@ -58,3 +71,5 @@ spec: storage: true subresources: status: {} +{{- end }} +{{- end }} diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/deployment.yaml b/deploy/helm/nvca-operator/nvca-operator/templates/deployment.yaml index df774a2e6..9a56cfbe9 100644 --- a/deploy/helm/nvca-operator/nvca-operator/templates/deployment.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/templates/deployment.yaml @@ -129,6 +129,11 @@ spec: value: /var/run/secrets/ngc-service-key/{{ default "ngcServiceKey" .Values.ngcConfig.serviceKeySecretKeyName }} - name: NVCA_CLUSTER_SOURCE value: {{ .Values.ngcConfig.clusterSource | default "ngc-managed" }} + {{- $controlPlaneID := include "nvcaop.controlPlaneID" . }} + {{- if $controlPlaneID }} + - name: NVCF_CONTROL_PLANE_ID + value: {{ $controlPlaneID | quote }} + {{- end }} {{- if and .Values.vaultConfig .Values.vaultConfig.oAuthClientMountPathTemplate }} - name: VAULT_OAUTH_CLIENT_MOUNT_PATH_TEMPLATE value: {{ .Values.vaultConfig.oAuthClientMountPathTemplate | quote }} @@ -216,6 +221,10 @@ spec: - "{{ .Values.nvcaHelmRepositoryPrefix}}" - --cluster-id - "{{ .Values.clusterID }}" + {{- if $controlPlaneID }} + - --control-plane-id + - {{ $controlPlaneID | quote }} + {{- end }} {{- if .Values.enableGXCache }} - --enable-gxcache {{- end}} @@ -232,11 +241,14 @@ spec: - --nvca-worker-degradation-period - {{ .Values.agent.workerDegradationPeriod | quote }} {{- end }} - {{- if and ((.Values.agent).secretMirrorLabelSelector) ((.Values.agent).secretMirrorNamespace) }} + {{- $secretMirrorNamespace := include "nvcaop.secretMirrorNamespace" . }} + {{- if $secretMirrorNamespace }} - --nvca-secret-mirror-source-namespace - - "{{ .Values.agent.secretMirrorNamespace }}" + - {{ $secretMirrorNamespace | quote }} + {{- end }} + {{- if ((.Values.agent).secretMirrorLabelSelector) }} - --nvca-secret-mirror-label-selector - - "{{ .Values.agent.secretMirrorLabelSelector }}" + - {{ .Values.agent.secretMirrorLabelSelector | quote }} {{- end }} {{- $agent := .Values.agent | default dict }} {{- $byooOtelCollectorImage := include "nvcaop.byooOtelCollectorImage" . }} @@ -315,7 +327,7 @@ spec: - /usr/bin/nvca-mirror - run - --target-namespace - - "nvca-system" + - {{ include "nvcaop.systemNamespace" . | quote }} - --log-level - "{{ .Values.logLevel }}" env: diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/image-pull-secret.yaml b/deploy/helm/nvca-operator/nvca-operator/templates/image-pull-secret.yaml index cc3d9128d..c5f037110 100644 --- a/deploy/helm/nvca-operator/nvca-operator/templates/image-pull-secret.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/templates/image-pull-secret.yaml @@ -18,6 +18,7 @@ apiVersion: v1 kind: Secret metadata: name: {{ default "nvca-operator-image-pull" .Values.imagePullSecretName }} + namespace: {{ .Release.Namespace }} labels: {{- include "nvcaop.labels" . | nindent 4 }} type: kubernetes.io/dockerconfigjson diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/ngc-service-key.yaml b/deploy/helm/nvca-operator/nvca-operator/templates/ngc-service-key.yaml index 3468ffe67..ee75390ed 100644 --- a/deploy/helm/nvca-operator/nvca-operator/templates/ngc-service-key.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/templates/ngc-service-key.yaml @@ -18,6 +18,7 @@ apiVersion: v1 kind: Secret metadata: name: ngc-service-key + namespace: {{ .Release.Namespace }} labels: {{- include "nvcaop.labels" . | nindent 4 }} data: diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/nvca-operator_rq.yaml b/deploy/helm/nvca-operator/nvca-operator/templates/nvca-operator_rq.yaml index 24f09df69..339b55ea7 100644 --- a/deploy/helm/nvca-operator/nvca-operator/templates/nvca-operator_rq.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/templates/nvca-operator_rq.yaml @@ -16,8 +16,8 @@ apiVersion: v1 kind: ResourceQuota metadata: - name: nvca-operator - namespace: nvca-operator + name: {{ include "nvcaop.fullname" . }} + namespace: {{ .Release.Namespace }} spec: scopeSelector: matchExpressions: diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/pre-delete-cleanup-rbac.yaml b/deploy/helm/nvca-operator/nvca-operator/templates/pre-delete-cleanup-rbac.yaml index 51c2e07ca..915fe1f21 100644 --- a/deploy/helm/nvca-operator/nvca-operator/templates/pre-delete-cleanup-rbac.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/templates/pre-delete-cleanup-rbac.yaml @@ -24,7 +24,7 @@ metadata: annotations: "helm.sh/hook": pre-delete "helm.sh/hook-weight": "-20" - "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded automountServiceAccountToken: false --- apiVersion: rbac.authorization.k8s.io/v1 @@ -37,7 +37,7 @@ metadata: annotations: "helm.sh/hook": pre-delete "helm.sh/hook-weight": "-20" - "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded rules: - apiGroups: ["nvcf.nvidia.io"] resources: ["nvcfbackends", "nvcfbackends/finalizers", "nvcfbackends/status"] diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/self-managed-nvcfbackend-cm.yaml b/deploy/helm/nvca-operator/nvca-operator/templates/self-managed-nvcfbackend-cm.yaml index 368cfc129..149297f4f 100644 --- a/deploy/helm/nvca-operator/nvca-operator/templates/self-managed-nvcfbackend-cm.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/templates/self-managed-nvcfbackend-cm.yaml @@ -37,6 +37,12 @@ metadata: {{- if eq .Values.ngcConfig.clusterSource "self-managed" }} data: cluster-dto.yaml: | + {{- $controlPlaneID := include "nvcaop.controlPlaneID" . }} + {{- if $controlPlaneID }} + controlPlaneID: {{ $controlPlaneID | quote }} + systemNamespace: {{ include "nvcaop.systemNamespace" . | quote }} + requestsNamespace: {{ include "nvcaop.requestsNamespace" . | quote }} + {{- end }} clusterId: {{ .Values.clusterID | quote }} clusterGroupId: {{ .Values.clusterGroupID | quote }} clusterName: {{ .Values.clusterName | default "nvcf-default" | quote }} diff --git a/deploy/helm/nvca-operator/nvca-operator/values.schema.json b/deploy/helm/nvca-operator/nvca-operator/values.schema.json index 889c2f792..f7e1d1c8b 100644 --- a/deploy/helm/nvca-operator/nvca-operator/values.schema.json +++ b/deploy/helm/nvca-operator/nvca-operator/values.schema.json @@ -108,6 +108,22 @@ "default": [], "items": {} }, + "controlPlane": { + "type": "object", + "description": "Optional identity used to isolate multiple control-plane compute agents in one Kubernetes cluster.", + "properties": { + "id": { + "type": "string", + "description": "Lowercase DNS label prefix. Empty preserves legacy resource names and namespaces.", + "default": "", + "maxLength": 20, + "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", + "not": { + "const": "default" + } + } + } + }, "serviceAccount": { "type": "object", "properties": { @@ -392,8 +408,8 @@ }, "secretMirrorNamespace": { "type": "string", - "description": "Default namespace to mirror custom secrets for nvcf workloads", - "default": "nvca-operator" + "description": "Namespace to source mirrored workload secrets from. Empty derives this control plane's operator namespace.", + "default": "" }, "secretMirrorLabelSelector": { "type": "string", diff --git a/deploy/helm/nvca-operator/nvca-operator/values.yaml b/deploy/helm/nvca-operator/nvca-operator/values.yaml index b2917e222..20c873a98 100644 --- a/deploy/helm/nvca-operator/nvca-operator/values.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/values.yaml @@ -51,6 +51,9 @@ generateImagePullSecret: true imagePullSecretName: "nvca-operator-image-pull" ## @param imagePullSecrets List of pre-existing imagePullSecret objects in the nvca-operator namespace to use for nvca and nvca-operator Pods. Each object must have a 'name' field. Example: [{name: "foo-bar"}, {name: "baz"}] imagePullSecrets: [] +## @param controlPlane.id Optional control-plane identity for running multiple isolated NVCA operators in one Kubernetes cluster. Empty preserves the legacy resource names and namespaces. +controlPlane: + id: "" ## Service Account configuration serviceAccount: ## @param serviceAccount.create Specifies whether a ServiceAccount should be created @@ -167,7 +170,7 @@ resources: ## @param agent.cacheMountOptionsEnabled Enable or disable CSI volume mount options for NVCA caches ## @param agent.cacheMountOptions Comma-separated string of CSI volume mount options (e.g., "ro,noatime,nouuid") used when cacheMountOptionsEnabled is true ## @param agent.workerDegradationPeriod Duration for determining if a worker is degraded (e.g., "90m", "1h30m") -## @param agent.secretMirrorNamespace Default namespace to mirror custom secrets for nvcf workloads +## @param agent.secretMirrorNamespace Namespace to source mirrored workload secrets from. Empty derives this control plane's operator namespace. ## @param agent.secretMirrorLabelSelector Label selector on the secrets in the sourceNamespace ## @param agent.customAnnotations Map of custom annotations to add to the agent pod ## @param agent.gpuProfiling.functionIds Comma/space/newline-separated NVCF function IDs (or "*" for all) whose pods NVCA labels for NVIDIA Nsight GPU profiling. Empty disables profiling. The operator creates and mirrors the nvca-gpu-profiling-config ConfigMap from this value at deploy/upgrade time. @@ -184,7 +187,7 @@ agent: cacheMountOptionsEnabled: true cacheMountOptions: "ro,norecovery,nouuid" workerDegradationPeriod: "" - secretMirrorNamespace: nvca-operator + secretMirrorNamespace: "" secretMirrorLabelSelector: "" customAnnotations: {} ## GPU (Nsight) profiling opt-in: NVCF function IDs to profile ("*" = all) and an optional diff --git a/deploy/helm/nvca-operator/tests/control_plane_isolation_test.sh b/deploy/helm/nvca-operator/tests/control_plane_isolation_test.sh new file mode 100755 index 000000000..37ea82930 --- /dev/null +++ b/deploy/helm/nvca-operator/tests/control_plane_isolation_test.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +source_chart="${repo_root}/../../../src/compute-plane-services/nvca/deployments/nvca-operator" +vendored_chart="${repo_root}/nvca-operator" +tmp_dir="$(mktemp -d)" + +cleanup() { + rm -rf "${tmp_dir}" +} +trap cleanup EXIT + +render_chart() { + local chart="$1" + local release="$2" + local namespace="$3" + local control_plane_id="$4" + local output="$5" + local secret_mirror_selector="${6-nvcf.nvidia.com/mirror-test=true}" + local secret_mirror_namespace="${7-}" + local -a identity_args=() + local -a secret_mirror_args=() + + if [[ -n "${control_plane_id}" ]]; then + identity_args+=(--set-string "controlPlane.id=${control_plane_id}") + fi + if [[ -n "${secret_mirror_selector}" ]]; then + secret_mirror_args+=(--set-string "agent.secretMirrorLabelSelector=${secret_mirror_selector}") + fi + if [[ -n "${secret_mirror_namespace}" ]]; then + secret_mirror_args+=(--set-string "agent.secretMirrorNamespace=${secret_mirror_namespace}") + fi + + helm template "${release}" "${chart}" \ + --include-crds \ + --namespace "${namespace}" \ + --set-string ngcConfig.serviceKey=test-service-key \ + --set-string ngcConfig.clusterSource=self-managed \ + --set-string selfManaged.icmsServiceURL=http://icms.example.invalid:8080 \ + --set-string selfManaged.revalServiceURL=http://reval.example.invalid:8080 \ + --set-string selfManaged.natsURL=nats://nats.example.invalid:4222 \ + "${identity_args[@]}" \ + "${secret_mirror_args[@]}" > "${output}" +} + +assert_equal() { + local expected="$1" + local actual="$2" + local description="$3" + + if [[ "${actual}" != "${expected}" ]]; then + printf 'expected %s to be %q, got %q\n' "${description}" "${expected}" "${actual}" >&2 + exit 1 + fi +} + +assert_manifest_not_contains() { + local needle="$1" + local manifest="$2" + local description="$3" + + if grep -Fq -- "${needle}" "${manifest}"; then + printf 'expected %s to be absent in %s\n' "${description}" "${manifest}" >&2 + exit 1 + fi +} + +manifest_value() { + local expression="$1" + local manifest="$2" + yq -r "${expression}" "${manifest}" +} + +assert_no_resource_collisions() { + local manifest_a="$1" + local manifest_b="$2" + local resources_a="${tmp_dir}/resources-a.txt" + local resources_b="${tmp_dir}/resources-b.txt" + local collisions="${tmp_dir}/collisions.txt" + + yq -r 'select(.kind != null and .kind != "CustomResourceDefinition") | + [.apiVersion, .kind, (.metadata.namespace // ""), .metadata.name] | @tsv' \ + "${manifest_a}" | sed '/^$/d' | sort -u > "${resources_a}" + yq -r 'select(.kind != null and .kind != "CustomResourceDefinition") | + [.apiVersion, .kind, (.metadata.namespace // ""), .metadata.name] | @tsv' \ + "${manifest_b}" | sed '/^$/d' | sort -u > "${resources_b}" + comm -12 "${resources_a}" "${resources_b}" > "${collisions}" + + if [[ -s "${collisions}" ]]; then + echo "control-plane chart renders collide:" >&2 + cat "${collisions}" >&2 + exit 1 + fi +} + +for chart in "${source_chart}" "${vendored_chart}"; do + chart_name="$(basename "$(dirname "${chart}")")-$(basename "${chart}")" + crd_template="${chart}/templates/crds/nvidia.io_nvcfbackends_crd.yaml" + legacy_manifest="${tmp_dir}/${chart_name}-legacy.yaml" + plane_a_manifest="${tmp_dir}/${chart_name}-plane-a.yaml" + plane_b_manifest="${tmp_dir}/${chart_name}-plane-b.yaml" + selector_off_manifest="${tmp_dir}/${chart_name}-plane-a-selector-off.yaml" + custom_source_manifest="${tmp_dir}/${chart_name}-plane-a-custom-source.yaml" + + render_chart "${chart}" nvca-operator nvca-operator "" "${legacy_manifest}" + render_chart "${chart}" plane-a-nvca-operator plane-a-nvca-operator plane-a "${plane_a_manifest}" + render_chart "${chart}" plane-b-nvca-operator plane-b-nvca-operator plane-b "${plane_b_manifest}" + render_chart "${chart}" plane-a-nvca-operator plane-a-nvca-operator plane-a "${selector_off_manifest}" "" + render_chart "${chart}" plane-a-nvca-operator plane-a-nvca-operator plane-a "${custom_source_manifest}" "mirror=true" "shared-secrets" + + # The CRD used to be owned as a normal Helm template. Keep it templated so + # an upgrade cannot prune it, preserve it on uninstall, and omit it when a + # different release already owns the shared definition. + [[ -f "${crd_template}" ]] || { + echo "expected ownership-gated CRD template in ${chart}" >&2 + exit 1 + } + [[ ! -e "${chart}/crds/nvidia.io_nvcfbackends_crd.yaml" ]] || { + echo "expected CRD to remain upgrade-safe under templates/, not crds/, in ${chart}" >&2 + exit 1 + } + grep -Fq 'helm.sh/resource-policy: keep' "${crd_template}" || { + echo "expected shared CRD to be retained on uninstall in ${chart}" >&2 + exit 1 + } + grep -Fq 'lookup "apiextensions.k8s.io/v1" "CustomResourceDefinition"' "${crd_template}" || { + echo "expected shared CRD ownership lookup in ${chart}" >&2 + exit 1 + } + grep -Fq 'meta.helm.sh/release-name' "${crd_template}" || { + echo "expected shared CRD release-owner gate in ${chart}" >&2 + exit 1 + } + grep -Fq 'meta.helm.sh/release-namespace' "${crd_template}" || { + echo "expected shared CRD release-namespace gate in ${chart}" >&2 + exit 1 + } + + # Empty identity is a strict compatibility mode: names and target namespaces + # remain unchanged, and no new runtime identity is emitted. + assert_equal nvca-operator "$(manifest_value 'select(.kind == "Deployment") | .metadata.name' "${legacy_manifest}")" "legacy Deployment name" + assert_equal nvca-operator "$(manifest_value 'select(.kind == "Deployment") | .metadata.namespace' "${legacy_manifest}")" "legacy Deployment namespace" + assert_equal nvca-operator "$(manifest_value 'select(.kind == "ResourceQuota") | .metadata.name' "${legacy_manifest}")" "legacy ResourceQuota name" + assert_equal nvca-operator "$(manifest_value 'select(.kind == "ResourceQuota") | .metadata.namespace' "${legacy_manifest}")" "legacy ResourceQuota namespace" + assert_equal nvca-system "$(manifest_value 'select(.kind == "Deployment") | .spec.template.spec.containers[] | select(.name == "nvca-mirror") | .args[3]' "${legacy_manifest}")" "legacy mirror target" + assert_manifest_not_contains '--control-plane-id' "${legacy_manifest}" "legacy control-plane CLI flag" + assert_manifest_not_contains 'NVCF_CONTROL_PLANE_ID' "${legacy_manifest}" "legacy control-plane environment variable" + legacy_args="$(manifest_value 'select(.kind == "Deployment") | .spec.template.spec.containers[] | select(.args[0] == "/usr/bin/nvca-operator") | .args | @json' "${legacy_manifest}")" + if [[ "${legacy_args}" != *'"--nvca-secret-mirror-source-namespace","nvca-operator"'* ]]; then + printf 'expected legacy secret mirror source namespace, got %s\n' "${legacy_args}" >&2 + exit 1 + fi + + for plane in a b; do + manifest_var="plane_${plane}_manifest" + manifest="${!manifest_var}" + control_plane_id="plane-${plane}" + operator_namespace="${control_plane_id}-nvca-operator" + agent_namespace="${control_plane_id}-nvca-system" + requests_namespace="${control_plane_id}-nvcf-backend" + + assert_equal "${operator_namespace}" "$(manifest_value 'select(.kind == "Deployment") | .metadata.name' "${manifest}")" "named Deployment name" + assert_equal "${operator_namespace}" "$(manifest_value 'select(.kind == "Deployment") | .metadata.namespace' "${manifest}")" "named Deployment namespace" + assert_equal "${control_plane_id}" "$(manifest_value 'select(.kind == "Deployment") | .metadata.labels."nvcf.nvidia.com/control-plane-id"' "${manifest}")" "named resource identity label" + assert_equal "${control_plane_id}" "$(manifest_value 'select(.kind == "Deployment") | .spec.template.spec.containers[] | select(.args[0] == "/usr/bin/nvca-operator") | .env[] | select(.name == "NVCF_CONTROL_PLANE_ID") | .value' "${manifest}")" "named runtime identity environment variable" + assert_equal "${control_plane_id}" "$(manifest_value 'select(.kind == "ConfigMap" and .metadata.name == "nvcfbackend-self-managed") | .data."cluster-dto.yaml" | from_yaml | .controlPlaneID' "${manifest}")" "named DTO identity" + assert_equal "${agent_namespace}" "$(manifest_value 'select(.kind == "ConfigMap" and .metadata.name == "nvcfbackend-self-managed") | .data."cluster-dto.yaml" | from_yaml | .systemNamespace' "${manifest}")" "named agent namespace" + assert_equal "${requests_namespace}" "$(manifest_value 'select(.kind == "ConfigMap" and .metadata.name == "nvcfbackend-self-managed") | .data."cluster-dto.yaml" | from_yaml | .requestsNamespace' "${manifest}")" "named requests namespace" + assert_equal "${agent_namespace}" "$(manifest_value 'select(.kind == "Deployment") | .spec.template.spec.containers[] | select(.name == "nvca-mirror") | .args[3]' "${manifest}")" "named mirror target" + assert_equal "${operator_namespace}" "$(manifest_value 'select(.kind == "ResourceQuota") | .metadata.name' "${manifest}")" "named ResourceQuota name" + assert_equal "${operator_namespace}" "$(manifest_value 'select(.kind == "ResourceQuota") | .metadata.namespace' "${manifest}")" "named ResourceQuota namespace" + + args="$(manifest_value 'select(.kind == "Deployment") | .spec.template.spec.containers[] | select(.args[0] == "/usr/bin/nvca-operator") | .args | @json' "${manifest}")" + if [[ "${args}" != *'"--control-plane-id","'"${control_plane_id}"'"'* ]]; then + printf 'expected named runtime args to include --control-plane-id %s, got %s\n' "${control_plane_id}" "${args}" >&2 + exit 1 + fi + if [[ "${args}" != *'"--nvca-secret-mirror-source-namespace","'"${operator_namespace}"'"'* ]]; then + printf 'expected named secret mirror source namespace %s, got %s\n' "${operator_namespace}" "${args}" >&2 + exit 1 + fi + if [[ "${args}" != *'"--nvca-secret-mirror-label-selector","nvcf.nvidia.com/mirror-test=true"'* ]]; then + printf 'expected named secret mirror selector, got %s\n' "${args}" >&2 + exit 1 + fi + done + + selector_off_args="$(manifest_value 'select(.kind == "Deployment") | .spec.template.spec.containers[] | select(.args[0] == "/usr/bin/nvca-operator") | .args | @json' "${selector_off_manifest}")" + if [[ "${selector_off_args}" != *'"--nvca-secret-mirror-source-namespace","plane-a-nvca-operator"'* ]] || + [[ "${selector_off_args}" == *'"--nvca-secret-mirror-label-selector"'* ]]; then + printf 'expected selector-off named render to keep only its plane-local source, got %s\n' "${selector_off_args}" >&2 + exit 1 + fi + + custom_source_args="$(manifest_value 'select(.kind == "Deployment") | .spec.template.spec.containers[] | select(.args[0] == "/usr/bin/nvca-operator") | .args | @json' "${custom_source_manifest}")" + if [[ "${custom_source_args}" != *'"--nvca-secret-mirror-source-namespace","shared-secrets"'* ]] || + [[ "${custom_source_args}" != *'"--nvca-secret-mirror-label-selector","mirror=true"'* ]]; then + printf 'expected explicit shared secret source and selector to remain authoritative, got %s\n' "${custom_source_args}" >&2 + exit 1 + fi + + assert_no_resource_collisions "${plane_a_manifest}" "${plane_b_manifest}" + + for cleanup_kind in ServiceAccount ClusterRole ClusterRoleBinding Job; do + cleanup_policy="$(manifest_value 'select(.kind == "'"${cleanup_kind}"'" and .metadata.name == "plane-a-nvca-operator-pre-delete-cleanup") | .metadata.annotations."helm.sh/hook-delete-policy"' "${plane_a_manifest}")" + assert_equal before-hook-creation,hook-succeeded "${cleanup_policy}" "${cleanup_kind} pre-delete hook cleanup policy" + done + + assert_equal 1 "$(grep -Fc 'kind: CustomResourceDefinition' "${legacy_manifest}")" "legacy CRD upgrade ownership" + assert_equal 0 "$(grep -Fc 'kind: CustomResourceDefinition' "${plane_a_manifest}")" "plane A defers CRD to shared prerequisites" + assert_equal 0 "$(grep -Fc 'kind: CustomResourceDefinition' "${plane_b_manifest}")" "plane B defers CRD to shared prerequisites" +done + +for chart in "${source_chart}" "${vendored_chart}"; do + for invalid_id in default Plane-A plane_a -plane plane- aaaaaaaaaaaaaaaaaaaaa; do + release_name="${invalid_id}-nvca-operator" + if helm template "${release_name}" "${chart}" \ + --namespace "${release_name}" \ + --set-string ngcConfig.serviceKey=test-service-key \ + --set-string "controlPlane.id=${invalid_id}" > /dev/null 2>&1; then + printf 'expected invalid controlPlane.id %q to fail for %s\n' "${invalid_id}" "${chart}" >&2 + exit 1 + fi + done + grep -Fq 'if eq $id "default"' "${chart}/templates/_helpers.tpl" || { + echo "expected helper-level reserved default rejection in ${chart}" >&2 + exit 1 + } + grep -Fq '"const": "default"' "${chart}/values.schema.json" || { + echo "expected schema-level reserved default rejection in ${chart}" >&2 + exit 1 + } +done + +if helm template nvca-operator "${source_chart}" \ + --namespace nvca-operator \ + --set-string ngcConfig.serviceKey=test-service-key \ + --set-string controlPlane.id=plane-a > /dev/null 2>&1; then + echo "expected named control plane to reject the legacy release and namespace" >&2 + exit 1 +fi + +echo "validated legacy compatibility, dual control-plane chart and secret-mirror isolation, runtime identity propagation, shared CRD deferral, and invalid identities" diff --git a/deploy/helm/nvcf-pki/README.md b/deploy/helm/nvcf-pki/README.md index 5b1cd2ee3..127836493 100644 --- a/deploy/helm/nvcf-pki/README.md +++ b/deploy/helm/nvcf-pki/README.md @@ -49,6 +49,7 @@ Chart defaults are defined in [values.yaml](./values.yaml). | --- | --- | --- | | `clusterIssuer.enabled` | `false` | Creates the `ClusterIssuer` when set to `true`. | | `clusterIssuer.name` | `nvcf-openbao-pki` | Name of the cluster-scoped issuer. | +| `clusterIssuer.controlPlaneID` | empty | Optional control-plane owner label used by named self-managed deployments. | | `clusterIssuer.server` | empty | OpenBao server URL reachable from cert-manager. | | `clusterIssuer.path` | empty | OpenBao PKI signing path, excluding the `/v1/` prefix. | | `clusterIssuer.auth.mountPath` | `/v1/auth/jwt` | OpenBao authentication mount used for login. | @@ -57,6 +58,10 @@ Chart defaults are defined in [values.yaml](./values.yaml). | `clusterIssuer.auth.serviceAccount.audience` | empty | Token audience accepted by the OpenBao JWT role. | The chart fails to render when it is enabled and any required value is empty. +When `clusterIssuer.controlPlaneID` is non-empty, the chart adds the +`nvcf.nvidia.com/control-plane-id` label. This label lets the self-managed +lifecycle remove retained issuers belonging to one named control plane without +removing another plane's or an external issuer. Example values: diff --git a/deploy/helm/nvcf-pki/scripts/check-render.sh b/deploy/helm/nvcf-pki/scripts/check-render.sh index 54d44225e..4fbecf4c4 100755 --- a/deploy/helm/nvcf-pki/scripts/check-render.sh +++ b/deploy/helm/nvcf-pki/scripts/check-render.sh @@ -145,6 +145,19 @@ assert_object_count "$enabled_render" 1 assert_clusterissuer_count "$enabled_render" 1 assert_clusterissuer_manifest "$enabled_render" "$issuer_name" +# Named control planes need an explicit ownership marker because Helm retains +# this cluster-scoped resource on uninstall. The empty/default chart value must +# continue to render the exact legacy manifest asserted above. +named_render="${tmpdir}/named.yaml" +render_enabled \ + --set-string clusterIssuer.name=alpha-nvcf-openbao-pki \ + --set-string clusterIssuer.controlPlaneID=alpha \ + >"$named_render" +assert_object_count "$named_render" 1 +assert_clusterissuer_count "$named_render" 1 +assert_contains "$named_render" 'nvcf.nvidia.com/control-plane-id: "alpha"' +assert_contains "$named_render" '"helm.sh/resource-policy": keep' + custom_name_render="${tmpdir}/custom-name.yaml" render_enabled --set-string clusterIssuer.name=custom-openbao-issuer >"$custom_name_render" assert_object_count "$custom_name_render" 1 diff --git a/deploy/helm/nvcf-pki/templates/clusterissuer.yaml b/deploy/helm/nvcf-pki/templates/clusterissuer.yaml index e752c2242..b702b2002 100644 --- a/deploy/helm/nvcf-pki/templates/clusterissuer.yaml +++ b/deploy/helm/nvcf-pki/templates/clusterissuer.yaml @@ -25,6 +25,10 @@ apiVersion: cert-manager.io/v1 kind: ClusterIssuer metadata: name: {{ $name | quote }} + {{- with .Values.clusterIssuer.controlPlaneID }} + labels: + nvcf.nvidia.com/control-plane-id: {{ . | quote }} + {{- end }} annotations: "helm.sh/resource-policy": keep spec: diff --git a/deploy/helm/nvcf-pki/values.yaml b/deploy/helm/nvcf-pki/values.yaml index f49250fc0..d03d714dc 100644 --- a/deploy/helm/nvcf-pki/values.yaml +++ b/deploy/helm/nvcf-pki/values.yaml @@ -16,6 +16,7 @@ clusterIssuer: enabled: false name: nvcf-openbao-pki + controlPlaneID: "" server: "" path: "" auth: diff --git a/deploy/helm/openbao/Makefile b/deploy/helm/openbao/Makefile index 17845d647..20b2934a7 100644 --- a/deploy/helm/openbao/Makefile +++ b/deploy/helm/openbao/Makefile @@ -1,4 +1,4 @@ -.PHONY: all deps build install uninstall clean package push-oci bump-version release +.PHONY: all deps build test install uninstall clean package push-oci bump-version release # Directory variables helm_dir := helm @@ -39,6 +39,10 @@ deps: build: deps +test: deps + @sh ./helm/scripts/test-control-plane-isolation.sh + @bash ../../../migrations/openbao/tests/namespace-isolation-test.sh + install: build @cd helm/secrets && kustomize edit set namespace ${namespace} && kustomize build . | kubectl apply -f - helm install $(release) $(helm_dir) -n $(namespace) --values $(values) $(if $(additional_values),--values $(additional_values),) --rollback-on-failure --create-namespace --wait --wait-for-jobs --timeout 20m @@ -107,4 +111,3 @@ prepare-local: deps echo '{"auths":{"nvcr.io":{"auth":""}}}' > ${DOCKER_CONFIG_JSON}; \ fi @echo "[prepare-local] Successfully prepared local environment" - diff --git a/deploy/helm/openbao/helm/scripts/test-control-plane-isolation.sh b/deploy/helm/openbao/helm/scripts/test-control-plane-isolation.sh new file mode 100755 index 000000000..680ea0c8f --- /dev/null +++ b/deploy/helm/openbao/helm/scripts/test-control-plane-isolation.sh @@ -0,0 +1,61 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eu + +chart_dir="$(cd "$(dirname "$0")/.." && pwd)" +tmpdir="$(mktemp -d)" +render="$tmpdir/render.yaml" +test_chart="$tmpdir/chart" +trap 'rm -rf "$tmpdir"' EXIT + +# Keep the regression self-contained for CI without leaving dependency +# archives in the source tree. +cp -R "$chart_dir" "$test_chart" +helm dependency build "$test_chart" >/dev/null + +helm template openbao-server "$test_chart" \ + --namespace plane-a-vault-system \ + --set openbao.fullnameOverride=plane-a-openbao-server \ + --set openbao.controlPlane.id=plane-a \ + --set openbao.server.image.registry=example.invalid \ + --set openbao.server.image.repository=openbao \ + --set openbao.migrations.image.registry=example.invalid \ + --set openbao.migrations.image.repository=migrations \ + --set openbao.migrations.env[0].name=CUSTOM_MIGRATION_ENV \ + --set openbao.migrations.env[0].value=preserved \ + >"$render" + +for expected in \ + 'name: CUSTOM_MIGRATION_ENV' \ + 'value: preserved' \ + 'name: BAO_SERVICE' \ + 'value: "plane-a-openbao-server.plane-a-vault-system.svc.cluster.local"' \ + 'name: OPENBAO_SERVER_INTERNAL_URL' \ + 'value: "http://plane-a-openbao-server.plane-a-vault-system.svc.cluster.local:8200"' \ + 'name: OPENBAO_JWT_AUDIENCE' \ + 'value: "http://openbao-server.vault-system.svc.cluster.local:8200"' \ + 'name: NVCF_NAMESPACE' \ + 'value: "plane-a-nvcf"' \ + 'name: SIS_NAMESPACE' \ + 'value: "plane-a-sis"' \ + 'name: API_KEYS_NAMESPACE' \ + 'value: "plane-a-api-keys"' \ + 'name: ESS_NAMESPACE' \ + 'value: "plane-a-ess"' \ + 'name: NATS_NAMESPACE' \ + 'value: "plane-a-nats-system"' \ + 'name: NVCF_UI_NAMESPACE' \ + 'value: "plane-a-nvcf-ui"' \ + 'name: NVCA_NAMESPACE' \ + 'value: "plane-a-nvca-system"' \ + 'name: NVCA_OPERATOR_NAMESPACE' \ + 'value: "plane-a-nvca-operator"'; do + if ! grep -Fq "$expected" "$render"; then + echo "FAIL: OpenBao render missing: $expected" >&2 + exit 1 + fi +done + +echo "OpenBao control-plane isolation render checks passed." diff --git a/deploy/helm/openbao/helm/templates/hook-post-02-migrations.yaml b/deploy/helm/openbao/helm/templates/hook-post-02-migrations.yaml index d0d7f5429..1dd52d087 100644 --- a/deploy/helm/openbao/helm/templates/hook-post-02-migrations.yaml +++ b/deploy/helm/openbao/helm/templates/hook-post-02-migrations.yaml @@ -14,6 +14,10 @@ # limitations under the License. {{- $serverFullname := include "nvcf-openbao.serverFullname" . }} +{{- $controlPlaneID := dig "controlPlane" "id" "" .Values.openbao | toString }} +{{- if and $controlPlaneID (or (gt (len $controlPlaneID) 20) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $controlPlaneID))) }} +{{- fail "openbao.controlPlane.id must be a DNS-1123 label of at most 20 characters" }} +{{- end }} apiVersion: batch/v1 kind: Job metadata: @@ -65,6 +69,8 @@ spec: value: "{{- .Values.openbao.migrations.issuerDiscovery.urlOverride | default "https://kubernetes.default.svc/.well-known/openid-configuration" -}}" - name: OIDC_DISCOVERY_INSECURE value: "{{ .Values.openbao.migrations.issuerDiscovery.insecure }}" + - name: OPENBAO_JWT_AUDIENCE + value: {{ required "openbao.migrations.jwtAudience is required" .Values.openbao.migrations.jwtAudience | quote }} {{- if .Values.openbao.migrations.issuerDiscovery.caBundleSecretName }} - name: OIDC_CA_BUNDLE value: "/etc/ssl/certs/oidc/ca.crt" @@ -72,6 +78,28 @@ spec: {{- with .Values.openbao.migrations.env }} {{- toYaml . | nindent 12 }} {{- end }} + {{- if $controlPlaneID }} + - name: BAO_SERVICE + value: {{ printf "%s.%s.svc.cluster.local" $serverFullname (include "nvcf-openbao.namespace" .) | quote }} + - name: OPENBAO_SERVER_INTERNAL_URL + value: {{ printf "http://%s.%s.svc.cluster.local:8200" $serverFullname (include "nvcf-openbao.namespace" .) | quote }} + - name: NVCF_NAMESPACE + value: {{ printf "%s-nvcf" $controlPlaneID | quote }} + - name: SIS_NAMESPACE + value: {{ printf "%s-sis" $controlPlaneID | quote }} + - name: API_KEYS_NAMESPACE + value: {{ printf "%s-api-keys" $controlPlaneID | quote }} + - name: ESS_NAMESPACE + value: {{ printf "%s-ess" $controlPlaneID | quote }} + - name: NATS_NAMESPACE + value: {{ printf "%s-nats-system" $controlPlaneID | quote }} + - name: NVCF_UI_NAMESPACE + value: {{ printf "%s-nvcf-ui" $controlPlaneID | quote }} + - name: NVCA_NAMESPACE + value: {{ printf "%s-nvca-system" $controlPlaneID | quote }} + - name: NVCA_OPERATOR_NAMESPACE + value: {{ printf "%s-nvca-operator" $controlPlaneID | quote }} + {{- end }} resources: {{- toYaml .Values.openbao.hooks.migrations.resources | nindent 12 }} volumes: diff --git a/deploy/helm/openbao/helm/values.yaml b/deploy/helm/openbao/helm/values.yaml index 94cd1aca8..2fde6e15e 100644 --- a/deploy/helm/openbao/helm/values.yaml +++ b/deploy/helm/openbao/helm/values.yaml @@ -20,6 +20,11 @@ openbao: # secrets, volumes, and bootstrap scripts. fullnameOverride: openbao-server + # Optional NVCF control-plane identity. When set, the migration hook receives + # plane-scoped service-account namespaces and the reachable OpenBao address. + controlPlane: + id: "" + hooks: initializeCluster: podAnnotations: {} @@ -43,6 +48,10 @@ openbao: memory: 256Mi migrations: + # Opaque trust-domain used by projected service-account tokens and OpenBao + # JWT roles. This is intentionally independent of the plane-specific + # OpenBao network address. + jwtAudience: http://openbao-server.vault-system.svc.cluster.local:8200 image: registry: "" repository: "" diff --git a/deploy/stacks/nvcf-compute-plane/Makefile b/deploy/stacks/nvcf-compute-plane/Makefile index a77d4ae1c..2f5033531 100644 --- a/deploy/stacks/nvcf-compute-plane/Makefile +++ b/deploy/stacks/nvcf-compute-plane/Makefile @@ -22,7 +22,18 @@ HELMFILE_BIN := $(MAKEFILE_DIR)/bin/helmfile HELM_BIN := $(MAKEFILE_DIR)/bin/helm HELM_PLUGINS_DIR := $(MAKEFILE_DIR)/bin/helm-plugins -# Detect OS and architecture +# Development mode flag - skip version checks since we use pinned binaries +DEV_MODE := 1 + +# Use ensure-binaries hook +INSTALL_PRE_HOOKS := ensure-binaries +APPLY_PRE_HOOKS := ensure-binaries + +# --- Include Core Targets --- +include Makefile.dist + +# Run platform-detection subprocesses only after Makefile.dist has captured, +# removed, and safely resolved the command-line control-plane identity. UNAME_S := $(shell uname -s) UNAME_M := $(shell uname -m) @@ -46,21 +57,11 @@ endif HELMFILE_DOWNLOAD_URL := https://github.com/helmfile/helmfile/releases/download/v$(HELMFILE_VERSION)/helmfile_$(HELMFILE_VERSION)_$(HELMFILE_OS)_$(HELMFILE_ARCH).tar.gz HELM_DOWNLOAD_URL := https://get.helm.sh/helm-v$(HELM_VERSION)-$(HELMFILE_OS)-$(HELMFILE_ARCH).tar.gz -# Development mode flag - skip version checks since we use pinned binaries -DEV_MODE := 1 - -# Use ensure-binaries hook -INSTALL_PRE_HOOKS := ensure-binaries -APPLY_PRE_HOOKS := ensure-binaries - -# --- Include Core Targets --- -include Makefile.dist - # --- Include Docker Targets (optional) --- -include helmfile-docker.mk # --- Development-Only Targets --- -.PHONY: dist clean-dist ensure-helm ensure-helmfile ensure-binaries render-local test-observability-profile test-register-cluster test-kube-context test-nvca-entrypoints test-local generate-golden +.PHONY: dist clean-dist ensure-helm ensure-helmfile ensure-binaries render-local test-observability-profile test-register-cluster test-kube-context test-nvca-entrypoints test-control-plane-id-safety test-control-plane-isolation test-local generate-golden # --- Binary Management (Development Only) --- ensure-helm: @@ -117,6 +118,7 @@ clean-dist: dist: clean-dist @echo ">>> Building distribution package..." @mkdir -p "$(DIST_DIR)/environments" + @mkdir -p "$(DIST_DIR)/scripts" # Copy Makefile.dist as the distribution Makefile @cp "$(MAKEFILE_DIR)/Makefile.dist" "$(DIST_DIR)/Makefile" @@ -127,6 +129,9 @@ dist: clean-dist # Copy environments/base.yaml @cp "$(MAKEFILE_DIR)/environments/base.yaml" "$(DIST_DIR)/environments/" + # Copy identity resolver used by the distributable Makefile + @cp "$(MAKEFILE_DIR)/scripts/resolve-control-plane-id.sh" "$(DIST_DIR)/scripts/" + # Copy helmfile.d files @cp -r "$(MAKEFILE_DIR)/helmfile.d" "$(DIST_DIR)/" @@ -162,12 +167,13 @@ GOLDEN_LOCAL_DIR ?= $(MAKEFILE_DIR)/testdata/golden/local # this repo's tests; `make template` on its own keeps helmfile's default layout. GOLDEN_OUTPUT_DIR_TEMPLATE ?= {{ .OutputDir }}/{{ .State.BaseName }}-{{ .Release.Name }} -render-local: dist +render-local: ensure-binaries dist @echo ">>> Rendering local environment into $(DIST_DIR)/out..." @cp -r testdata/environments/* $(DIST_DIR)/environments/ @cp -r testdata/registration $(DIST_DIR)/ @cd $(DIST_DIR) && CLUSTER_NAME=ncp-local NCA_ID=ncp-local HELMFILE_ENV=local \ - make template OUTPUT_DIR_TEMPLATE='$(GOLDEN_OUTPUT_DIR_TEMPLATE)' + make template DEV_MODE=1 DEV_BIN_DIR="$(MAKEFILE_DIR)/bin" \ + OUTPUT_DIR_TEMPLATE='$(GOLDEN_OUTPUT_DIR_TEMPLATE)' test-observability-profile: @tests/observability-profile.sh @@ -181,7 +187,13 @@ test-kube-context: test-nvca-entrypoints: render-local @tests/nvca-entrypoints.sh "$(DIST_DIR)/out" -test-local: test-observability-profile test-register-cluster test-kube-context test-nvca-entrypoints +test-control-plane-id-safety: + @tests/control-plane-id-safety.sh + +test-control-plane-isolation: ensure-binaries + @tests/control-plane-isolation.sh + +test-local: test-observability-profile test-register-cluster test-kube-context test-nvca-entrypoints test-control-plane-id-safety test-control-plane-isolation @tests/verify-golden.sh "$(GOLDEN_LOCAL_DIR)" "$(DIST_DIR)/out" generate-golden: render-local @@ -215,6 +227,8 @@ help: @echo " install Initial deployment to cluster (helmfile sync)" @echo " apply Update existing deployment (helmfile apply)" @echo " destroy Remove all releases and clean up namespaces" + @echo " install-shared-prerequisites Install cluster-wide prerequisite releases" + @echo " destroy-shared-prerequisites Remove cluster-wide prerequisite releases" @echo " clean Remove generated output directory" @echo "" @echo "Distribution Targets:" @@ -226,12 +240,15 @@ help: @echo " test-register-cluster Verify generated profile registration handoff" @echo " test-kube-context Verify lifecycle and cleanup context propagation" @echo " test-nvca-entrypoints Verify rendered NVCA images use available entrypoints" + @echo " test-control-plane-id-safety Verify literal identity handling under GNU Make" + @echo " test-control-plane-isolation Verify dual control-plane compute isolation and teardown" @echo " test-local Render the local env and diff against golden testdata" @echo " generate-golden Render the local env and update golden testdata" @echo "" @echo "Configuration:" @echo " HELMFILE_ENV Environment name (default: $(HELMFILE_ENV))" @echo " Maps to: environments/.yaml" + @echo " CONTROL_PLANE_ID Optional identity override; defaults to global.controlPlane.id" @echo " HELMFILE_SELECTOR Optional release selector (e.g., name=nvca-operator)" @echo " KUBECONFIG_FILE Path to kubeconfig file (pass to all targets in" @echo " multi-cluster setups so registration reads JWKS" diff --git a/deploy/stacks/nvcf-compute-plane/Makefile.dist b/deploy/stacks/nvcf-compute-plane/Makefile.dist index 2658bee04..9fbeff10d 100644 --- a/deploy/stacks/nvcf-compute-plane/Makefile.dist +++ b/deploy/stacks/nvcf-compute-plane/Makefile.dist @@ -21,6 +21,48 @@ CLUSTER_NAME ?= # Determine the absolute path of the directory containing this Makefile. MAKEFILE_DIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +# Optional identity for one of multiple isolated NVCF control planes in the +# same Kubernetes cluster. A command-line variable is recursively expanded by +# GNU Make when it is exported, so capture its literal bytes before removing it +# from Make's automatic command-line export set. GNU Make 4+ provides the file +# function. The 3.x compatibility branch rejects dollar-bearing command-line +# input before overriding the automatically exported variable. +ENVIRONMENT_VALUES_FILE := $(MAKEFILE_DIR)/environments/$(HELMFILE_ENV).yaml +CONTROL_PLANE_ID_HELPER := $(MAKEFILE_DIR)/scripts/resolve-control-plane-id.sh +MAKE_MAJOR_VERSION := $(firstword $(subst ., ,$(MAKE_VERSION))) +ifeq ($(filter 0 1 2 3,$(MAKE_MAJOR_VERSION)),) +unexport CONTROL_PLANE_ID +CONTROL_PLANE_ID_RAW_FILE := $(shell mktemp -t nvcf-control-plane-id.XXXXXX) +$(file >$(CONTROL_PLANE_ID_RAW_FILE),$(value CONTROL_PLANE_ID)) +override undefine CONTROL_PLANE_ID +CONTROL_PLANE_ID_RESOLUTION := $(shell "$(CONTROL_PLANE_ID_HELPER)" file \ + "$(CONTROL_PLANE_ID_RAW_FILE)" "$(MAKEFILE_DIR)/environments/base.yaml" \ + "$(ENVIRONMENT_VALUES_FILE)") +$(shell rm -f "$(CONTROL_PLANE_ID_RAW_FILE)") +else +LEGACY_RAW_CONTROL_PLANE_ID := $(strip $(value CONTROL_PLANE_ID)) +override CONTROL_PLANE_ID := +ifneq ($(findstring $$,$(LEGACY_RAW_CONTROL_PLANE_ID)),) +CONTROL_PLANE_ID_RESOLUTION := error format +LEGACY_EFFECTIVE_CONTROL_PLANE_ID := +else ifneq ($(LEGACY_RAW_CONTROL_PLANE_ID),) +CONTROL_PLANE_ID_RESOLUTION := ok +LEGACY_EFFECTIVE_CONTROL_PLANE_ID := $(LEGACY_RAW_CONTROL_PLANE_ID) +else +CONTROL_PLANE_ID_RESOLUTION := $(shell "$(CONTROL_PLANE_ID_HELPER)" environment "" \ + "$(MAKEFILE_DIR)/environments/base.yaml" "$(ENVIRONMENT_VALUES_FILE)") +LEGACY_EFFECTIVE_CONTROL_PLANE_ID := $(word 2,$(CONTROL_PLANE_ID_RESOLUTION)) +endif +endif +CONTROL_PLANE_ID_RESOLUTION_STATUS := $(word 1,$(CONTROL_PLANE_ID_RESOLUTION)) +CONTROL_PLANE_ID_RESOLUTION_ERROR := $(word 2,$(CONTROL_PLANE_ID_RESOLUTION)) +ifeq ($(filter 0 1 2 3,$(MAKE_MAJOR_VERSION)),) +override CONTROL_PLANE_ID := $(if $(filter ok,$(CONTROL_PLANE_ID_RESOLUTION_STATUS)),$(word 2,$(CONTROL_PLANE_ID_RESOLUTION)),) +else +override CONTROL_PLANE_ID := $(if $(filter ok,$(CONTROL_PLANE_ID_RESOLUTION_STATUS)),$(LEGACY_EFFECTIVE_CONTROL_PLANE_ID),) +endif +export CONTROL_PLANE_ID + # Directory where templated manifests and the Helmfile registration handoff # will be stored. OUTPUT_DIR ?= $(MAKEFILE_DIR)/out @@ -35,8 +77,11 @@ REQUIRED_HELMFILE_MAJOR := 1 REQUIRED_HELMFILE_MINOR := 1 REQUIRED_HELM_MAJOR := 3 -# Namespaces owned by the compute-plane stack (used by destroy target) -NAMESPACES := nvca-operator grove-system dynamo-system kai-scheduler +# Namespaces owned by the selected lifecycle. Legacy mode keeps the historical +# all-in-one cleanup; named mode only removes its operator namespace. The +# operator cleanup job owns its derived agent, request, and model-cache namespaces. +LEGACY_NAMESPACES := nvca-operator grove-system dynamo-system kai-scheduler +EFFECTIVE_SELECTOR := $(if $(HELMFILE_SELECTOR),$(HELMFILE_SELECTOR),$(if $(CONTROL_PLANE_ID),release-group=workers,)) # Kubeconfig file targeting a specific cluster. Used by helmfile, kubectl, # and nvcf-cli. Required in multi-cluster workflows to avoid registering @@ -49,7 +94,8 @@ KUBECTL_CONTEXT_FLAG = $(if $(COMPUTE_KUBE_CONTEXT),--context "$(COMPUTE_KUBE_CO # PATH and HELM_PLUGINS override for development mode (when using local binaries) ifdef DEV_MODE -PATH_OVERRIDE = PATH="$(MAKEFILE_DIR)/bin:$$PATH" HELM_PLUGINS="$(MAKEFILE_DIR)/bin/helm-plugins" +DEV_BIN_DIR ?= $(MAKEFILE_DIR)/bin +PATH_OVERRIDE = PATH="$(DEV_BIN_DIR):$$PATH" HELM_PLUGINS="$(DEV_BIN_DIR)/helm-plugins" else PATH_OVERRIDE = endif @@ -59,10 +105,16 @@ INSTALL_PRE_HOOKS ?= APPLY_PRE_HOOKS ?= # --- Targets --- -.PHONY: check-versions check-cluster-name template install sync apply destroy clean register-cluster help +.PHONY: check-versions check-yq check-cluster-name check-control-plane-id prepare-control-plane-namespace template install sync apply destroy install-shared-prerequisites destroy-shared-prerequisites clean register-cluster help .DEFAULT_GOAL := help # --- Prerequisite Checks --- +check-yq: + @command -v yq >/dev/null 2>&1 && yq --version 2>/dev/null | grep -Eq 'version v4\.' || { \ + echo "Error: 'yq' v4 is required to resolve global.controlPlane.id" >&2; \ + exit 1; \ + } + check-versions: ifdef DEV_MODE @echo ">>> Development mode: Using pinned binaries (skipping system version checks)" @@ -147,6 +199,51 @@ ifndef CLUSTER_NAME $(error CLUSTER_NAME is required: make $@ CLUSTER_NAME=) endif +check-control-plane-id: + @if [ "$(CONTROL_PLANE_ID_RESOLUTION_STATUS)" != "ok" ]; then \ + case "$(CONTROL_PLANE_ID_RESOLUTION_ERROR)" in \ + yq) echo "Error: 'yq' v4 is required to resolve global.controlPlane.id" >&2 ;; \ + configuration) echo "Error: unable to resolve global.controlPlane.id from the environment values" >&2 ;; \ + reserved) echo "Error: CONTROL_PLANE_ID 'default' is reserved" >&2 ;; \ + length) echo "Error: CONTROL_PLANE_ID must be at most 20 characters" >&2 ;; \ + *) echo "Error: CONTROL_PLANE_ID must be a lowercase RFC 1123 DNS label" >&2 ;; \ + esac; \ + exit 1; \ + elif [ -n "$${CONTROL_PLANE_ID:-}" ]; then \ + control_plane_id="$${CONTROL_PLANE_ID}"; \ + if [ "$${control_plane_id}" = "default" ]; then \ + echo "Error: CONTROL_PLANE_ID 'default' is reserved" >&2; \ + exit 1; \ + fi; \ + if ! printf '%s' "$${control_plane_id}" | grep -Eq '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$$'; then \ + echo "Error: CONTROL_PLANE_ID must be a lowercase RFC 1123 DNS label" >&2; \ + exit 1; \ + fi; \ + if [ "$${#control_plane_id}" -gt 20 ]; then \ + echo "Error: CONTROL_PLANE_ID must be at most 20 characters" >&2; \ + exit 1; \ + fi; \ + fi + +# Helm's --create-namespace path does not apply chart labels to the Namespace. +# Establish the ownership marker before Helm writes any named release resources, +# and refuse to adopt an existing namespace owned by another control plane. +prepare-control-plane-namespace: check-control-plane-id + @if [ -n "$${CONTROL_PLANE_ID:-}" ]; then \ + ns="$${CONTROL_PLANE_ID}-nvca-operator"; \ + if kubectl $(KUBECONFIG_FLAG) $(KUBECTL_CONTEXT_FLAG) get namespace "$$ns" >/dev/null 2>&1; then \ + actual_control_plane_id=$$(kubectl $(KUBECONFIG_FLAG) $(KUBECTL_CONTEXT_FLAG) get namespace "$$ns" -o jsonpath='{.metadata.labels.nvcf\.nvidia\.com/control-plane-id}'); \ + if [ "$$actual_control_plane_id" != "$${CONTROL_PLANE_ID}" ]; then \ + echo "Error: refusing to install into namespace $$ns: expected nvcf.nvidia.com/control-plane-id=$${CONTROL_PLANE_ID}, got '$$actual_control_plane_id'" >&2; \ + exit 1; \ + fi; \ + else \ + kubectl $(KUBECONFIG_FLAG) $(KUBECTL_CONTEXT_FLAG) create namespace "$$ns"; \ + kubectl $(KUBECONFIG_FLAG) $(KUBECTL_CONTEXT_FLAG) label namespace "$$ns" \ + nvcf.nvidia.com/control-plane-id="$${CONTROL_PLANE_ID}" --overwrite; \ + fi; \ + fi + # --- Core Targets --- # Optional override for helmfile's --output-dir-template. Empty by default, so @@ -156,7 +253,7 @@ endif # otherwise make the fixtures differ per checkout location. OUTPUT_DIR_TEMPLATE ?= -template: clean check-versions check-cluster-name +template: clean check-versions check-yq check-cluster-name check-control-plane-id @if [ ! -f "$(REGISTRATION_VALUES_DIR)/$(CLUSTER_NAME)-register-values.yaml" ]; then \ echo "ERROR: Registration values not found at $(REGISTRATION_VALUES_DIR)/$(CLUSTER_NAME)-register-values.yaml"; \ echo " Run 'make register-cluster CLUSTER_NAME=$(CLUSTER_NAME)' first."; \ @@ -177,13 +274,13 @@ template: clean check-versions check-cluster-name CLUSTER_NAME="$(CLUSTER_NAME)" \ NCA_ID="${NCA_ID}" \ OUTPUT_DIR="$(abspath $(OUTPUT_DIR))" \ - $(PATH_OVERRIDE) helmfile --environment default $(KUBECONFIG_FLAG) $(HELMFILE_KUBE_CONTEXT_FLAG) template \ + $(PATH_OVERRIDE) helmfile --environment default $(KUBECONFIG_FLAG) $(HELMFILE_KUBE_CONTEXT_FLAG) $(if $(EFFECTIVE_SELECTOR),--selector $(EFFECTIVE_SELECTOR)) template \ --output-dir "$(OUTPUT_DIR)" \ $(if $(OUTPUT_DIR_TEMPLATE),--output-dir-template '$(OUTPUT_DIR_TEMPLATE)') @echo ">>> Templating complete. Manifests generated in $(OUTPUT_DIR)" -install: check-versions check-cluster-name $(INSTALL_PRE_HOOKS) +install: check-versions check-yq check-cluster-name prepare-control-plane-namespace $(INSTALL_PRE_HOOKS) @if [ ! -f "$(REGISTRATION_VALUES_DIR)/$(CLUSTER_NAME)-register-values.yaml" ]; then \ echo "ERROR: Registration values not found at $(REGISTRATION_VALUES_DIR)/$(CLUSTER_NAME)-register-values.yaml"; \ echo " Run 'make register-cluster CLUSTER_NAME=$(CLUSTER_NAME)' first."; \ @@ -193,7 +290,7 @@ install: check-versions check-cluster-name $(INSTALL_PRE_HOOKS) @echo " Environment: $(HELMFILE_ENV)" $(if $(KUBECONFIG_FILE),@echo " Kubeconfig: $(KUBECONFIG_FILE)") $(if $(COMPUTE_KUBE_CONTEXT),@echo " Kube context: $(COMPUTE_KUBE_CONTEXT)") - $(if $(HELMFILE_SELECTOR),@echo " Selector: $(HELMFILE_SELECTOR)") + $(if $(EFFECTIVE_SELECTOR),@echo " Selector: $(EFFECTIVE_SELECTOR)") mkdir -p "$(OUTPUT_DIR)" cp $(REGISTRATION_VALUES_DIR)/$(CLUSTER_NAME)-register-values.yaml $(OUTPUT_DIR)/ @@ -202,13 +299,13 @@ install: check-versions check-cluster-name $(INSTALL_PRE_HOOKS) CLUSTER_NAME="$(CLUSTER_NAME)" \ NCA_ID="${NCA_ID}" \ OUTPUT_DIR="$(abspath $(OUTPUT_DIR))" \ - $(PATH_OVERRIDE) helmfile --environment default $(KUBECONFIG_FLAG) $(HELMFILE_KUBE_CONTEXT_FLAG) $(if $(HELMFILE_SELECTOR),--selector $(HELMFILE_SELECTOR)) sync + $(PATH_OVERRIDE) helmfile --environment default $(KUBECONFIG_FLAG) $(HELMFILE_KUBE_CONTEXT_FLAG) $(if $(EFFECTIVE_SELECTOR),--selector $(EFFECTIVE_SELECTOR)) sync @echo ">>> Compute-plane install complete." sync: install -apply: check-versions check-cluster-name $(APPLY_PRE_HOOKS) +apply: check-versions check-yq check-cluster-name prepare-control-plane-namespace $(APPLY_PRE_HOOKS) @if [ ! -f "$(REGISTRATION_VALUES_DIR)/$(CLUSTER_NAME)-register-values.yaml" ]; then \ echo "ERROR: Registration values not found at $(REGISTRATION_VALUES_DIR)/$(CLUSTER_NAME)-register-values.yaml"; \ echo " Run 'make register-cluster CLUSTER_NAME=$(CLUSTER_NAME)' first."; \ @@ -218,7 +315,7 @@ apply: check-versions check-cluster-name $(APPLY_PRE_HOOKS) @echo " Environment: $(HELMFILE_ENV)" $(if $(KUBECONFIG_FILE),@echo " Kubeconfig: $(KUBECONFIG_FILE)") $(if $(COMPUTE_KUBE_CONTEXT),@echo " Kube context: $(COMPUTE_KUBE_CONTEXT)") - $(if $(HELMFILE_SELECTOR),@echo " Selector: $(HELMFILE_SELECTOR)") + $(if $(EFFECTIVE_SELECTOR),@echo " Selector: $(EFFECTIVE_SELECTOR)") mkdir -p "$(OUTPUT_DIR)" cp $(REGISTRATION_VALUES_DIR)/$(CLUSTER_NAME)-register-values.yaml $(OUTPUT_DIR)/ @@ -227,37 +324,87 @@ apply: check-versions check-cluster-name $(APPLY_PRE_HOOKS) CLUSTER_NAME="$(CLUSTER_NAME)" \ NCA_ID="${NCA_ID}" \ OUTPUT_DIR="$(abspath $(OUTPUT_DIR))" \ - $(PATH_OVERRIDE) helmfile --environment default $(KUBECONFIG_FLAG) $(HELMFILE_KUBE_CONTEXT_FLAG) $(if $(HELMFILE_SELECTOR),--selector $(HELMFILE_SELECTOR)) apply + $(PATH_OVERRIDE) helmfile --environment default $(KUBECONFIG_FLAG) $(HELMFILE_KUBE_CONTEXT_FLAG) $(if $(EFFECTIVE_SELECTOR),--selector $(EFFECTIVE_SELECTOR)) apply @echo ">>> Helmfile apply complete." -destroy: check-versions check-cluster-name +destroy: check-versions check-yq check-cluster-name check-control-plane-id + @if [ -n "$${CONTROL_PLANE_ID:-}" ]; then \ + ns="$${CONTROL_PLANE_ID}-nvca-operator"; \ + if ! kubectl $(KUBECONFIG_FLAG) $(KUBECTL_CONTEXT_FLAG) get namespace "$$ns" >/dev/null 2>&1; then \ + echo "Error: refusing to delete namespace $$ns before Helm teardown: namespace does not exist" >&2; \ + exit 1; \ + fi; \ + actual_control_plane_id=$$(kubectl $(KUBECONFIG_FLAG) $(KUBECTL_CONTEXT_FLAG) get namespace "$$ns" -o jsonpath='{.metadata.labels.nvcf\.nvidia\.com/control-plane-id}'); \ + if [ "$$actual_control_plane_id" != "$${CONTROL_PLANE_ID}" ]; then \ + echo "Error: refusing to delete namespace $$ns before Helm teardown: expected nvcf.nvidia.com/control-plane-id=$${CONTROL_PLANE_ID}, got '$$actual_control_plane_id'" >&2; \ + exit 1; \ + fi; \ + fi @echo ">>> Destroying compute-plane releases for cluster '$(CLUSTER_NAME)'..." @echo " Environment: $(HELMFILE_ENV)" $(if $(KUBECONFIG_FILE),@echo " Kubeconfig: $(KUBECONFIG_FILE)") $(if $(COMPUTE_KUBE_CONTEXT),@echo " Kube context: $(COMPUTE_KUBE_CONTEXT)") - $(if $(HELMFILE_SELECTOR),@echo " Selector: $(HELMFILE_SELECTOR)") + $(if $(EFFECTIVE_SELECTOR),@echo " Selector: $(EFFECTIVE_SELECTOR)") HELMFILE_ENV="$(HELMFILE_ENV)" \ CLUSTER_NAME="$(CLUSTER_NAME)" \ NCA_ID="${NCA_ID}" \ OUTPUT_DIR="$(abspath $(OUTPUT_DIR))" \ - $(PATH_OVERRIDE) helmfile --environment default $(KUBECONFIG_FLAG) $(HELMFILE_KUBE_CONTEXT_FLAG) $(if $(HELMFILE_SELECTOR),--selector $(HELMFILE_SELECTOR)) destroy + $(PATH_OVERRIDE) helmfile --environment default $(KUBECONFIG_FLAG) $(HELMFILE_KUBE_CONTEXT_FLAG) $(if $(EFFECTIVE_SELECTOR),--selector $(EFFECTIVE_SELECTOR)) destroy @echo ">>> Helmfile destroy complete." -ifndef HELMFILE_SELECTOR - @echo ">>> Deleting namespaces... $(NAMESPACES)" - @for ns in $(NAMESPACES); do \ +ifeq ($(HELMFILE_SELECTOR),) + @if [ -n "$${CONTROL_PLANE_ID:-}" ]; then \ + ns="$${CONTROL_PLANE_ID}-nvca-operator"; \ + echo ">>> Deleting namespace... $$ns"; \ if kubectl $(KUBECONFIG_FLAG) $(KUBECTL_CONTEXT_FLAG) get namespace "$$ns" >/dev/null 2>&1; then \ + actual_control_plane_id=$$(kubectl $(KUBECONFIG_FLAG) $(KUBECTL_CONTEXT_FLAG) get namespace "$$ns" -o jsonpath='{.metadata.labels.nvcf\.nvidia\.com/control-plane-id}'); \ + if [ "$$actual_control_plane_id" != "$${CONTROL_PLANE_ID}" ]; then \ + echo "Error: refusing to delete namespace $$ns: expected nvcf.nvidia.com/control-plane-id=$${CONTROL_PLANE_ID}, got '$$actual_control_plane_id'" >&2; \ + exit 1; \ + fi; \ kubectl $(KUBECONFIG_FLAG) $(KUBECTL_CONTEXT_FLAG) delete namespace "$$ns" --wait=true; \ fi; \ - done + else \ + echo ">>> Deleting namespaces... $(LEGACY_NAMESPACES)"; \ + for ns in $(LEGACY_NAMESPACES); do \ + if kubectl $(KUBECONFIG_FLAG) $(KUBECTL_CONTEXT_FLAG) get namespace "$$ns" >/dev/null 2>&1; then \ + kubectl $(KUBECONFIG_FLAG) $(KUBECTL_CONTEXT_FLAG) delete namespace "$$ns" --wait=true; \ + fi; \ + done; \ + fi @echo ">>> Namespace cleanup complete." else @echo ">>> Skipping namespace cleanup (selector was used)" endif +install-shared-prerequisites: check-versions $(INSTALL_PRE_HOOKS) + @echo ">>> Installing shared compute-plane prerequisites..." + HELMFILE_ENV="$(HELMFILE_ENV)" \ + $(PATH_OVERRIDE) helmfile --file "$(MAKEFILE_DIR)/helmfile.d/01-dependencies.yaml.gotmpl" --environment default $(KUBECONFIG_FLAG) $(HELMFILE_KUBE_CONTEXT_FLAG) sync + +destroy-shared-prerequisites: check-versions + @if ! active_named_namespaces=$$(kubectl $(KUBECONFIG_FLAG) $(KUBECTL_CONTEXT_FLAG) get namespaces \ + -l nvcf.nvidia.com/control-plane-id -o name); then \ + echo "Error: refusing to destroy shared prerequisites: unable to verify active named compute-plane namespaces" >&2; \ + exit 1; \ + fi; \ + if [ -n "$$active_named_namespaces" ]; then \ + echo "Error: refusing to destroy shared prerequisites while named compute-plane namespaces remain:" >&2; \ + printf '%s\n' "$$active_named_namespaces" >&2; \ + exit 1; \ + fi + @echo ">>> Destroying shared compute-plane prerequisites..." + HELMFILE_ENV="$(HELMFILE_ENV)" \ + $(PATH_OVERRIDE) helmfile --file "$(MAKEFILE_DIR)/helmfile.d/01-dependencies.yaml.gotmpl" --environment default $(KUBECONFIG_FLAG) $(HELMFILE_KUBE_CONTEXT_FLAG) destroy + @for ns in nvca-shared-system nvca-operator grove-system dynamo-system kai-scheduler; do \ + if kubectl $(KUBECONFIG_FLAG) $(KUBECTL_CONTEXT_FLAG) get namespace "$$ns" >/dev/null 2>&1; then \ + kubectl $(KUBECONFIG_FLAG) $(KUBECTL_CONTEXT_FLAG) delete namespace "$$ns" --wait=true; \ + fi; \ + done + clean: @echo ">>> Cleaning output directory: $(OUTPUT_DIR)" rm -rf "$(OUTPUT_DIR)" @@ -336,10 +483,13 @@ help: @echo " install Initial deployment to cluster (helmfile sync)" @echo " apply Update existing deployment (helmfile apply)" @echo " destroy Remove all releases and clean up namespaces" + @echo " install-shared-prerequisites Install cluster-wide prerequisite releases" + @echo " destroy-shared-prerequisites Remove cluster-wide prerequisite releases" @echo " clean Remove generated output directory" @echo "" @echo "Configuration:" @echo " HELMFILE_ENV Environment name (default: $(HELMFILE_ENV))" + @echo " CONTROL_PLANE_ID Optional identity override; defaults to global.controlPlane.id" @echo " HELMFILE_SELECTOR Optional release selector (e.g., name=nvca-operator)" @echo " KUBECONFIG_FILE Path to kubeconfig file (pass to all targets in" @echo " multi-cluster setups so registration reads JWKS" diff --git a/deploy/stacks/nvcf-compute-plane/README.md b/deploy/stacks/nvcf-compute-plane/README.md index 5a2d1e933..c1fdc20ff 100644 --- a/deploy/stacks/nvcf-compute-plane/README.md +++ b/deploy/stacks/nvcf-compute-plane/README.md @@ -9,6 +9,7 @@ ML-framework operators (Grove, Dynamo) onto GPU clusters registered with an NVCF - `helmfile` v1.1.x (v1.2.0+ breaks ordering; see version note below) - `helm` v3.x - `helm-diff` plugin +- `yq` v4 - `nvcf-cli` (for cluster registration) - A kubeconfig pointing at the target GPU cluster @@ -109,6 +110,55 @@ compute-plane stack does not override. Only set `global.nvcaOperator.selfManaged.imageCredHelper.imageTag` when pinning a tested replacement helper image. +For pre-release validation, `global.nvcaOperator.chartPath` can point to a +local NVCA operator chart. Leave it empty for the pinned released chart. + +## Multiple Isolated Control Planes + +Set the same control-plane identity used by the selected self-managed control +plane. The identity must be a lowercase DNS label no longer than 20 characters: + +```yaml +global: + controlPlane: + id: plane-a +``` + +A named compute-plane install creates the Helm release and operator namespace +`plane-a-nvca-operator`. The operator then owns only the derived namespaces +`plane-a-nvca-system`, `plane-a-nvcf-backend`, and +`plane-a-nvca-modelcache-init`. Empty identity keeps the legacy names. + +Install cluster-wide prerequisites once, independently from either control +plane. This includes the shared `NVCFBackend` CRD; named worker releases never +own or remove that definition. Named `install`, `apply`, and `destroy` commands +automatically read the identity from the selected environment and select only +the per-instance NVCA release. `CONTROL_PLANE_ID` remains available as an +explicit override: + +```sh +make install-shared-prerequisites HELMFILE_ENV= + +make install \ + CLUSTER_NAME=gpu-plane-a \ + HELMFILE_ENV=plane-a + +make install \ + CLUSTER_NAME=gpu-plane-b \ + HELMFILE_ENV=plane-b +``` + +The command-line identity must match `global.controlPlane.id` when both are +set. Named `install` and `apply` create the operator namespace with its +`nvcf.nvidia.com/control-plane-id` ownership label, or verify that an existing +namespace has the expected label before Helm writes resources. Named `destroy` +checks that label before uninstall and again before deleting the namespace. + +Re-run `install-shared-prerequisites` before a worker upgrade that changes the +CRD. Removing one instance leaves shared prerequisites and the other instance +untouched. `make destroy-shared-prerequisites` refuses to run while any labeled +named compute-plane namespaces remain; remove every named compute plane first. + ## Helmfile Version Note Helmfile v1.2.0+ changed `helmfile.d/` processing to parallel mode which breaks diff --git a/deploy/stacks/nvcf-compute-plane/charts/nvca-shared-crds/Chart.yaml b/deploy/stacks/nvcf-compute-plane/charts/nvca-shared-crds/Chart.yaml new file mode 100644 index 000000000..d86b5d5ba --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/charts/nvca-shared-crds/Chart.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v2 +name: nvca-shared-crds +description: Cluster-scoped CRDs shared by isolated NVCA operator releases +type: application +version: 0.1.0 diff --git a/deploy/stacks/nvcf-compute-plane/charts/nvca-shared-crds/templates/nvidia.io_nvcfbackends_crd.yaml b/deploy/stacks/nvcf-compute-plane/charts/nvca-shared-crds/templates/nvidia.io_nvcfbackends_crd.yaml new file mode 100644 index 000000000..8474b57c7 --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/charts/nvca-shared-crds/templates/nvidia.io_nvcfbackends_crd.yaml @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: nvcfbackends.nvcf.nvidia.io + labels: + app.kubernetes.io/name: nvca-shared-crds + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} +spec: + group: nvcf.nvidia.io + names: + kind: NVCFBackend + listKind: NVCFBackendList + plural: nvcfbackends + singular: nvcfbackend + scope: Namespaced + versions: + - name: v1 + additionalPrinterColumns: + - name: Age + jsonPath: .metadata.creationTimestamp + description: Age of this resource + type: date + - name: Version + jsonPath: .status.version + type: string + description: Current version of the backend + - name: Health + jsonPath: .status.agentStatus + description: Health status of the backend + type: string + schema: + openAPIV3Schema: + properties: + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/stacks/nvcf-compute-plane/environments/base.yaml b/deploy/stacks/nvcf-compute-plane/environments/base.yaml index 5fad5b849..ac12e3052 100644 --- a/deploy/stacks/nvcf-compute-plane/environments/base.yaml +++ b/deploy/stacks/nvcf-compute-plane/environments/base.yaml @@ -3,6 +3,12 @@ global: + # Optional identity for attaching this compute plane to one of multiple + # isolated NVCF control planes in the same Kubernetes cluster. Empty keeps + # the legacy release and namespace names. + controlPlane: + id: "" + # ============================================================================= # Helm Chart Sources Configuration # ============================================================================= @@ -31,6 +37,8 @@ global: # NVCA Operator Configuration # ============================================================================= nvcaOperator: + # Optional local chart override for development and pre-release validation. + chartPath: "" imageTag: "3.2.19" selfManaged: nvcaVersion: "3.2.19" diff --git a/deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl b/deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl index 53179c791..82cc19c49 100644 --- a/deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl +++ b/deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl @@ -20,6 +20,25 @@ environments: --- +{{- $configuredControlPlaneID := dig "global" "controlPlane" "id" "" .Values -}} +{{- $environmentControlPlaneID := env "CONTROL_PLANE_ID" -}} +{{- if and $configuredControlPlaneID $environmentControlPlaneID (ne $configuredControlPlaneID $environmentControlPlaneID) -}} +{{- fail (printf "CONTROL_PLANE_ID %q does not match global.controlPlane.id %q" $environmentControlPlaneID $configuredControlPlaneID) -}} +{{- end -}} +{{- $controlPlaneID := $environmentControlPlaneID | default $configuredControlPlaneID -}} +{{- if not (kindIs "string" $controlPlaneID) -}} +{{- fail "global.controlPlane.id must be a string" -}} +{{- end -}} +{{- if eq $controlPlaneID "default" -}} +{{- fail "global.controlPlane.id \"default\" is reserved" -}} +{{- end -}} +{{- if and $controlPlaneID (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $controlPlaneID)) -}} +{{- fail (printf "global.controlPlane.id must be a lowercase RFC 1123 DNS label, got %q" $controlPlaneID) -}} +{{- end -}} +{{- if gt (len $controlPlaneID) 20 -}} +{{- fail "global.controlPlane.id must be at most 20 characters" -}} +{{- end -}} + repositories: - name: kai-scheduler url: ghcr.io/kai-scheduler/kai-scheduler @@ -41,6 +60,17 @@ helmDefaults: releases: + # Named control planes share the NVCFBackend API definition. Keeping the CRD + # in an explicit release lets either plane be removed or upgraded without + # transferring ownership from one worker release to the other. + - name: nvca-shared-crds + installed: {{ ne $controlPlaneID "" }} + chart: ../charts/nvca-shared-crds + namespace: nvca-shared-system + labels: + release-group: shared-prerequisites + wait: true + - name: kai-scheduler # Required by grove-operator and dynamo-operator. Release name and namespace # must be "kai-scheduler" per KAI installation requirements. @@ -228,7 +258,7 @@ releases: limit: -1 overQuotaWeight: 1 labels: - release-group: workers + release-group: shared-prerequisites wait: true waitForJobs: true @@ -265,7 +295,7 @@ releases: groveOperator: enabled: false labels: - release-group: workers + release-group: shared-prerequisites wait: true waitForJobs: true @@ -348,7 +378,7 @@ releases: crdInstaller: enabled: true labels: - release-group: workers + release-group: shared-prerequisites wait: true waitForJobs: true @@ -377,7 +407,7 @@ releases: groveOperator: enabled: true labels: - release-group: workers + release-group: shared-prerequisites wait: true waitForJobs: true @@ -478,6 +508,6 @@ releases: metricsService: enabled: true labels: - release-group: workers + release-group: shared-prerequisites wait: true waitForJobs: true diff --git a/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl b/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl index 4ba7cf39e..4cdb8d825 100644 --- a/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl +++ b/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl @@ -83,6 +83,30 @@ helmDefaults: {{- $registrationValues := readFile $registrationValuesPath | fromYaml | default dict }} {{- $registrationAgentMergeConfig := dig "agentConfig" "mergeConfig" "" $registrationValues }} {{- $environmentAgentMergeConfig := dig "agentConfig" "mergeConfig" "" .Values }} +{{- $configuredControlPlaneID := dig "global" "controlPlane" "id" "" .Values }} +{{- $environmentControlPlaneID := env "CONTROL_PLANE_ID" }} +{{- if and $configuredControlPlaneID $environmentControlPlaneID (ne $configuredControlPlaneID $environmentControlPlaneID) }} +{{- fail (printf "CONTROL_PLANE_ID %q does not match global.controlPlane.id %q" $environmentControlPlaneID $configuredControlPlaneID) }} +{{- end }} +{{- $controlPlaneID := $environmentControlPlaneID | default $configuredControlPlaneID }} +{{- if not (kindIs "string" $controlPlaneID) }} +{{- fail "global.controlPlane.id must be a string" }} +{{- end }} +{{- if eq $controlPlaneID "default" }} +{{- fail "global.controlPlane.id \"default\" is reserved" }} +{{- end }} +{{- if and $controlPlaneID (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $controlPlaneID)) }} +{{- fail (printf "global.controlPlane.id must be a lowercase RFC 1123 DNS label, got %q" $controlPlaneID) }} +{{- end }} +{{- if gt (len $controlPlaneID) 20 }} +{{- fail "global.controlPlane.id must be at most 20 characters" }} +{{- end }} +{{- $operatorReleaseName := "nvca-operator" }} +{{- $operatorNamespace := "nvca-operator" }} +{{- if $controlPlaneID }} +{{- $operatorReleaseName = printf "%s-nvca-operator" $controlPlaneID }} +{{- $operatorNamespace = $operatorReleaseName }} +{{- end }} {{- $observabilityProfile := dig "observability" "profile" "compute" .Values }} {{- if not (has $observabilityProfile (list "disabled" "control" "compute" "all")) }} {{- fail (printf "observability.profile must be disabled, control, compute, or all, got %q" $observabilityProfile) }} @@ -162,12 +186,15 @@ helmDefaults: releases: - - name: nvca-operator + - name: {{ $operatorReleaseName }} # Released chart from nvca-operator-deploy. Compute-plane base values own # the operator and NVCA versions used by this stack. - chart: nvcf/helm-nvca-operator + {{- $nvcaOperatorChartPath := dig "chartPath" "" $nvcaOp }} + chart: {{ $nvcaOperatorChartPath | default "nvcf/helm-nvca-operator" | quote }} + {{- if not $nvcaOperatorChartPath }} version: 1.21.3 - namespace: nvca-operator + {{- end }} + namespace: {{ $operatorNamespace }} values: - ../global.yaml.gotmpl # Cluster-scoped registration values produced by @@ -199,6 +226,8 @@ releases: chart default stays enabled for ngc-managed installs, which rely on it being generated from ngcConfig.serviceKey. */}} generateImagePullSecret: false + controlPlane: + id: {{ $controlPlaneID | quote }} ngcConfig: clusterSource: self-managed clusterName: {{ requiredEnv "CLUSTER_NAME" }} @@ -273,8 +302,8 @@ releases: {{- $agent := dig "agent" dict $nvcaOp }} {{- $agentTolerations := dig "tolerations" list $agent }} {{- $workloadTolerations := dig "workload" "tolerations" list $agent }} - {{- if or $agentTolerations $workloadTolerations }} agent: + secretMirrorNamespace: {{ $operatorNamespace | quote }} {{- with $agentTolerations }} tolerations: {{ toYaml . | nindent 12 }} @@ -284,7 +313,6 @@ releases: tolerations: {{ toYaml . | nindent 14 }} {{- end }} - {{- end }} labels: release-group: workers wait: true diff --git a/deploy/stacks/nvcf-compute-plane/scripts/resolve-control-plane-id.sh b/deploy/stacks/nvcf-compute-plane/scripts/resolve-control-plane-id.sh new file mode 100755 index 000000000..5c5f0e10e --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/scripts/resolve-control-plane-id.sh @@ -0,0 +1,77 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eu + +capture_mode="${1:-}" +raw_file="${2:-}" +base_values_file="${3:-}" +environment_values_file="${4:-}" + +emit_error() { + printf 'error %s\n' "$1" + exit 0 +} + +case "${capture_mode}" in + file) + [ -n "${raw_file}" ] || emit_error capture + trap 'rm -f "${raw_file}"' EXIT HUP INT TERM + [ -f "${raw_file}" ] || emit_error capture + [ "$(wc -l < "${raw_file}" | tr -d '[:space:]')" = "1" ] || emit_error format + IFS= read -r requested_control_plane_id < "${raw_file}" || requested_control_plane_id="" + ;; + environment) + requested_control_plane_id="${NVCF_RAW_CONTROL_PLANE_ID:-}" + ;; + *) + emit_error capture + ;; +esac + +requested_control_plane_id="$( + printf '%s' "${requested_control_plane_id}" | + sed 's/^[[:space:]]*//; s/[[:space:]]*$//' +)" + +if [ -n "${requested_control_plane_id}" ]; then + resolved_control_plane_id="${requested_control_plane_id}" +else + command -v yq >/dev/null 2>&1 || emit_error yq + yq --version 2>/dev/null | grep -Eq 'version v4\.' || emit_error yq + [ -f "${base_values_file}" ] || emit_error configuration + + if [ -f "${environment_values_file}" ]; then + resolved_control_plane_id="$( + # The dollar-prefixed identifier belongs to yq, not the shell. + # shellcheck disable=SC2016 + yq eval-all -r '. as $item ireduce ({}; . * $item) | .global.controlPlane.id // ""' \ + "${base_values_file}" "${environment_values_file}" 2>/dev/null + )" || emit_error configuration + else + resolved_control_plane_id="$( + yq -r '.global.controlPlane.id // ""' "${base_values_file}" 2>/dev/null + )" || emit_error configuration + fi +fi + +resolved_control_plane_id="$( + printf '%s' "${resolved_control_plane_id}" | + sed 's/^[[:space:]]*//; s/[[:space:]]*$//' +)" + +if [ -z "${resolved_control_plane_id}" ]; then + printf 'ok\n' + exit 0 +fi + +[ "${resolved_control_plane_id}" != "default" ] || emit_error reserved +case "${resolved_control_plane_id}" in + -* | *- | *[!a-z0-9-]*) emit_error format ;; +esac +[ "${#resolved_control_plane_id}" -le 20 ] || emit_error length + +# This is the only path that emits caller-controlled data. The validation above +# restricts it to one short DNS-label token, safe for subsequent Make expansion. +printf 'ok %s\n' "${resolved_control_plane_id}" diff --git a/deploy/stacks/nvcf-compute-plane/tests/control-plane-id-safety.sh b/deploy/stacks/nvcf-compute-plane/tests/control-plane-id-safety.sh new file mode 100755 index 000000000..38905acb5 --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/tests/control-plane-id-safety.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +stack_root="$(cd "$(dirname "$0")/.." && pwd)" +tmp_dir="$(mktemp -d)" +test_stack="${tmp_dir}/path with spaces/stack" + +cleanup() { + rm -rf "${tmp_dir}" +} +trap cleanup EXIT + +mkdir -p "${test_stack}/environments" +cp "${stack_root}/Makefile.dist" "${test_stack}/Makefile" +if [[ -d "${stack_root}/scripts" ]]; then + cp -R "${stack_root}/scripts" "${test_stack}/" +fi + +cat > "${test_stack}/environments/base.yaml" <<'EOF' +global: + controlPlane: + id: plane-env +EOF +cat > "${test_stack}/environments/legacy.yaml" <<'EOF' +global: + controlPlane: + id: "" +EOF +cat > "${test_stack}/environments/reserved.yaml" <<'EOF' +global: + controlPlane: + id: default +EOF + +cat >> "${test_stack}/Makefile" <<'EOF' + +.PHONY: print-control-plane-id +print-control-plane-id: check-control-plane-id + @printf '%s\n' "$${CONTROL_PLANE_ID:-}" +EOF + +assert_equal() { + local expected="$1" + local actual="$2" + local description="$3" + + if [[ "${actual}" != "${expected}" ]]; then + printf 'expected %s to be %q, got %q\n' "${description}" "${expected}" "${actual}" >&2 + exit 1 + fi +} + +resolved_from_environment="$(make --no-print-directory -s -C "${test_stack}" print-control-plane-id)" +assert_equal plane-env "${resolved_from_environment}" "environment control-plane identity" + +resolved_legacy="$( + make --no-print-directory -s -C "${test_stack}" print-control-plane-id HELMFILE_ENV=legacy +)" +assert_equal "" "${resolved_legacy}" "empty legacy control-plane identity" + +resolved_from_command_line="$({ + make --no-print-directory -s -C "${test_stack}" print-control-plane-id \ + CONTROL_PLANE_ID=plane-cli +})" +assert_equal plane-cli "${resolved_from_command_line}" "command-line control-plane identity" + +twenty_character_id=12345678901234567890 +resolved_twenty_character_id="$({ + make --no-print-directory -s -C "${test_stack}" print-control-plane-id \ + "CONTROL_PLANE_ID=${twenty_character_id}" +})" +assert_equal "${twenty_character_id}" "${resolved_twenty_character_id}" \ + "20-character control-plane identity" + +if make --no-print-directory -s -C "${test_stack}" check-control-plane-id \ + CONTROL_PLANE_ID=123456789012345678901 > /dev/null 2>&1; then + echo "expected a 21-character CONTROL_PLANE_ID to be rejected" >&2 + exit 1 +fi + +if make --no-print-directory -s -C "${test_stack}" check-control-plane-id \ + CONTROL_PLANE_ID=default > /dev/null 2>&1; then + echo "expected reserved CONTROL_PLANE_ID=default to be rejected" >&2 + exit 1 +fi + +if make --no-print-directory -s -C "${test_stack}" check-control-plane-id \ + HELMFILE_ENV=reserved > /dev/null 2>&1; then + echo "expected reserved environment controlPlane.id=default to be rejected" >&2 + exit 1 +fi + +if make --no-print-directory -s -C "${test_stack}" check-control-plane-id \ + "CONTROL_PLANE_ID=plane-a'quoted" > /dev/null 2>&1; then + echo "expected quoted CONTROL_PLANE_ID to be rejected literally" >&2 + exit 1 +fi + +make_expression_marker="${tmp_dir}/make-expression-executed" +injected_control_plane_id="\$(shell touch ${make_expression_marker})" +if make --no-print-directory -s -C "${test_stack}" check-control-plane-id \ + "CONTROL_PLANE_ID=${injected_control_plane_id}" > /dev/null 2>&1; then + echo "expected Make-expression CONTROL_PLANE_ID to be rejected" >&2 + exit 1 +fi +if [[ -e "${make_expression_marker}" ]]; then + echo "CONTROL_PLANE_ID expanded and executed a Make shell expression" >&2 + exit 1 +fi + +dev_make_expression_marker="${tmp_dir}/dev-make-expression-executed" +dev_injected_control_plane_id="\$(shell touch ${dev_make_expression_marker})" +if make --no-print-directory -s -C "${stack_root}" check-control-plane-id \ + "CONTROL_PLANE_ID=${dev_injected_control_plane_id}" > /dev/null 2>&1; then + echo "expected the development Make entrypoint to reject a Make-expression CONTROL_PLANE_ID" >&2 + exit 1 +fi +if [[ -e "${dev_make_expression_marker}" ]]; then + echo "development Makefile expanded and executed a Make shell expression" >&2 + exit 1 +fi + +missing_helper_stack="${tmp_dir}/missing-helper-stack" +missing_helper_tmp="${tmp_dir}/missing-helper-tmp" +mkdir -p "${missing_helper_stack}/environments" "${missing_helper_tmp}" +cp "${stack_root}/Makefile.dist" "${missing_helper_stack}/Makefile" +cp "${stack_root}/environments/base.yaml" "${missing_helper_stack}/environments/" +TMPDIR="${missing_helper_tmp}" make --no-print-directory -s -C "${missing_helper_stack}" \ + check-control-plane-id CONTROL_PLANE_ID=plane-cli > /dev/null 2>&1 || true +if find "${missing_helper_tmp}" -type f -name 'nvcf-control-plane-id.*' -print -quit | grep -q .; then + echo "Make left a raw CONTROL_PLANE_ID tempfile behind when the resolver was unavailable" >&2 + exit 1 +fi + +echo "validated literal control-plane identity capture and sanitized resolution" diff --git a/deploy/stacks/nvcf-compute-plane/tests/control-plane-isolation.sh b/deploy/stacks/nvcf-compute-plane/tests/control-plane-isolation.sh new file mode 100755 index 000000000..9a4774897 --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/tests/control-plane-isolation.sh @@ -0,0 +1,541 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +stack_root="$(cd "$(dirname "$0")/.." && pwd)" +monorepo_root="$(cd "${stack_root}/../../.." && pwd)" +chart_path="${monorepo_root}/deploy/helm/nvca-operator/nvca-operator" +tmp_dir="$(mktemp -d)" +test_stack="${tmp_dir}/stack" + +cleanup() { + rm -rf "${tmp_dir}" +} +trap cleanup EXIT + +cp -R "${stack_root}/." "${test_stack}" + +write_environment() { + local id="$1" + local environment_file="$2" + + cat > "${environment_file}" < "${output_dir}/${cluster_name}-register-values.yaml" < "${tmp_dir}/${id}.yaml" +} + +render_shared_prerequisites() { + local id="$1" + local output_dir="${tmp_dir}/shared-out" + + mkdir -p "${output_dir}" + HELMFILE_ENV="${id}" \ + PATH="${stack_root}/bin:${PATH}" \ + HELM_PLUGINS="${stack_root}/bin/helm-plugins" \ + "${stack_root}/bin/helmfile" --file "${test_stack}/helmfile.d/01-dependencies.yaml.gotmpl" \ + --environment default --selector release-group=shared-prerequisites template \ + --output-dir "${output_dir}/rendered" \ + --output-dir-template '{{ .OutputDir }}/{{ .Release.Name }}' + + find "${output_dir}/rendered" -type f -name '*.yaml' -exec cat {} + > "${tmp_dir}/shared.yaml" +} + +render_environment_override_pair() { + local control_plane_id="$1" + local environment_name="override-empty" + local cluster_name="cluster-override" + local output_dir="${tmp_dir}/override-out" + + write_environment "" "${test_stack}/environments/${environment_name}.yaml" + write_registration "${cluster_name}" "${output_dir}" + + CONTROL_PLANE_ID="${control_plane_id}" \ + HELMFILE_ENV="${environment_name}" \ + CLUSTER_NAME="${cluster_name}" \ + NCA_ID=nvcf-default \ + OUTPUT_DIR="${output_dir}" \ + PATH="${stack_root}/bin:${PATH}" \ + HELM_PLUGINS="${stack_root}/bin/helm-plugins" \ + "${stack_root}/bin/helmfile" --file "${test_stack}/helmfile.d/02-nvca.yaml.gotmpl" \ + --environment default --selector release-group=workers template \ + --output-dir "${output_dir}/worker-rendered" \ + --output-dir-template '{{ .OutputDir }}/{{ .Release.Name }}' + + CONTROL_PLANE_ID="${control_plane_id}" \ + HELMFILE_ENV="${environment_name}" \ + PATH="${stack_root}/bin:${PATH}" \ + HELM_PLUGINS="${stack_root}/bin/helm-plugins" \ + "${stack_root}/bin/helmfile" --file "${test_stack}/helmfile.d/01-dependencies.yaml.gotmpl" \ + --environment default --selector release-group=shared-prerequisites template \ + --output-dir "${output_dir}/shared-rendered" \ + --output-dir-template '{{ .OutputDir }}/{{ .Release.Name }}' + + find "${output_dir}/worker-rendered" -type f -name '*.yaml' -exec cat {} + > "${tmp_dir}/override-worker.yaml" + find "${output_dir}/shared-rendered" -type f -name '*.yaml' -exec cat {} + > "${tmp_dir}/override-shared.yaml" +} + +assert_equal() { + local expected="$1" + local actual="$2" + local description="$3" + + if [[ "${actual}" != "${expected}" ]]; then + printf 'expected %s to be %q, got %q\n' "${description}" "${expected}" "${actual}" >&2 + exit 1 + fi +} + +assert_no_collisions() { + local manifest_a="$1" + local manifest_b="$2" + local resources_a="${tmp_dir}/resources-a.txt" + local resources_b="${tmp_dir}/resources-b.txt" + local collisions="${tmp_dir}/collisions.txt" + + yq -r 'select(.kind != null and .kind != "CustomResourceDefinition") | + [.apiVersion, .kind, (.metadata.namespace // ""), .metadata.name] | @tsv' \ + "${manifest_a}" | sed '/^$/d' | sort -u > "${resources_a}" + yq -r 'select(.kind != null and .kind != "CustomResourceDefinition") | + [.apiVersion, .kind, (.metadata.namespace // ""), .metadata.name] | @tsv' \ + "${manifest_b}" | sed '/^$/d' | sort -u > "${resources_b}" + comm -12 "${resources_a}" "${resources_b}" > "${collisions}" + + if [[ -s "${collisions}" ]]; then + echo "compute-plane stack renders collide:" >&2 + cat "${collisions}" >&2 + exit 1 + fi +} + +run_named_destroy() { + local namespace_label="$1" + local lifecycle_log="$2" + local output_file="$3" + local fake_bin="${tmp_dir}/fake-bin" + + mkdir -p "${fake_bin}" + cat > "${fake_bin}/helmfile" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf 'helmfile:%s\n' "$*" >> "${LIFECYCLE_LOG}" +EOF + cat > "${fake_bin}/kubectl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +command_line="$*" +if [[ "${command_line}" == *'get namespaces'* ]] && [[ "${command_line}" == *'nvcf.nvidia.com/control-plane-id'* ]]; then + printf '%s' "${FAKE_ACTIVE_NAMED_NAMESPACES:-}" + exit 0 +fi +if [[ "${command_line}" == *'get namespace'* ]]; then + namespace="" + previous="" + for argument in "$@"; do + if [[ "${previous}" == "namespace" ]]; then + namespace="${argument}" + break + fi + previous="${argument}" + done + printf 'get:%s\n' "${namespace}" >> "${LIFECYCLE_LOG}" + if [[ "${namespace}" == "plane-a-nvca-operator" ]]; then + if [[ "${FAKE_PLANE_A_EXISTS:-1}" != "1" ]]; then + exit 1 + fi + printf '%s' "${FAKE_PLANE_A_LABEL}" + exit 0 + fi + if [[ "${namespace}" == "plane-b-nvca-operator" ]]; then + printf '%s' 'plane-b' + exit 0 + fi + exit 1 +fi + +if [[ "${command_line}" == *'create namespace'* ]]; then + namespace="" + previous="" + for argument in "$@"; do + if [[ "${previous}" == "namespace" ]]; then + namespace="${argument}" + break + fi + previous="${argument}" + done + printf 'create:%s\n' "${namespace}" >> "${LIFECYCLE_LOG}" + exit 0 +fi + +if [[ "${command_line}" == *'label namespace'* ]]; then + namespace="" + ownership_label="" + previous="" + for argument in "$@"; do + if [[ "${previous}" == "namespace" ]]; then + namespace="${argument}" + fi + if [[ "${argument}" == nvcf.nvidia.com/control-plane-id=* ]]; then + ownership_label="${argument#*=}" + fi + previous="${argument}" + done + printf 'label:%s:%s\n' "${namespace}" "${ownership_label}" >> "${LIFECYCLE_LOG}" + exit 0 +fi + +if [[ "${command_line}" == *'delete namespace'* ]]; then + namespace="" + previous="" + for argument in "$@"; do + if [[ "${previous}" == "namespace" ]]; then + namespace="${argument}" + break + fi + previous="${argument}" + done + printf 'delete:%s\n' "${namespace}" >> "${LIFECYCLE_LOG}" + exit 0 +fi + +printf 'unexpected-kubectl:%s\n' "${command_line}" >> "${LIFECYCLE_LOG}" +exit 1 +EOF + chmod +x "${fake_bin}/helmfile" "${fake_bin}/kubectl" + + PATH="${fake_bin}:${PATH}" \ + DEV_BIN_DIR="${fake_bin}" \ + LIFECYCLE_LOG="${lifecycle_log}" \ + FAKE_PLANE_A_LABEL="${namespace_label}" \ + make -C "${test_stack}" destroy \ + CLUSTER_NAME=cluster-a \ + HELMFILE_ENV=override-empty \ + CONTROL_PLANE_ID=plane-a \ + OUTPUT_DIR="${tmp_dir}/plane-a-out" > "${output_file}" 2>&1 +} + +run_named_install() { + local namespace_exists="$1" + local namespace_label="$2" + local lifecycle_log="$3" + local output_file="$4" + local fake_bin="${tmp_dir}/fake-bin" + + PATH="${fake_bin}:${PATH}" \ + DEV_BIN_DIR="${fake_bin}" \ + LIFECYCLE_LOG="${lifecycle_log}" \ + FAKE_PLANE_A_EXISTS="${namespace_exists}" \ + FAKE_PLANE_A_LABEL="${namespace_label}" \ + make -C "${test_stack}" install \ + CLUSTER_NAME=cluster-a \ + HELMFILE_ENV=override-empty \ + CONTROL_PLANE_ID=plane-a \ + REGISTRATION_VALUES_DIR="${tmp_dir}/plane-a-out" \ + OUTPUT_DIR="${tmp_dir}/named-install-out" > "${output_file}" 2>&1 +} + +run_shared_destroy() { + local active_named_namespaces="$1" + local lifecycle_log="$2" + local output_file="$3" + local fake_bin="${tmp_dir}/fake-bin" + + PATH="${fake_bin}:${PATH}" \ + DEV_BIN_DIR="${fake_bin}" \ + LIFECYCLE_LOG="${lifecycle_log}" \ + FAKE_PLANE_A_LABEL=plane-a \ + FAKE_ACTIVE_NAMED_NAMESPACES="${active_named_namespaces}" \ + make -C "${test_stack}" destroy-shared-prerequisites \ + HELMFILE_ENV=override-empty > "${output_file}" 2>&1 +} + +"${stack_root}/bin/helmfile" version >/dev/null 2>&1 || { + echo "pinned helmfile is missing; run 'make ensure-binaries' first" >&2 + exit 1 +} + +render_plane plane-a cluster-a +render_plane plane-b cluster-b +render_shared_prerequisites plane-a +render_environment_override_pair plane-env + +manifest_a="${tmp_dir}/plane-a.yaml" +manifest_b="${tmp_dir}/plane-b.yaml" +shared_manifest="${tmp_dir}/shared.yaml" +override_worker_manifest="${tmp_dir}/override-worker.yaml" +override_shared_manifest="${tmp_dir}/override-shared.yaml" + +assert_equal plane-a-nvca-operator "$(yq -r 'select(.kind == "Deployment") | .metadata.name' "${manifest_a}")" "plane A operator name" +assert_equal plane-b-nvca-operator "$(yq -r 'select(.kind == "Deployment") | .metadata.name' "${manifest_b}")" "plane B operator name" +assert_equal plane-a-nvca-operator "$(yq -r 'select(.kind == "Deployment") | .metadata.namespace' "${manifest_a}")" "plane A operator namespace" +assert_equal plane-b-nvca-operator "$(yq -r 'select(.kind == "Deployment") | .metadata.namespace' "${manifest_b}")" "plane B operator namespace" +assert_equal 0 "$(grep -Fc 'kind: CustomResourceDefinition' "${manifest_a}")" "plane A worker CRD ownership" +assert_equal 0 "$(grep -Fc 'kind: CustomResourceDefinition' "${manifest_b}")" "plane B worker CRD ownership" +assert_equal 1 "$(grep -Fc 'kind: CustomResourceDefinition' "${shared_manifest}")" "shared prerequisite CRD ownership" +assert_equal nvcfbackends.nvcf.nvidia.io "$(yq -r 'select(.kind == "CustomResourceDefinition") | .metadata.name' "${shared_manifest}")" "shared prerequisite CRD name" +assert_equal plane-env-nvca-operator "$(yq -r 'select(.kind == "Deployment") | .metadata.name' "${override_worker_manifest}")" "environment-overridden worker name" +assert_equal 1 "$(grep -Fc 'kind: CustomResourceDefinition' "${override_shared_manifest}")" "environment-overridden shared prerequisite CRD ownership" +assert_no_collisions "${manifest_a}" "${manifest_b}" + +override_shared_dry_run="$({ + make -C "${test_stack}" -n install-shared-prerequisites \ + CONTROL_PLANE_ID=plane-env \ + HELMFILE_ENV=override-empty +} 2>&1)" +if [[ "${override_shared_dry_run}" != *'helmfile.d/01-dependencies.yaml.gotmpl'* ]]; then + printf 'install-shared-prerequisites dry-run did not select the shared dependency helmfile:\n%s\n' "${override_shared_dry_run}" >&2 + exit 1 +fi + +destroy_dry_run="$({ + make -C "${test_stack}" -n destroy \ + CLUSTER_NAME=cluster-a \ + HELMFILE_ENV=plane-a \ + OUTPUT_DIR="${tmp_dir}/plane-a-out" +} 2>&1)" +if [[ "${destroy_dry_run}" != *'--selector release-group=workers'* ]]; then + printf 'expected named destroy to select only the per-instance release, got:\n%s\n' "${destroy_dry_run}" >&2 + exit 1 +fi +if [[ "${destroy_dry_run}" == *'delete namespace "nvca-shared-system"'* ]] || + [[ "${destroy_dry_run}" == *'delete namespace "kai-scheduler"'* ]] || + [[ "${destroy_dry_run}" == *'delete namespace "grove-system"'* ]] || + [[ "${destroy_dry_run}" == *'delete namespace "dynamo-system"'* ]]; then + printf 'named destroy attempts to delete shared prerequisite namespaces:\n%s\n' "${destroy_dry_run}" >&2 + exit 1 +fi +expected_namespace_cleanup="ns=\"\${CONTROL_PLANE_ID}-nvca-operator\"" +if [[ "${destroy_dry_run}" != *"${expected_namespace_cleanup}"* ]]; then + printf 'named destroy does not limit namespace cleanup to plane A:\n%s\n' "${destroy_dry_run}" >&2 + exit 1 +fi +if [[ "${destroy_dry_run}" != *'nvcf\.nvidia\.com/control-plane-id'* ]]; then + printf 'named destroy does not verify namespace ownership before cleanup:\n%s\n' "${destroy_dry_run}" >&2 + exit 1 +fi + +legacy_destroy_dry_run="$({ + make -C "${test_stack}" -n destroy \ + CLUSTER_NAME=legacy \ + HELMFILE_ENV=default \ + OUTPUT_DIR="${tmp_dir}/legacy-out" +} 2>&1)" +if [[ "${legacy_destroy_dry_run}" == *'--selector release-group=workers'* ]]; then + echo "legacy destroy unexpectedly selects only the per-instance release" >&2 + exit 1 +fi +for legacy_namespace in nvca-operator grove-system dynamo-system kai-scheduler; do + if [[ "${legacy_destroy_dry_run}" != *"${legacy_namespace}"* ]]; then + printf 'legacy destroy no longer includes namespace %s\n' "${legacy_namespace}" >&2 + exit 1 + fi +done + +positive_lifecycle_log="${tmp_dir}/positive-lifecycle.log" +positive_lifecycle_output="${tmp_dir}/positive-lifecycle.out" +: > "${positive_lifecycle_log}" +run_named_destroy plane-a "${positive_lifecycle_log}" "${positive_lifecycle_output}" +if ! grep -Fxq 'delete:plane-a-nvca-operator' "${positive_lifecycle_log}"; then + printf 'owned plane A namespace was not deleted:\n%s\n' "$(cat "${positive_lifecycle_log}")" >&2 + exit 1 +fi +if grep -Fxq 'delete:plane-b-nvca-operator' "${positive_lifecycle_log}"; then + printf 'plane A teardown deleted plane B namespace:\n%s\n' "$(cat "${positive_lifecycle_log}")" >&2 + exit 1 +fi + +for negative_case in absent mismatched; do + negative_label="" + if [[ "${negative_case}" == "mismatched" ]]; then + negative_label=plane-b + fi + negative_lifecycle_log="${tmp_dir}/${negative_case}-lifecycle.log" + negative_lifecycle_output="${tmp_dir}/${negative_case}-lifecycle.out" + : > "${negative_lifecycle_log}" + if run_named_destroy "${negative_label}" "${negative_lifecycle_log}" "${negative_lifecycle_output}"; then + printf 'named destroy unexpectedly accepted %s namespace ownership:\n%s\n' \ + "${negative_case}" "$(cat "${negative_lifecycle_output}")" >&2 + exit 1 + fi + if grep -Fxq 'delete:plane-a-nvca-operator' "${negative_lifecycle_log}"; then + printf 'named destroy deleted plane A with %s ownership label:\n%s\n' \ + "${negative_case}" "$(cat "${negative_lifecycle_log}")" >&2 + exit 1 + fi + if grep -Fq 'helmfile:' "${negative_lifecycle_log}"; then + printf 'named destroy invoked Helmfile before rejecting %s ownership:\n%s\n' \ + "${negative_case}" "$(cat "${negative_lifecycle_log}")" >&2 + exit 1 + fi + if ! grep -Fq 'refusing to delete' "${negative_lifecycle_output}"; then + printf 'named destroy did not explain %s ownership refusal:\n%s\n' \ + "${negative_case}" "$(cat "${negative_lifecycle_output}")" >&2 + exit 1 + fi +done + +active_shared_lifecycle_log="${tmp_dir}/active-shared-lifecycle.log" +active_shared_lifecycle_output="${tmp_dir}/active-shared-lifecycle.out" +: > "${active_shared_lifecycle_log}" +if run_shared_destroy 'namespace/plane-b-nvca-operator' \ + "${active_shared_lifecycle_log}" "${active_shared_lifecycle_output}"; then + printf 'shared prerequisite destroy accepted an active named worker:\n%s\n' \ + "$(cat "${active_shared_lifecycle_output}")" >&2 + exit 1 +fi +if grep -Fq 'helmfile:' "${active_shared_lifecycle_log}"; then + printf 'shared prerequisite destroy invoked Helmfile with an active named worker:\n%s\n' \ + "$(cat "${active_shared_lifecycle_log}")" >&2 + exit 1 +fi +if ! grep -Fq 'refusing to destroy shared prerequisites' "${active_shared_lifecycle_output}"; then + printf 'shared prerequisite destroy did not explain its active-worker refusal:\n%s\n' \ + "$(cat "${active_shared_lifecycle_output}")" >&2 + exit 1 +fi + +inactive_shared_lifecycle_log="${tmp_dir}/inactive-shared-lifecycle.log" +inactive_shared_lifecycle_output="${tmp_dir}/inactive-shared-lifecycle.out" +: > "${inactive_shared_lifecycle_log}" +run_shared_destroy '' "${inactive_shared_lifecycle_log}" "${inactive_shared_lifecycle_output}" +if ! grep -Fq 'helmfile:' "${inactive_shared_lifecycle_log}"; then + printf 'shared prerequisite destroy did not invoke Helmfile after a clean preflight:\n%s\n' \ + "$(cat "${inactive_shared_lifecycle_log}")" >&2 + exit 1 +fi + +new_namespace_lifecycle_log="${tmp_dir}/new-namespace-lifecycle.log" +new_namespace_lifecycle_output="${tmp_dir}/new-namespace-lifecycle.out" +: > "${new_namespace_lifecycle_log}" +run_named_install 0 plane-a "${new_namespace_lifecycle_log}" "${new_namespace_lifecycle_output}" +expected_new_namespace_lifecycle=$'get:plane-a-nvca-operator\ncreate:plane-a-nvca-operator\nlabel:plane-a-nvca-operator:plane-a' +if [[ "$(head -n 3 "${new_namespace_lifecycle_log}")" != "${expected_new_namespace_lifecycle}" ]]; then + printf 'named install did not establish namespace ownership before Helmfile:\n%s\n' \ + "$(cat "${new_namespace_lifecycle_log}")" >&2 + exit 1 +fi +if ! grep -Fq 'helmfile:' "${new_namespace_lifecycle_log}"; then + printf 'named install did not invoke Helmfile after namespace ownership was established:\n%s\n' \ + "$(cat "${new_namespace_lifecycle_log}")" >&2 + exit 1 +fi + +if make -C "${test_stack}" check-control-plane-id CONTROL_PLANE_ID=default > /dev/null 2>&1; then + echo "expected Make lifecycle validation to reject reserved CONTROL_PLANE_ID=default" >&2 + exit 1 +fi + +if CONTROL_PLANE_ID=default HELMFILE_ENV=override-empty \ + PATH="${stack_root}/bin:${PATH}" HELM_PLUGINS="${stack_root}/bin/helm-plugins" \ + "${stack_root}/bin/helmfile" --file "${test_stack}/helmfile.d/01-dependencies.yaml.gotmpl" \ + --environment default --selector release-group=shared-prerequisites template \ + --output-dir "${tmp_dir}/reserved-shared-rendered" > /dev/null 2>&1; then + echo "expected 01-dependencies to reject reserved CONTROL_PLANE_ID=default" >&2 + exit 1 +fi + +if CONTROL_PLANE_ID=default HELMFILE_ENV=override-empty CLUSTER_NAME=cluster-override \ + NCA_ID=nvcf-default OUTPUT_DIR="${tmp_dir}/override-out" \ + PATH="${stack_root}/bin:${PATH}" HELM_PLUGINS="${stack_root}/bin/helm-plugins" \ + "${stack_root}/bin/helmfile" --file "${test_stack}/helmfile.d/02-nvca.yaml.gotmpl" \ + --environment default --selector release-group=workers template \ + --output-dir "${tmp_dir}/reserved-worker-rendered" > /dev/null 2>&1; then + echo "expected 02-nvca to reject reserved CONTROL_PLANE_ID=default" >&2 + exit 1 +fi + +make_expression_marker="${tmp_dir}/make-expression-executed" +injected_control_plane_id="\$(shell touch ${make_expression_marker})" +if make -C "${test_stack}" check-control-plane-id \ + "CONTROL_PLANE_ID=${injected_control_plane_id}" > /dev/null 2>&1; then + echo "expected literal Make-expression identity to be rejected" >&2 + exit 1 +fi +if [[ -e "${make_expression_marker}" ]]; then + echo "CONTROL_PLANE_ID expanded and executed a Make expression" >&2 + exit 1 +fi + +if make -C "${test_stack}" check-control-plane-id \ + "CONTROL_PLANE_ID=plane-a'quoted" > /dev/null 2>&1; then + echo "expected quoted CONTROL_PLANE_ID to be rejected literally" >&2 + exit 1 +fi + +write_environment plane_a "${test_stack}/environments/invalid.yaml" +write_registration invalid "${tmp_dir}/invalid-out" +if HELMFILE_ENV=invalid CLUSTER_NAME=invalid NCA_ID=nvcf-default OUTPUT_DIR="${tmp_dir}/invalid-out" \ + PATH="${stack_root}/bin:${PATH}" HELM_PLUGINS="${stack_root}/bin/helm-plugins" \ + "${stack_root}/bin/helmfile" --file "${test_stack}/helmfile.d/02-nvca.yaml.gotmpl" \ + --environment default --selector release-group=workers template \ + --output-dir "${tmp_dir}/invalid-rendered" > /dev/null 2>&1; then + echo "expected invalid global.controlPlane.id to fail" >&2 + exit 1 +fi + +echo "validated dual compute-plane isolation, literal identity handling, reserved-ID rejection, namespace ownership, and guarded worker/shared teardown" diff --git a/deploy/stacks/nvcf-compute-plane/tests/kube-context.sh b/deploy/stacks/nvcf-compute-plane/tests/kube-context.sh index 50b07f301..11c2cede5 100755 --- a/deploy/stacks/nvcf-compute-plane/tests/kube-context.sh +++ b/deploy/stacks/nvcf-compute-plane/tests/kube-context.sh @@ -6,7 +6,9 @@ test_dir="$(mktemp -d)" trap 'rm -rf "${test_dir}"' EXIT cp "${stack_dir}/Makefile.dist" "${test_dir}/Makefile" -mkdir -p "${test_dir}/registration" +cp -R "${stack_dir}/scripts" "${test_dir}/" +mkdir -p "${test_dir}/environments" "${test_dir}/registration" +cp "${stack_dir}/environments/base.yaml" "${test_dir}/environments/" printf 'clusterID: generated-id\n' > "${test_dir}/registration/gpu-a-register-values.yaml" for target in template install apply destroy; do diff --git a/deploy/stacks/nvcf-compute-plane/tests/register-cluster.sh b/deploy/stacks/nvcf-compute-plane/tests/register-cluster.sh index dc322bf00..23710cdd2 100755 --- a/deploy/stacks/nvcf-compute-plane/tests/register-cluster.sh +++ b/deploy/stacks/nvcf-compute-plane/tests/register-cluster.sh @@ -10,9 +10,12 @@ test_dir="$(cd "${test_dir}" && pwd -P)" mkdir -p \ "${test_dir}/compute-plane" \ + "${test_dir}/compute-plane/environments" \ "${test_dir}/self-managed/out" \ "${test_dir}/bin" cp "${stack_dir}/Makefile.dist" "${test_dir}/compute-plane/Makefile" +cp -R "${stack_dir}/scripts" "${test_dir}/compute-plane/" +cp "${stack_dir}/environments/base.yaml" "${test_dir}/compute-plane/environments/" profile="${test_dir}/self-managed/out/control-plane-profile.yaml" printf 'generated-control-plane-profile\n' > "${profile}" diff --git a/deploy/stacks/self-managed/Makefile b/deploy/stacks/self-managed/Makefile index 9de36aa9c..b43007eec 100644 --- a/deploy/stacks/self-managed/Makefile +++ b/deploy/stacks/self-managed/Makefile @@ -1,8 +1,10 @@ # Public developer entry point for the self-managed stack. include Makefile.dist -.PHONY: test test-published-charts +.PHONY: test test-control-plane-isolation test-control-plane-lifecycle test-published-charts test: + @tests/control-plane-isolation.test.sh + @tests/control-plane-lifecycle.test.sh @tests/llm-router-worker-address.sh @tests/llm-router-split-cluster.sh @tests/llm-router-local-chart.sh @@ -24,6 +26,12 @@ test: @tests/apikeys-env-wiring.sh @tests/nats-placement-tags.sh +test-control-plane-isolation: + @tests/control-plane-isolation.test.sh + +test-control-plane-lifecycle: + @tests/control-plane-lifecycle.test.sh + test-published-charts: @: "$${NVCF_PUBLISHED_CHART_REGISTRY:?NVCF_PUBLISHED_CHART_REGISTRY is required}" @: "$${NVCF_PUBLISHED_CHART_REPOSITORY:?NVCF_PUBLISHED_CHART_REPOSITORY is required}" diff --git a/deploy/stacks/self-managed/Makefile.dist b/deploy/stacks/self-managed/Makefile.dist index 5b99749e9..a9fbea06b 100644 --- a/deploy/stacks/self-managed/Makefile.dist +++ b/deploy/stacks/self-managed/Makefile.dist @@ -19,6 +19,53 @@ HELMFILE_ENV ?= default # Determine the absolute path of the directory containing this Makefile. MAKEFILE_DIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +# Optional stable identity for one of multiple control planes sharing a +# Kubernetes cluster. Explicit Make input wins; otherwise resolve the selected +# environment so Helmfile and lifecycle cleanup target the same plane. +REQUESTED_CONTROL_PLANE_ID := $(value CONTROL_PLANE_ID) +REQUESTED_CONTROL_PLANE_DOMAIN := $(value CONTROL_PLANE_DOMAIN) +REQUESTED_CONTROL_PLANE_SHARED_GATEWAY := $(value CONTROL_PLANE_SHARED_GATEWAY) +REQUESTED_CONTROL_PLANE_GRPC_GATEWAY := $(value CONTROL_PLANE_GRPC_GATEWAY) +REQUESTED_CONTROL_PLANE_NATS_GATEWAY := $(value CONTROL_PLANE_NATS_GATEWAY) +unexport CONTROL_PLANE_ID CONTROL_PLANE_DOMAIN +unexport CONTROL_PLANE_SHARED_GATEWAY CONTROL_PLANE_GRPC_GATEWAY CONTROL_PLANE_NATS_GATEWAY +ENVIRONMENT_VALUES_FILE := $(MAKEFILE_DIR)/environments/$(HELMFILE_ENV).yaml +CONFIGURED_CONTROL_PLANE_ID := $(shell if command -v yq >/dev/null 2>&1; then \ + if [ -f "$(ENVIRONMENT_VALUES_FILE)" ]; then \ + yq eval-all -r '. as $$item ireduce ({}; . * $$item) | .global.controlPlane.id // ""' \ + "$(MAKEFILE_DIR)/environments/base.yaml" "$(ENVIRONMENT_VALUES_FILE)"; \ + else \ + yq -r '.global.controlPlane.id // ""' "$(MAKEFILE_DIR)/environments/base.yaml"; \ + fi; \ +fi) +CONFIGURED_CONTROL_PLANE_DOMAIN := $(shell if command -v yq >/dev/null 2>&1; then \ + if [ -f "$(ENVIRONMENT_VALUES_FILE)" ]; then \ + yq eval-all -r '. as $$item ireduce ({}; . * $$item) | .global.domain // ""' \ + "$(MAKEFILE_DIR)/environments/base.yaml" "$(ENVIRONMENT_VALUES_FILE)"; \ + else \ + yq -r '.global.domain // ""' "$(MAKEFILE_DIR)/environments/base.yaml"; \ + fi; \ +fi) +override CONTROL_PLANE_ID := $(if $(REQUESTED_CONTROL_PLANE_ID),$(REQUESTED_CONTROL_PLANE_ID),$(CONFIGURED_CONTROL_PLANE_ID)) +MATCHING_CONFIGURED_CONTROL_PLANE_DOMAIN := $(if $(strip $(CONFIGURED_CONTROL_PLANE_ID)),$(if $(filter $(CONTROL_PLANE_ID),$(CONFIGURED_CONTROL_PLANE_ID)),$(CONFIGURED_CONTROL_PLANE_DOMAIN),),) +override CONTROL_PLANE_DOMAIN := $(if $(REQUESTED_CONTROL_PLANE_DOMAIN),$(REQUESTED_CONTROL_PLANE_DOMAIN),$(MATCHING_CONFIGURED_CONTROL_PLANE_DOMAIN)) +export CONTROL_PLANE_ID CONTROL_PLANE_DOMAIN + +CONTROL_PLANE_PREFIX := $(if $(strip $(CONTROL_PLANE_ID)),$(strip $(CONTROL_PLANE_ID))-,) +override CONTROL_PLANE_SHARED_GATEWAY := $(if $(REQUESTED_CONTROL_PLANE_SHARED_GATEWAY),$(REQUESTED_CONTROL_PLANE_SHARED_GATEWAY),$(CONTROL_PLANE_PREFIX)shared-gw) +override CONTROL_PLANE_GRPC_GATEWAY := $(if $(REQUESTED_CONTROL_PLANE_GRPC_GATEWAY),$(REQUESTED_CONTROL_PLANE_GRPC_GATEWAY),$(CONTROL_PLANE_PREFIX)grpc-gw) +override CONTROL_PLANE_NATS_GATEWAY := $(if $(REQUESTED_CONTROL_PLANE_NATS_GATEWAY),$(REQUESTED_CONTROL_PLANE_NATS_GATEWAY),$(CONTROL_PLANE_PREFIX)nats-gw) +ifneq ($(REQUESTED_CONTROL_PLANE_SHARED_GATEWAY),) +export CONTROL_PLANE_SHARED_GATEWAY +endif +ifneq ($(REQUESTED_CONTROL_PLANE_GRPC_GATEWAY),) +export CONTROL_PLANE_GRPC_GATEWAY +endif +ifneq ($(REQUESTED_CONTROL_PLANE_NATS_GATEWAY),) +export CONTROL_PLANE_NATS_GATEWAY +endif +CONTROL_PLANE_STATE_ARGS := $(if $(strip $(CONTROL_PLANE_ID)),--state-values-set-string global.controlPlane.id=$(strip $(CONTROL_PLANE_ID)) --state-values-set-string global.controlPlane.sharedInfrastructure=external --state-values-set certManager.enabled=false --state-values-set-string global.domain=$(strip $(CONTROL_PLANE_DOMAIN)) --state-values-set-string ingress.gatewayApi.gateways.shared.name=$(strip $(CONTROL_PLANE_SHARED_GATEWAY)) --state-values-set-string ingress.gatewayApi.gateways.grpc.name=$(strip $(CONTROL_PLANE_GRPC_GATEWAY)) --state-values-set-string ingress.gatewayApi.gateways.nats.name=$(strip $(CONTROL_PLANE_NATS_GATEWAY)),) + # Directory where templated manifests will be stored OUTPUT_DIR ?= $(MAKEFILE_DIR)/out @@ -31,8 +78,15 @@ REQUIRED_HELMFILE_MAJOR := 1 REQUIRED_HELMFILE_MINOR := 1 REQUIRED_HELM_MAJOR := 3 -# Namespaces created by the NVCF stack (used by destroy target) -NAMESPACES := cassandra-system nats-system vault-system ncp nvcf api-keys ess sis +# Namespaces created by the NVCF stack (used by install/destroy targets). +LEGACY_NAMESPACES := cassandra-system nats-system vault-system ncp nvcf api-keys ess sis +NAMED_NAMESPACE_SUFFIXES := cassandra-system nats-system vault-system nvcf api-keys ess sis ingress +NAMED_NAMESPACES := $(addprefix $(CONTROL_PLANE_PREFIX),$(NAMED_NAMESPACE_SUFFIXES)) +NAMESPACES := $(if $(strip $(CONTROL_PLANE_ID)),$(NAMED_NAMESPACES),$(LEGACY_NAMESPACES)) +CONTROL_PLANE_NAMESPACE_TOOL := $(MAKEFILE_DIR)/scripts/control-plane-namespaces.sh +CONTROL_PLANE_CLUSTERISSUER_TOOL := $(MAKEFILE_DIR)/scripts/control-plane-clusterissuers.sh +CONTROL_PLANE_VALIDATOR := $(MAKEFILE_DIR)/scripts/validate-control-plane-config.sh +KUBECTL_ENV := $(if $(KUBECONFIG_FILE),KUBECONFIG="$(KUBECONFIG_FILE)",) # Optional kubeconfig file - only used if defined KUBECONFIG_FILE ?= @@ -53,10 +107,20 @@ INSTALL_PRE_HOOKS ?= APPLY_PRE_HOOKS ?= # --- Targets --- -.PHONY: check-versions template install sync apply destroy clean help +.PHONY: check-versions check-yq validate-control-plane-id prepare-control-plane-namespaces template install sync apply destroy clean help .DEFAULT_GOAL := help +# Namespace ownership checks, lifecycle hooks, and Helmfile operations mutate +# shared cluster state and must retain their declared ordering under `make -j`. +.NOTPARALLEL: + # --- Prerequisite Checks --- +check-yq: + @command -v yq >/dev/null 2>&1 && yq --version 2>/dev/null | grep -Eq 'version v?4\.' || { \ + echo "Error: 'yq' v4 is required to resolve the selected control-plane environment" >&2; \ + exit 1; \ + } + check-versions: ifdef DEV_MODE @echo ">>> Development mode: Using pinned binaries (skipping system version checks)" @@ -138,7 +202,15 @@ endif # --- Core Targets --- -template: clean check-versions +validate-control-plane-id: check-yq + @"$(CONTROL_PLANE_VALIDATOR)" + +prepare-control-plane-namespaces: validate-control-plane-id +ifneq ($(strip $(CONTROL_PLANE_ID)),) + @$(KUBECTL_ENV) "$(CONTROL_PLANE_NAMESPACE_TOOL)" prepare "$(CONTROL_PLANE_ID)" $(NAMED_NAMESPACES) +endif + +template: clean check-versions validate-control-plane-id @echo ">>> Templating Helmfile..." @echo " Helmfile Directory: helmfile.d/" @echo " Environment: $(HELMFILE_ENV)" @@ -149,11 +221,11 @@ template: clean check-versions HELMFILE_ENV="$(HELMFILE_ENV)" \ OUTPUT_DIR="$(OUTPUT_DIR)" \ - $(PATH_OVERRIDE) helmfile --environment default $(KUBECONFIG_FLAG) template --output-dir "$(OUTPUT_DIR)" + $(PATH_OVERRIDE) helmfile --environment default $(KUBECONFIG_FLAG) $(CONTROL_PLANE_STATE_ARGS) template --output-dir "$(OUTPUT_DIR)" @echo ">>> Templating complete. Manifests generated in $(OUTPUT_DIR)" -install: check-versions $(INSTALL_PRE_HOOKS) +install: check-versions prepare-control-plane-namespaces $(INSTALL_PRE_HOOKS) @echo ">>> Installing Helmfile configuration..." @echo " Helmfile Directory: helmfile.d/" @echo " Environment: $(HELMFILE_ENV)" @@ -161,13 +233,13 @@ install: check-versions $(INSTALL_PRE_HOOKS) $(if $(HELMFILE_SELECTOR),@echo " Selector: $(HELMFILE_SELECTOR)") HELMFILE_ENV="$(HELMFILE_ENV)" \ - $(PATH_OVERRIDE) helmfile --environment default $(KUBECONFIG_FLAG) $(if $(HELMFILE_SELECTOR),--selector $(HELMFILE_SELECTOR)) sync + $(PATH_OVERRIDE) helmfile --environment default $(KUBECONFIG_FLAG) $(CONTROL_PLANE_STATE_ARGS) $(if $(HELMFILE_SELECTOR),--selector $(HELMFILE_SELECTOR)) sync @echo ">>> Helmfile sync complete." sync: install -apply: check-versions $(APPLY_PRE_HOOKS) +apply: check-versions prepare-control-plane-namespaces $(APPLY_PRE_HOOKS) @echo ">>> Applying Helmfile configuration..." @echo " Helmfile Directory: helmfile.d/" @echo " Environment: $(HELMFILE_ENV)" @@ -175,11 +247,14 @@ apply: check-versions $(APPLY_PRE_HOOKS) $(if $(HELMFILE_SELECTOR),@echo " Selector: $(HELMFILE_SELECTOR)") HELMFILE_ENV="$(HELMFILE_ENV)" \ - $(PATH_OVERRIDE) helmfile --environment default $(KUBECONFIG_FLAG) $(if $(HELMFILE_SELECTOR),--selector $(HELMFILE_SELECTOR)) apply + $(PATH_OVERRIDE) helmfile --environment default $(KUBECONFIG_FLAG) $(CONTROL_PLANE_STATE_ARGS) $(if $(HELMFILE_SELECTOR),--selector $(HELMFILE_SELECTOR)) apply @echo ">>> Helmfile apply complete." -destroy: check-versions +destroy: check-versions validate-control-plane-id +ifneq ($(strip $(CONTROL_PLANE_ID)),) + @$(KUBECTL_ENV) "$(CONTROL_PLANE_NAMESPACE_TOOL)" verify "$(CONTROL_PLANE_ID)" $(NAMED_NAMESPACES) +endif @echo ">>> Destroying Helmfile releases..." @echo " Helmfile Directory: helmfile.d/" @echo " Environment: $(HELMFILE_ENV)" @@ -187,17 +262,24 @@ destroy: check-versions $(if $(HELMFILE_SELECTOR),@echo " Selector: $(HELMFILE_SELECTOR)") HELMFILE_ENV="$(HELMFILE_ENV)" \ - $(PATH_OVERRIDE) helmfile --environment default $(KUBECONFIG_FLAG) $(if $(HELMFILE_SELECTOR),--selector $(HELMFILE_SELECTOR)) destroy + $(PATH_OVERRIDE) helmfile --environment default $(KUBECONFIG_FLAG) $(CONTROL_PLANE_STATE_ARGS) $(if $(HELMFILE_SELECTOR),--selector $(HELMFILE_SELECTOR)) destroy @echo ">>> Helmfile destroy complete." ifndef HELMFILE_SELECTOR +ifneq ($(strip $(CONTROL_PLANE_ID)),) + @$(KUBECTL_ENV) "$(CONTROL_PLANE_CLUSTERISSUER_TOOL)" cleanup "$(CONTROL_PLANE_ID)" +endif @echo ">>> Deleting namespaces... $(NAMESPACES)" - @for ns in $(NAMESPACES); do \ - if kubectl get namespace "$$ns" >/dev/null 2>&1; then \ - kubectl delete namespace "$$ns" --wait=true; \ +ifneq ($(strip $(CONTROL_PLANE_ID)),) + @$(KUBECTL_ENV) "$(CONTROL_PLANE_NAMESPACE_TOOL)" cleanup "$(CONTROL_PLANE_ID)" $(NAMED_NAMESPACES) +else + @for ns in $(LEGACY_NAMESPACES); do \ + if kubectl $(KUBECONFIG_FLAG) get namespace "$$ns" >/dev/null 2>&1; then \ + kubectl $(KUBECONFIG_FLAG) delete namespace "$$ns" --wait=true; \ fi; \ done +endif @echo ">>> Namespace cleanup complete." else @echo ">>> Skipping namespace cleanup (selector was used)" @@ -226,6 +308,9 @@ help: @echo "Configuration:" @echo " HELMFILE_ENV Environment name (default: $(HELMFILE_ENV))" @echo " Must match a file: environments/.yaml" + @echo " CONTROL_PLANE_ID Optional namespace prefix for an isolated control plane" + @echo " CONTROL_PLANE_DOMAIN Required unique DNS domain for a named control plane" + @echo " CONTROL_PLANE_{SHARED,GRPC,NATS}_GATEWAY Optional prefixed external Gateway names" @echo " HELMFILE_SELECTOR Optional release selector (e.g., name=api-keys)" @echo " KUBECONFIG_FILE Optional path to kubeconfig file" @echo " OUTPUT_DIR Output directory for template (default: ./out)" diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml index 88b4dd6d3..b464b60a0 100644 --- a/deploy/stacks/self-managed/environments/base.yaml +++ b/deploy/stacks/self-managed/environments/base.yaml @@ -3,6 +3,14 @@ global: + # Stable identity for one control plane in a shared Kubernetes cluster. + # Empty preserves legacy single-plane names and namespaces. Named planes + # use - and require shared infrastructure to be + # managed outside the plane lifecycle. + controlPlane: + id: "" + sharedInfrastructure: bundled + # Domain for external access (used by Gateway API HTTPRoutes) domain: "localhost" @@ -424,9 +432,11 @@ addons: # Optional overrides; defaults are usually correct. # issuerKind: ClusterIssuer # issuerName: nvcf-openbao-pki - # Stack management defaults to true only for the default - # ClusterIssuer/nvcf-openbao-pki configuration. Set this explicitly to - # manage a custom ClusterIssuer. Leave it false for an external issuer. + # Stack management defaults to true for the default issuer. Legacy mode + # can manage a custom ClusterIssuer when this is explicitly true. A + # named control plane can manage only its canonical + # -nvcf-openbao-pki issuer. Leave this false for an + # external issuer. # clusterIssuer: # enabled: true # namespace: vault-system @@ -453,8 +463,10 @@ addons: pullPolicy: IfNotPresent config: mappingPath: /etc/vanity-gateway/config/config.yaml - nvcfApiEndpoint: http://invocation.nvcf.svc.cluster.local:8080 - llmGatewayEndpoint: http://llm-api-gateway.nvcf.svc.cluster.local:8080 + # Empty derives the endpoint from global.controlPlane.id. + nvcfApiEndpoint: "" + # Empty derives the endpoint from global.controlPlane.id. + llmGatewayEndpoint: "" # Empty by default because the Vanity Gateway chart does not create Vault # Agent injection. Set this only when another component mounts the secrets # file into the pod. diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl index 761961bae..341f7fd04 100644 --- a/deploy/stacks/self-managed/global.yaml.gotmpl +++ b/deploy/stacks/self-managed/global.yaml.gotmpl @@ -24,6 +24,26 @@ tolerations: {{- end -}} {{- end -}} +{{- $controlPlaneID := dig "global" "controlPlane" "id" "" .Values | toString -}} +{{- $controlPlanePrefix := "" -}} +{{- if $controlPlaneID -}} +{{- $controlPlanePrefix = printf "%s-" $controlPlaneID -}} +{{- end -}} +{{- $apiKeysNamespace := printf "%sapi-keys" $controlPlanePrefix -}} +{{- $sisNamespace := printf "%ssis" $controlPlanePrefix -}} +{{- $nvcfNamespace := printf "%snvcf" $controlPlanePrefix -}} +{{- $nvcfUiNamespace := printf "%snvcf-ui" $controlPlanePrefix -}} +{{- $essNamespace := printf "%sess" $controlPlanePrefix -}} +{{- $natsNamespace := printf "%snats-system" $controlPlanePrefix -}} +{{- $vaultNamespace := printf "%svault-system" $controlPlanePrefix -}} +{{- $cassandraNamespace := printf "%scassandra-system" $controlPlanePrefix -}} +{{- $ingressNamespace := .Values.ingress.gatewayApi.controllerNamespace -}} +{{- if $controlPlaneID -}} +{{- $ingressNamespace = printf "%singress" $controlPlanePrefix -}} +{{- end -}} +{{- $openbaoFullname := printf "%sopenbao-server" $controlPlanePrefix -}} +{{- $adminIssuerFullname := printf "%sadmin-token-issuer-proxy" $controlPlanePrefix -}} + cassandra: global: {{- if .Values.global.imagePullSecrets }} @@ -54,6 +74,9 @@ cassandra: size: {{ .Values.global.storageSize | default "10Gi" }} migrations: + {{- if $controlPlaneID }} + controlPlaneID: {{ $controlPlaneID | quote }} + {{- end }} image: registry: {{ .Values.global.image.registry }} repository: {{ .Values.global.image.repository }}/nvcf-cassandra-migrations @@ -109,6 +132,9 @@ cert-manager: repository: {{ .Values.global.image.registry }}/{{ .Values.global.image.repository }}/cert-manager-startupapicheck openbao: + fullnameOverride: {{ $openbaoFullname }} + controlPlane: + id: {{ $controlPlaneID | quote }} {{- if .Values.global.imagePullSecrets }} global: imagePullSecrets: @@ -159,6 +185,27 @@ openbao: {{- $nvcfUiEnabled := dig "addons" "nvcfUi" "enabled" false .Values }} {{- with dig "openbao" "injector" "webhook" dict .Values }} webhook: + {{- if $controlPlaneID }} + {{- range $key, $value := . }} + {{- if ne $key "namespaceSelector" }} + {{ $key }}: + {{- toYaml $value | nindent 8 }} + {{- end }} + {{- end }} + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: + - {{ $apiKeysNamespace }} + - {{ $essNamespace }} + - {{ $natsNamespace }} + - {{ $nvcfNamespace }} + - {{ $sisNamespace }} + {{- if $nvcfUiEnabled }} + - {{ $nvcfUiNamespace }} + {{- end }} + {{- else }} {{- $webhook := deepCopy . }} {{- $matchExpressions := dig "namespaceSelector" "matchExpressions" list $webhook }} {{- if $nvcfUiEnabled }} @@ -172,6 +219,7 @@ openbao: {{- end }} {{- end }} {{- toYaml $webhook | nindent 6 }} + {{- end }} {{- end }} podDisruptionBudget: minAvailable: {{ dig "openbao" "injector" "podDisruptionBudget" "minAvailable" 1 .Values }} @@ -193,10 +241,41 @@ openbao: size: {{ .Values.global.storageSize | default "10Gi" }} ha: + {{- if $controlPlaneID }} + raft: + config: | + ui = true + storage "raft" { + path = "/openbao/data/" + retry_join { + leader_api_addr = "http://{{ $openbaoFullname }}-0.{{ $openbaoFullname }}-internal:8200" + } + retry_join { + leader_api_addr = "http://{{ $openbaoFullname }}-1.{{ $openbaoFullname }}-internal:8200" + } + retry_join { + leader_api_addr = "http://{{ $openbaoFullname }}-2.{{ $openbaoFullname }}-internal:8200" + } + } + listener "tcp" { + tls_disable = 1 + address = "[::]:8200" + cluster_address = "[::]:8201" + } + plugin_directory = "/openbao/plugins/" + service_registration "kubernetes" {} + disable_standby_reads = true + {{- end }} {{- with dig "openbao" "server" "ha" "disruptionBudget" dict .Values }} disruptionBudget: {{- toYaml . | nindent 8 }} {{- end }} + {{- if $controlPlaneID }} + volumes: + - name: openbao-server-unseal + secret: + secretName: {{ $openbaoFullname }}-unseal + {{- end }} nats: {{- $natsServerTags := dig "nats" "config" "merge" "server_tags" (list "dc:ncp" "aws-region:ncp") .Values }} @@ -233,6 +312,10 @@ nats: image: registry: {{ .Values.global.image.registry }} repository: {{ .Values.global.image.repository }}/alpine-k8s + rbac: + openbao: + serviceAccountName: {{ $openbaoFullname }}-initialize-cluster + namespace: {{ $vaultNamespace }} {{- $natsNs := include "nvcf.nodeSelector" (dict "type" "controlplane" "selectors" .Values.global.nodeSelectors) -}} {{- $natsTol := include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) -}} {{- if or $natsNs $natsTol }} @@ -286,7 +369,11 @@ apikeys: startupProbe: {{- toYaml . | nindent 4 }} {{- end }} - {{- with dig "apikeys" "env" dict .Values }} + {{- $apikeysEnv := deepCopy (dig "apikeys" "env" dict .Values) }} + {{- if $controlPlaneID }} + {{- $apikeysEnv = mergeOverwrite (dict "SPRING_CASSANDRA_CONTACT_POINTS" (printf "cassandra.%s.svc.cluster.local" $cassandraNamespace)) $apikeysEnv }} + {{- end }} + {{- with $apikeysEnv }} env: {{- toYaml . | nindent 4 }} {{- end }} @@ -299,6 +386,13 @@ natsAuthCalloutService: image: registry: {{ .Values.global.image.registry }} repository: {{ .Values.global.image.repository }}/nvcf-nats-auth-callout-service + serviceConfig: + service: + nats_url: "nats://nats-0.nats-headless.{{ $natsNamespace }}:4222" + plugin_configs: + nvca-webhook: + config: + url: "http://api.{{ $sisNamespace }}.svc.cluster.local:8080/v1/nvca/nats-authorize" {{- $workerEndpoints := dig "global" "workerEndpoints" dict .Values }} {{- /* Worker sidecar registry host/repository, shared by the NVCF and NVCT env @@ -316,9 +410,39 @@ natsAuthCalloutService: {{- $nvctWorkerGrpcServiceURL := dig "nvctGrpcServiceURL" "" $workerEndpoints }} {{- $invocationWorkerBaseURL := dig "invocationServiceURL" "" $workerEndpoints }} {{- $grpcProxyWorkerConnectURL := dig "grpcProxyWorkerConnectURL" "" $workerEndpoints }} +{{- if and $controlPlaneID (or (not $nvcfWorkerServiceURL) (eq $nvcfWorkerServiceURL "http://api.nvcf.svc.cluster.local:8080")) -}} +{{- $nvcfWorkerServiceURL = printf "http://api.%s.svc.cluster.local:8080" $nvcfNamespace -}} +{{- end -}} +{{- if and $controlPlaneID (or (not $nvcfWorkerGrpcServiceURL) (eq $nvcfWorkerGrpcServiceURL "http://api.nvcf.svc.cluster.local:9090")) -}} +{{- $nvcfWorkerGrpcServiceURL = printf "http://api.%s.svc.cluster.local:9090" $nvcfNamespace -}} +{{- end -}} +{{- if and $controlPlaneID (or (not $nvcfNatsWorkerServiceURL) (eq $nvcfNatsWorkerServiceURL "nats://nats.nats-system.svc.cluster.local:4222")) -}} +{{- $nvcfNatsWorkerServiceURL = printf "nats://nats.%s.svc.cluster.local:4222" $natsNamespace -}} +{{- end -}} +{{- if and $controlPlaneID (or (not $grpcProxyNatsServiceURL) (eq $grpcProxyNatsServiceURL "nats://nats.nats-system.svc.cluster.local:4222")) -}} +{{- $grpcProxyNatsServiceURL = printf "nats://nats.%s.svc.cluster.local:4222" $natsNamespace -}} +{{- end -}} +{{- if and $controlPlaneID (or (not $essWorkerBaseURL) (eq $essWorkerBaseURL "http://ess-api.ess.svc.cluster.local:8080")) -}} +{{- $essWorkerBaseURL = printf "http://ess-api.%s.svc.cluster.local:8080" $essNamespace -}} +{{- end -}} +{{- if and $controlPlaneID (or (not $nvctWorkerServiceURL) (eq $nvctWorkerServiceURL "http://nvct-api.nvcf.svc.cluster.local:8080")) -}} +{{- $nvctWorkerServiceURL = printf "http://nvct-api.%s.svc.cluster.local:8080" $nvcfNamespace -}} +{{- end -}} +{{- if and $controlPlaneID (or (not $nvctWorkerGrpcServiceURL) (eq $nvctWorkerGrpcServiceURL "http://nvct-api.nvcf.svc.cluster.local:9090")) -}} +{{- $nvctWorkerGrpcServiceURL = printf "http://nvct-api.%s.svc.cluster.local:9090" $nvcfNamespace -}} +{{- end -}} +{{- if and $controlPlaneID (or (not $invocationWorkerBaseURL) (eq $invocationWorkerBaseURL "http://invocation.nvcf.svc.cluster.local:8080")) -}} +{{- $invocationWorkerBaseURL = printf "http://invocation.%s.svc.cluster.local:8080" $nvcfNamespace -}} +{{- end -}} +{{- if and $controlPlaneID (or (not $grpcProxyWorkerConnectURL) (eq $grpcProxyWorkerConnectURL "http://grpc.nvcf.svc.cluster.local:10086")) -}} +{{- $grpcProxyWorkerConnectURL = printf "http://grpc.%s.svc.cluster.local:10086" $nvcfNamespace -}} +{{- end -}} +{{- if and $controlPlaneID (or (not $grpcProxyNVCFGrpcServiceURL) (eq $grpcProxyNVCFGrpcServiceURL "http://api.nvcf.svc.cluster.local:9090")) -}} +{{- $grpcProxyNVCFGrpcServiceURL = printf "http://api.%s.svc.cluster.local:9090" $nvcfNamespace -}} +{{- end -}} {{- $grpcProxyWorkerConnectBaseURL := dig "grpcproxy" "workerConnectBaseURL" "" .Values | default $grpcProxyWorkerConnectURL }} {{- $llmRequestRouterGrpcPort := dig "addons" "llm" "requestRouter" "service" "grpcPort" 50071 .Values }} -{{- $llmRequestRouterDefaultAddress := printf "llm-request-router.nvcf.svc.cluster.local:%v" $llmRequestRouterGrpcPort }} +{{- $llmRequestRouterDefaultAddress := printf "llm-request-router.%s.svc.cluster.local:%v" $nvcfNamespace $llmRequestRouterGrpcPort }} {{- $llmRequestRouterWorkerAddress := dig "llmRequestRouterAddress" "" $workerEndpoints | trim | default $llmRequestRouterDefaultAddress }} {{- $llmRequestRouterWorkerAuthority := $llmRequestRouterWorkerAddress }} {{- if hasPrefix "https://" $llmRequestRouterWorkerAuthority }} @@ -490,6 +614,7 @@ api: # registry and repository below in your environment file instead of editing # this template. accountBootstrap: + openbaoServiceAddress: "{{ $openbaoFullname }}.{{ $vaultNamespace }}.svc.cluster.local:8200" image: registry: {{ dig "api" "accountBootstrap" "image" "registry" "" .Values | default "docker.io" }} repository: {{ dig "api" "accountBootstrap" "image" "repository" "" .Values | default "alpine/k8s" }} @@ -532,6 +657,17 @@ api: "NVCF_SIDECARS_HOSTNAME" $sidecarsHost "NVCF_SIDECARS_REPOSITORY" $sidecarsRepo "MANAGEMENT_TRACING_ENABLED" (printf "%v" .Values.global.observability.tracing.enabled) -}} + {{- if $controlPlaneID }} + {{- $_ := set $apiEnv "SPRING_CASSANDRA_CONTACT_POINTS" (printf "cassandra.%s.svc.cluster.local" $cassandraNamespace) }} + {{- $_ := set $apiEnv "SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER_URI" (printf "http://api.%s.svc.cluster.local" $nvcfNamespace) }} + {{- $_ := set $apiEnv "SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_JWK_SET_URI" (printf "http://%s.%s.svc.cluster.local:8200/v1/services/nvcf-api/jwt/jwks" $openbaoFullname $vaultNamespace) }} + {{- $_ := set $apiEnv "NVCF_LLM_REQUEST_ROUTER_WORKER_ADDRESS" (printf "llm-request-router.%s.svc.cluster.local:50071" $nvcfNamespace) }} + {{- $_ := set $apiEnv "NVCF_NOTARY_BASE_URL" (printf "http://notary.%s.svc.cluster.local:8080" $nvcfNamespace) }} + {{- $_ := set $apiEnv "NVCF_REVAL_BASE_URL" (printf "http://reval.%s.svc.cluster.local:8080" $nvcfNamespace) }} + {{- $_ := set $apiEnv "NVCF_ICMS_BASE_URL" (printf "http://api.%s.svc.cluster.local:8080" $sisNamespace) }} + {{- $_ := set $apiEnv "NVCF_API_KEYS_BASE_URL" (printf "http://api-keys.%s.svc.cluster.local:8080" $apiKeysNamespace) }} + {{- $_ := set $apiEnv "NVCF_NATS_URL" (printf "nats://nats.%s.svc.cluster.local:4222" $natsNamespace) }} + {{- end }} {{- with $nvcfWorkerServiceURL }} {{- $_ := set $apiEnv "NVCF_FQDN" . }} {{- end }} @@ -576,11 +712,12 @@ invocation: {{- end }} {{- if .Values.rateLimiter.enabled }} {{- $_ := set $invocationEnv "RATE_LIMIT_ENABLED" "true" }} - {{- $_ := set $invocationEnv "RATE_LIMIT_ADDRESS" "http://ratelimiter.nvcf.svc.cluster.local:7777" }} - {{- end }} - {{- with $invocationWorkerBaseURL }} - {{- $_ := set $invocationEnv "WORKER_STREAM_PROPERTIES__SELF_ADDRESS" . }} + {{- $_ := set $invocationEnv "RATE_LIMIT_ADDRESS" (printf "http://ratelimiter.%s.svc.cluster.local:7777" $nvcfNamespace) }} {{- end }} + {{- $_ := set $invocationEnv "NATS_PROPERTIES__NATS_ADDRESS" (printf "nats://nats.%s.svc.cluster.local:4222" $natsNamespace) }} + {{- $_ := set $invocationEnv "NVCF_API_ADDRESS" (printf "http://api.%s.svc.cluster.local:9090" $nvcfNamespace) }} + {{- $_ := set $invocationEnv "REGIONAL_NVCF_API_GRPC_ADDRESS" (printf "http://api.%s.svc.cluster.local:9090" $nvcfNamespace) }} + {{- $_ := set $invocationEnv "WORKER_STREAM_PROPERTIES__SELF_ADDRESS" ($invocationWorkerBaseURL | default (printf "http://invocation.%s.svc.cluster.local:8080" $nvcfNamespace)) }} {{- $invocationEnv = mergeOverwrite $invocationEnv (deepCopy $configuredInvocationEnv) }} fullnameOverride: invocation-service {{- if .Values.global.imagePullSecrets }} @@ -626,6 +763,18 @@ nvctApi: repository: {{ .Values.global.image.repository }}/nvct-service-oss env: + {{- if $controlPlaneID }} + SPRING_CASSANDRA_CONTACT_POINTS: "cassandra.{{ $cassandraNamespace }}.svc.cluster.local" + SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER_URI: {{ dig "nvctApi" "jwt" "issuerUri" (printf "http://nvct-api.%s.svc.cluster.local" $nvcfNamespace) .Values | quote }} + SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_JWK_SET_URI: {{ dig "nvctApi" "jwt" "jwkSetUri" (printf "http://%s.%s.svc.cluster.local:8200/v1/services/nvct-api/jwt/jwks" $openbaoFullname $vaultNamespace) .Values | quote }} + NVCT_NOTARY_BASE_URL: "http://notary.{{ $nvcfNamespace }}.svc.cluster.local:8080" + NVCT_NVCF_BASE_URL: "http://api.{{ $nvcfNamespace }}.svc.cluster.local:8080" + NVCT_REVAL_BASE_URL: "http://reval.{{ $nvcfNamespace }}.svc.cluster.local:8080" + NVCT_API_KEYS_BASE_URL: "http://api-keys.{{ $apiKeysNamespace }}.svc.cluster.local:8080" + {{- if not (dig "nvctApi" "icmsBaseUrl" nil .Values) }} + NVCT_ICMS_BASE_URL: "http://api.{{ $sisNamespace }}.svc.cluster.local:8080" + {{- end }} + {{- end }} NVCT_SIDECARS_HOSTNAME: {{ $sidecarsHost }} NVCT_SIDECARS_REPOSITORY: {{ $sidecarsRepo }} {{- with $nvctWorkerServiceURL }} @@ -687,7 +836,7 @@ grpcproxy: {{- end }} {{- if .Values.rateLimiter.enabled }} RATE_LIMIT_ENABLED: "true" - RATE_LIMIT_ADDR: "http://ratelimiter.nvcf.svc.cluster.local:7777" + RATE_LIMIT_ADDR: "http://ratelimiter.{{ $nvcfNamespace }}.svc.cluster.local:7777" {{- end }} {{- with dig "grpcproxy" "podDisruptionBudget" dict .Values }} podDisruptionBudget: @@ -707,10 +856,12 @@ rateLimiter: {{ .Values.global.nodeSelectors.controlplane.key }}: {{ .Values.global.nodeSelectors.controlplane.value }} {{- end }} replicaCount: {{ .Values.rateLimiter.replicaCount }} - {{- if .Values.global.observability.tracing.enabled }} env: + NVCF_API_URL: "http://api.{{ $nvcfNamespace }}.svc.cluster.local:9090" + OAUTH2_JWKS_URL: "http://{{ $openbaoFullname }}.{{ $vaultNamespace }}.svc.cluster.local:8200/v1/services/ratelimiter-api/jwt/jwks" + {{- if .Values.global.observability.tracing.enabled }} OTEL_EXPORTER_OTLP_ENDPOINT: "{{ .Values.global.observability.tracing.collectorProtocol }}://{{ .Values.global.observability.tracing.collectorEndpoint }}:{{ .Values.global.observability.tracing.collectorPort }}" - {{- end }} + {{- end }} {{- with dig "rateLimiter" "podDisruptionBudget" dict .Values }} podDisruptionBudget: {{- toYaml . | nindent 4 }} @@ -733,6 +884,9 @@ ess: {{- end }} env: + {{- if $controlPlaneID }} + SPRING_CASSANDRA_CONTACT_POINTS: "cassandra.{{ $cassandraNamespace }}.svc.cluster.local" + {{- end }} # Observability MANAGEMENT_TRACING_ENABLED: {{ .Values.global.observability.tracing.enabled | quote }} {{- if .Values.global.observability.tracing.enabled }} @@ -760,6 +914,11 @@ notary: {{- end }} env: + {{- if $controlPlaneID }} + SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER_URI: "http://api.{{ $nvcfNamespace }}.svc.cluster.local" + SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_JWK_SET_URI: "http://{{ $openbaoFullname }}.{{ $vaultNamespace }}.svc.cluster.local:8200/v1/services/nvcf-api/jwt/jwks" + NOTARY_ISSUER_URL: "http://notary.{{ $nvcfNamespace }}.svc.cluster.local:8080" + {{- end }} # Observability MANAGEMENT_TRACING_ENABLED: {{ .Values.global.observability.tracing.enabled | quote }} {{- if .Values.global.observability.tracing.enabled }} @@ -788,7 +947,18 @@ sis: # LLS/TURN HMAC rotation configuration lls: enabled: {{ dig "addons" "lls" "enabled" false .Values }} + {{- if $controlPlaneID }} + namespace: {{ $vaultNamespace }} + turn: + serviceAccountName: {{ dig "addons" "lls" "turn" "serviceAccountName" "turn" .Values | quote }} + serviceAccountNamespace: {{ dig "addons" "lls" "turn" "serviceAccountNamespace" (printf "%sgdn-streaming" $controlPlanePrefix) .Values | quote }} + {{- end }} hmacRotation: + {{- if $controlPlaneID }} + baoService: {{ printf "%s.%s.svc.cluster.local" $openbaoFullname $vaultNamespace }} + serviceAccountName: {{ printf "%s-initialize-cluster" $openbaoFullname }} + rootTokenSecretName: {{ printf "%s-root-token" $openbaoFullname }} + {{- end }} image: registry: {{ .Values.global.image.registry }} repository: {{ .Values.global.image.repository }}/nvcf-openbao-migrations @@ -797,6 +967,14 @@ sis: {{- end }} env: + {{- if $controlPlaneID }} + ICMS_NATS_NATS_URL: "nats://nats.{{ $natsNamespace }}.svc.cluster.local:4222" + SPRING_CASSANDRA_CONTACT_POINTS: "cassandra.{{ $cassandraNamespace }}.svc.cluster.local" + SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ADMIN_ISSUER_URI: "http://api.{{ $nvcfNamespace }}.svc.cluster.local" + SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ADMIN_JWK_SET_URI: "http://{{ $openbaoFullname }}.{{ $vaultNamespace }}.svc.cluster.local:8200/v1/services/nvcf-api/jwt/jwks" + SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER_URI: "http://api.{{ $sisNamespace }}.svc.cluster.local" + SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_JWK_SET_URI: "http://{{ $openbaoFullname }}.{{ $vaultNamespace }}.svc.cluster.local:8200/v1/services/sis-api/jwt/jwks" + {{- end }} # Observability MANAGEMENT_TRACING_ENABLED: {{ .Values.global.observability.tracing.enabled | quote }} MANAGEMENT_PROMETHEUS_METRICS_EXPORT_ENABLED: {{ .Values.global.observability.metrics.enabled | quote }} @@ -806,7 +984,7 @@ sis: {{- end }} adminIssuerProxy: - fullnameOverride: admin-token-issuer-proxy + fullnameOverride: {{ $adminIssuerFullname }} {{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml .Values.global.imagePullSecrets | nindent 4 }} @@ -814,6 +992,9 @@ adminIssuerProxy: image: registry: {{ .Values.global.image.registry }} repository: {{ .Values.global.image.repository }}/admin-token-issuer-proxy + config: + vaultAddr: "http://{{ $openbaoFullname }}.{{ $vaultNamespace }}.svc.cluster.local:8200" + serviceMetadataURL: "http://api-keys.{{ $apiKeysNamespace }}.svc.cluster.local:8080/v1/services" {{- with include "nvcf.nodeSelector" (dict "type" "controlplane" "selectors" .Values.global.nodeSelectors) }} {{- . | nindent 2 }} {{- end }} @@ -823,6 +1004,7 @@ adminIssuerProxy: gateway: enabled: true namespace: {{ .Values.ingress.gatewayApi.gateways.shared.namespace }} + routeNamespace: {{ $ingressNamespace }} gatewayRef: name: {{ required "ingress.gatewayApi.gateways.shared.name is required" .Values.ingress.gatewayApi.gateways.shared.name }} hostname: "api-keys.{{ .Values.global.domain }}" @@ -859,8 +1041,16 @@ stateMetrics: {{- $victoriaMetricsNamespace := dig "victoriaMetrics" "namespace" (dig "observability" "namespace" "monitoring" .Values) .Values }} {{- $bundledPromqlEndpoint := printf "http://vmsingle.%s.svc.cluster.local:8428" $victoriaMetricsNamespace }} {{- $promqlEndpoint := ternary $bundledPromqlEndpoint (dig "metricsBackend" "promqlEndpoint" "" .Values) (eq $metricsBackendMode "install") }} +{{- $functionCassandraContactPoints := dig "functionAutoscaler" "cassandra" "contactPoints" "cassandra.cassandra-system.svc.cluster.local" .Values }} +{{- if and $controlPlaneID (eq $functionCassandraContactPoints "cassandra.cassandra-system.svc.cluster.local") }} +{{- $functionCassandraContactPoints = printf "cassandra.%s.svc.cluster.local" $cassandraNamespace }} +{{- end }} +{{- $functionNvcfApiAddress := dig "functionAutoscaler" "nvcfApi" "grpcAddress" "http://api.nvcf.svc.cluster.local:9090" .Values }} +{{- if and $controlPlaneID (eq $functionNvcfApiAddress "http://api.nvcf.svc.cluster.local:9090") }} +{{- $functionNvcfApiAddress = printf "http://api.%s.svc.cluster.local:9090" $nvcfNamespace }} +{{- end }} functionautoscaler: - namespace: nvcf + namespace: {{ $nvcfNamespace }} fullnameOverride: function-autoscaler {{- if .Values.global.imagePullSecrets }} imagePullSecrets: @@ -876,9 +1066,9 @@ functionautoscaler: CONFIG: /etc/server/config/settings-local.yaml SECRETS_PATH: /vault/secrets/secrets.json REGION: {{ dig "functionAutoscaler" "region" "local" .Values | quote }} - CASSANDRA__CONTACT_POINTS: {{ dig "functionAutoscaler" "cassandra" "contactPoints" "cassandra.cassandra-system.svc.cluster.local" .Values | quote }} + CASSANDRA__CONTACT_POINTS: {{ $functionCassandraContactPoints | quote }} CASSANDRA__IS_DEVELOPMENT: {{ dig "functionAutoscaler" "cassandra" "isDevelopment" false .Values | quote }} - NVCF_API__NVCF_API_GRPC_ADDRESS: {{ dig "functionAutoscaler" "nvcfApi" "grpcAddress" "http://api.nvcf.svc.cluster.local:9090" .Values | quote }} + NVCF_API__NVCF_API_GRPC_ADDRESS: {{ $functionNvcfApiAddress | quote }} NVCF_API__DISABLE_AUTH: {{ dig "functionAutoscaler" "nvcfApi" "disableAuth" true .Values | quote }} NVCF_API__DRY_RUN: {{ dig "functionAutoscaler" "nvcfApi" "dryRun" false .Values | quote }} TIMESERIES_DB__TIMESERIES_DB_URL: {{ required "metricsBackend.promqlEndpoint is required for control and all profiles" $promqlEndpoint | quote }} @@ -917,6 +1107,11 @@ reval: securityContext: runAsNonRoot: true runAsUser: 65532 + serviceConfig: + auth: + jwkSetUrl: "http://{{ $openbaoFullname }}.{{ $vaultNamespace }}.svc.cluster.local:8200/v1/services/reval/jwt/jwks" + oidc: + introspectUrl: "http://api.{{ $sisNamespace }}.svc.cluster.local:8080/v1/nvca/tokens/introspect" {{- with include "nvcf.nodeSelector" (dict "type" "controlplane" "selectors" .Values.global.nodeSelectors) }} {{- . | nindent 2 }} {{- end }} @@ -951,10 +1146,12 @@ llmApiGateway: {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }} {{- . | nindent 2 }} {{- end }} - {{- if dig "addons" "llm" "gateway" "auth" "grpcInsecure" false .Values }} config: + requestRouterUrl: "http://llm-request-router-headless.{{ $nvcfNamespace }}.svc.cluster.local:8000" + nvcfGrpcAddr: "api.{{ $nvcfNamespace }}.svc.cluster.local:9090" + {{- if dig "addons" "llm" "gateway" "auth" "grpcInsecure" false .Values }} nvcfGrpcInsecure: true - {{- end }} + {{- end }} metrics: enabled: {{ dig "addons" "llm" "gateway" "metrics" "enabled" true .Values }} serviceMonitor: @@ -976,6 +1173,13 @@ llmApiGateway: {{- $llmWorkerRouteEnabled := dig "ingress" "gatewayApi" "routes" "llmWorker" "enabled" false .Values }} {{- $grpcTlsEnabled := dig "addons" "llm" "requestRouter" "grpcTls" "enabled" false .Values }} {{- $grpcTlsAllowInsecureHttp := dig "addons" "llm" "requestRouter" "grpcTls" "allowInsecureHttp" false .Values }} +{{- $grpcTlsSecretName := dig "addons" "llm" "requestRouter" "grpcTls" "secretName" "llm-request-router-grpc-tls" .Values | toString }} +{{- if and $controlPlaneID (eq $grpcTlsSecretName "llm-request-router-grpc-tls") }} +{{- $grpcTlsSecretName = printf "%sllm-request-router-grpc-tls" $controlPlanePrefix }} +{{- end }} +{{- if and $controlPlaneID $grpcTlsEnabled (not (hasPrefix $controlPlanePrefix $grpcTlsSecretName)) }} +{{- fail (printf "named control-plane gRPC TLS secret %q must start with %q" $grpcTlsSecretName $controlPlanePrefix) }} +{{- end }} {{- if and $llmEnabled $backendRouterEnabled (ne (empty $pylonGrpcDialAddress) (empty $pylonReverseTunnelDialAddress)) }} {{- fail "addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress and addons.llm.requestRouter.backendRouter.pylonReverseTunnelDialAddress must either both be set or both be omitted" }} {{- end }} @@ -1019,6 +1223,8 @@ llmRequestRouter: image: registry: {{ .Values.global.image.registry }} repository: {{ .Values.global.image.repository }}/stargate + auth: + workerAuthEndpoint: "http://api.{{ $nvcfNamespace }}.svc.cluster.local:9090" {{- /* Backend routing follows the LLM addon rather than being separately opted into. A worker holds one registration stream and one reverse tunnel per replica, and @@ -1052,12 +1258,12 @@ llmRequestRouter: enabled: {{ $grpcTlsEnabled }} allowInsecureHttp: {{ $grpcTlsAllowInsecureHttp }} mode: {{ dig "addons" "llm" "requestRouter" "grpcTls" "mode" "certManager" .Values | quote }} - secretName: {{ dig "addons" "llm" "requestRouter" "grpcTls" "secretName" "llm-request-router-grpc-tls" .Values | quote }} + secretName: {{ $grpcTlsSecretName | quote }} dnsNames: {{- dig "addons" "llm" "requestRouter" "grpcTls" "dnsNames" (list) .Values | toYaml | nindent 6 }} issuerRef: kind: {{ dig "addons" "llm" "requestRouter" "grpcTls" "issuerRef" "kind" "" .Values | default (dig "addons" "llm" "pki" "issuerKind" "ClusterIssuer" .Values) | quote }} - name: {{ dig "addons" "llm" "requestRouter" "grpcTls" "issuerRef" "name" "" .Values | default (dig "addons" "llm" "pki" "issuerName" "nvcf-openbao-pki" .Values) | quote }} + name: {{ dig "addons" "llm" "requestRouter" "grpcTls" "issuerRef" "name" "" .Values | default (dig "addons" "llm" "pki" "issuerName" (printf "%snvcf-openbao-pki" $controlPlanePrefix) .Values) | quote }} {{- with dig "addons" "llm" "requestRouter" "grpcTls" "issuerRef" "group" "" .Values }} group: {{ . | quote }} {{- end }} @@ -1140,7 +1346,10 @@ llmRequestRouter: keyPath: {{ dig "addons" "llm" "pki" "keyPath" "/etc/stargate/tls/tls.key" .Values | quote }} quicInsecure: false {{- else }} - {{- $secretName := dig "addons" "llm" "pki" "secretName" "stargate-quic-tls" .Values }} + {{- $secretName := dig "addons" "llm" "pki" "secretName" (printf "%sstargate-quic-tls" $controlPlanePrefix) .Values }} + {{- if and $controlPlaneID (not (hasPrefix $controlPlanePrefix ($secretName | toString))) }} + {{- fail (printf "named control-plane QUIC TLS secret %q must start with %q" $secretName $controlPlanePrefix) }} + {{- end }} {{- /* dig only falls back for a missing path, so explicit null, empty, and wrongly typed values reach these checks. Reject them instead of coercing them: a malformed management flag that silently resolves to false skips @@ -1148,7 +1357,7 @@ llmRequestRouter: below still renders a Certificate referencing the default issuer. Keep this contract identical to the dependency stage. */ -}} {{- $issuerKind := dig "addons" "llm" "pki" "issuerKind" "ClusterIssuer" .Values }} - {{- $issuerName := dig "addons" "llm" "pki" "issuerName" "nvcf-openbao-pki" .Values }} + {{- $issuerName := dig "addons" "llm" "pki" "issuerName" (printf "%snvcf-openbao-pki" $controlPlanePrefix) .Values }} {{- if not (kindIs "string" $issuerKind) }} {{- fail "addons.llm.pki.issuerKind must be the string \"ClusterIssuer\" or \"Issuer\"" }} {{- end }} @@ -1164,7 +1373,7 @@ llmRequestRouter: {{- if gt (len $issuerName) 253 }} {{- fail "addons.llm.pki.issuerName must be at most 253 characters" }} {{- end }} - {{- $managedIssuer := and (eq $issuerKind "ClusterIssuer") (eq $issuerName "nvcf-openbao-pki") }} + {{- $managedIssuer := and (eq $issuerKind "ClusterIssuer") (eq $issuerName (printf "%snvcf-openbao-pki" $controlPlanePrefix)) }} {{- $pkiValues := dig "addons" "llm" "pki" dict .Values }} {{- if kindIs "map" $pkiValues }} {{- $clusterIssuerValues := dig "clusterIssuer" dict $pkiValues }} @@ -1181,7 +1390,14 @@ llmRequestRouter: {{- if and $managedIssuer (ne $issuerKind "ClusterIssuer") }} {{- fail "addons.llm.pki.clusterIssuer management supports only issuerKind=ClusterIssuer" }} {{- end }} + {{- if and $controlPlaneID $managedIssuer (ne $issuerName (printf "%snvcf-openbao-pki" $controlPlanePrefix)) }} + {{- fail (printf "managed ClusterIssuer name %q must equal canonical name %q" $issuerName (printf "%snvcf-openbao-pki" $controlPlanePrefix)) }} + {{- end }} {{- $dnsNames := dig "addons" "llm" "pki" "dnsNames" (list) .Values }} + {{- $legacyManagedDnsNames := list "llm-request-router.nvcf.svc.cluster.local" "*.llm-request-router-headless.nvcf.svc.cluster.local" }} + {{- if and $controlPlaneID (deepEqual $dnsNames $legacyManagedDnsNames) }} + {{- $dnsNames = list (printf "llm-request-router.%s.svc.cluster.local" $nvcfNamespace) (printf "*.llm-request-router-headless.%s.svc.cluster.local" $nvcfNamespace) }} + {{- end }} {{- if eq (len $dnsNames) 0 }} {{- fail "addons.llm.pki.dnsNames must contain at least one DNS name when addons.llm.pki.enabled is true" }} {{- end }} @@ -1207,8 +1423,12 @@ llmRequestRouter: {{- $pkiImageTag := dig "addons" "llm" "pki" "image" "tag" (dig "openbao" "migrations" "image" "tag" "" .Values) .Values | default "0.16.2" }} pki: enabled: true - namespace: {{ dig "addons" "llm" "pki" "namespace" "vault-system" .Values | quote }} + namespace: {{ dig "addons" "llm" "pki" "namespace" $vaultNamespace .Values | quote }} allowedDomains: {{ $allowedDomains | quote }} + baoService: {{ dig "addons" "llm" "pki" "baoService" (printf "%s.%s.svc.cluster.local" $openbaoFullname $vaultNamespace) .Values | quote }} + serviceAccountName: {{ dig "addons" "llm" "pki" "serviceAccountName" (printf "%s-initialize-cluster" $openbaoFullname) .Values | quote }} + rootTokenSecretName: {{ dig "addons" "llm" "pki" "rootTokenSecretName" (printf "%s-root-token" $openbaoFullname) .Values | quote }} + sisServiceAccountNamespace: {{ $sisNamespace | quote }} image: registry: {{ dig "addons" "llm" "pki" "image" "registry" .Values.global.image.registry .Values | quote }} repository: {{ dig "addons" "llm" "pki" "image" "repository" (printf "%s/nvcf-openbao-migrations" .Values.global.image.repository) .Values | quote }} @@ -1242,8 +1462,8 @@ vanityGateway: pullPolicy: {{ dig "addons" "vanityGateway" "image" "pullPolicy" "IfNotPresent" .Values }} config: mappingPath: {{ dig "addons" "vanityGateway" "config" "mappingPath" "/etc/vanity-gateway/config/config.yaml" .Values | quote }} - nvcfApiEndpoint: {{ dig "addons" "vanityGateway" "config" "nvcfApiEndpoint" "http://invocation.nvcf.svc.cluster.local:8080" .Values | quote }} - llmGatewayEndpoint: {{ dig "addons" "vanityGateway" "config" "llmGatewayEndpoint" "http://llm-api-gateway.nvcf.svc.cluster.local:8080" .Values | quote }} + nvcfApiEndpoint: {{ dig "addons" "vanityGateway" "config" "nvcfApiEndpoint" "" .Values | default (printf "http://invocation.%s.svc.cluster.local:8080" $nvcfNamespace) | quote }} + llmGatewayEndpoint: {{ dig "addons" "vanityGateway" "config" "llmGatewayEndpoint" "" .Values | default (printf "http://llm-api-gateway.%s.svc.cluster.local:8080" $nvcfNamespace) | quote }} {{- if .Values.global.observability.tracing.enabled }} otelExporterOtlpEndpoint: "{{ .Values.global.observability.tracing.collectorProtocol }}://{{ .Values.global.observability.tracing.collectorEndpoint }}:{{ .Values.global.observability.tracing.collectorPort }}" {{- else }} @@ -1473,6 +1693,7 @@ nvcfGatewayRoutes: {{- $natsRouteEnabled := dig "ingress" "gatewayApi" "routes" "nats" "enabled" false .Values }} {{- $essRouteEnabled := dig "ingress" "gatewayApi" "routes" "ess" "enabled" false .Values }} domain: "{{ .Values.global.domain }}" + routeNamespace: {{ $ingressNamespace }} gateways: shared: name: {{ required "ingress.gatewayApi.gateways.shared.name is required" .Values.ingress.gatewayApi.gateways.shared.name }} @@ -1488,29 +1709,41 @@ nvcfGatewayRoutes: {{- end }} {{- if $llmWorkerRouteEnabled }} llmGrpc: - name: {{ required "ingress.gatewayApi.gateways.llmGrpc.name is required when ingress.gatewayApi.routes.llmWorker.enabled is true" .Values.ingress.gatewayApi.gateways.llmGrpc.name }} - namespace: {{ required "ingress.gatewayApi.gateways.llmGrpc.namespace is required when ingress.gatewayApi.routes.llmWorker.enabled is true" .Values.ingress.gatewayApi.gateways.llmGrpc.namespace }} + name: {{ dig "ingress" "gatewayApi" "gateways" "llmGrpc" "name" "" .Values | default (printf "%sllm-grpc-gw" $controlPlanePrefix) | quote }} + namespace: {{ dig "ingress" "gatewayApi" "gateways" "llmGrpc" "namespace" "" .Values | default .Values.ingress.gatewayApi.gateways.shared.namespace | quote }} listenerName: {{ dig "ingress" "gatewayApi" "gateways" "llmGrpc" "listenerName" "llm-grpc" .Values }} llmQuic: - name: {{ required "ingress.gatewayApi.gateways.llmQuic.name is required when ingress.gatewayApi.routes.llmWorker.enabled is true" .Values.ingress.gatewayApi.gateways.llmQuic.name }} - namespace: {{ required "ingress.gatewayApi.gateways.llmQuic.namespace is required when ingress.gatewayApi.routes.llmWorker.enabled is true" .Values.ingress.gatewayApi.gateways.llmQuic.namespace }} + name: {{ dig "ingress" "gatewayApi" "gateways" "llmQuic" "name" "" .Values | default (printf "%sllm-quic-gw" $controlPlanePrefix) | quote }} + namespace: {{ dig "ingress" "gatewayApi" "gateways" "llmQuic" "namespace" "" .Values | default .Values.ingress.gatewayApi.gateways.shared.namespace | quote }} listenerName: {{ dig "ingress" "gatewayApi" "gateways" "llmQuic" "listenerName" "llm-quic" .Values }} {{- end }} routes: nvcfApi: + name: {{ printf "%snvcf-api" $controlPlanePrefix }} + backend: + namespace: {{ $nvcfNamespace }} routeAnnotations: {{ dig "ingress" "gatewayApi" "routes" "nvcfApi" "routeAnnotations" dict .Values | toYaml | nindent 8 | trim }} grpc: enabled: {{ dig "ingress" "gatewayApi" "routes" "nvcfApi" "grpc" "enabled" false .Values }} + name: {{ printf "%snvcf-api-grpc" $controlPlanePrefix }} + backend: + namespace: {{ $nvcfNamespace }} {{- with $nvcfApiGrpcRouteHostnames }} hostnames: {{- toYaml . | nindent 10 }} {{- end }} nvctApi: + name: {{ printf "%snvct-api" $controlPlanePrefix }} + backend: + namespace: {{ $nvcfNamespace }} routeAnnotations: {{ dig "ingress" "gatewayApi" "routes" "nvctApi" "routeAnnotations" dict .Values | toYaml | nindent 8 | trim }} grpc: enabled: {{ dig "ingress" "gatewayApi" "routes" "nvctApi" "grpc" "enabled" false .Values }} + name: {{ printf "%snvct-api-grpc" $controlPlanePrefix }} + backend: + namespace: {{ $nvcfNamespace }} {{- with $nvctApiGrpcRouteHostnames }} hostnames: {{- toYaml . | nindent 10 }} @@ -1524,6 +1757,9 @@ nvcfGatewayRoutes: # false to drop the route (for example, to keep api-keys off the public edge in # a split/multi-cluster deployment). apiKeys: + name: {{ printf "%sapi-keys" $controlPlanePrefix }} + backend: + namespace: {{ $apiKeysNamespace }} {{- $apiKeysRoute := dig "ingress" "gatewayApi" "routes" "apiKeys" dict .Values }} {{- if hasKey $apiKeysRoute "enabled" }} enabled: {{ $apiKeysRoute.enabled }} @@ -1531,6 +1767,9 @@ nvcfGatewayRoutes: routeAnnotations: {{ dig "routeAnnotations" dict $apiKeysRoute | toYaml | nindent 8 | trim }} sis: + name: {{ printf "%ssis" $controlPlanePrefix }} + backend: + namespace: {{ $sisNamespace }} {{- $sisRoute := dig "ingress" "gatewayApi" "routes" "sis" dict .Values }} {{- if hasKey $sisRoute "enabled" }} enabled: {{ $sisRoute.enabled }} @@ -1538,6 +1777,9 @@ nvcfGatewayRoutes: routeAnnotations: {{ dig "routeAnnotations" dict $sisRoute | toYaml | nindent 8 | trim }} reval: + name: {{ printf "%sreval" $controlPlanePrefix }} + backend: + namespace: {{ $nvcfNamespace }} {{- $revalRoute := dig "ingress" "gatewayApi" "routes" "reval" dict .Values }} {{- if hasKey $revalRoute "enabled" }} enabled: {{ $revalRoute.enabled }} @@ -1545,23 +1787,33 @@ nvcfGatewayRoutes: routeAnnotations: {{ dig "routeAnnotations" dict $revalRoute | toYaml | nindent 8 | trim }} invocation: + name: {{ printf "%sinvocation-service" $controlPlanePrefix }} + backend: + namespace: {{ $nvcfNamespace }} routeAnnotations: {{ dig "ingress" "gatewayApi" "routes" "invocation" "routeAnnotations" dict .Values | toYaml | nindent 8 | trim }} + llmApiGateway: + enabled: {{ dig "addons" "llm" "enabled" false .Values }} + name: {{ printf "%sllm-api-gateway" $controlPlanePrefix }} + backend: + namespace: {{ $nvcfNamespace }} llmInvocation: enabled: {{ dig "addons" "llm" "enabled" false .Values }} + name: {{ printf "%sllm-invocation" $controlPlanePrefix }} backend: name: llm-api-gateway - namespace: nvcf + namespace: {{ $nvcfNamespace }} port: 8080 routeAnnotations: {{ dig "ingress" "gatewayApi" "routes" "llmInvocation" "routeAnnotations" dict .Values | toYaml | nindent 8 | trim }} vanityGateway: enabled: {{ dig "addons" "vanityGateway" "enabled" false .Values }} + name: {{ printf "%svanity-gateway" $controlPlanePrefix }} hostnames: {{ dig "ingress" "gatewayApi" "routes" "vanityGateway" "hostnames" (list) .Values | default (list (printf "vanity.%s" .Values.global.domain)) | toYaml | nindent 8 | trim }} backend: name: {{ dig "ingress" "gatewayApi" "routes" "vanityGateway" "backend" "name" "vanity-gateway" .Values }} - namespace: {{ dig "ingress" "gatewayApi" "routes" "vanityGateway" "backend" "namespace" "nvcf" .Values }} + namespace: {{ dig "ingress" "gatewayApi" "routes" "vanityGateway" "backend" "namespace" $nvcfNamespace .Values }} port: {{ dig "ingress" "gatewayApi" "routes" "vanityGateway" "backend" "port" 8080 .Values }} routeAnnotations: {{ dig "ingress" "gatewayApi" "routes" "vanityGateway" "routeAnnotations" dict .Values | toYaml | nindent 8 | trim }} @@ -1570,30 +1822,46 @@ nvcfGatewayRoutes: # nvcf-ui namespace, nvcf-ui.). nvcfUi: enabled: {{ dig "addons" "nvcfUi" "enabled" false .Values }} + name: {{ printf "%snvcf-ui" $controlPlanePrefix }} + backend: + namespace: {{ $nvcfUiNamespace }} routeAnnotations: {{ dig "ingress" "gatewayApi" "routes" "nvcfUi" "routeAnnotations" dict .Values | toYaml | nindent 8 | trim }} eventLedger: enabled: {{ dig "addons" "eventLedger" "enabled" false .Values }} + name: {{ printf "%sevent-ledger" $controlPlanePrefix }} + backend: + namespace: {{ $nvcfNamespace }} routeAnnotations: {{ dig "ingress" "gatewayApi" "routes" "eventLedger" "routeAnnotations" dict .Values | toYaml | nindent 8 | trim }} grpc: + name: {{ printf "%sgrpc" $controlPlanePrefix }} + backend: + namespace: {{ $nvcfNamespace }} routeAnnotations: {{ dig "ingress" "gatewayApi" "routes" "grpc" "routeAnnotations" dict .Values | toYaml | nindent 8 | trim }} grpcWorker: enabled: {{ dig "ingress" "gatewayApi" "routes" "grpcWorker" "enabled" false .Values }} + name: {{ printf "%sgrpc-worker" $controlPlanePrefix }} listenerName: {{ dig "ingress" "gatewayApi" "routes" "grpcWorker" "listenerName" "worker-tcp" .Values }} + backend: + namespace: {{ $nvcfNamespace }} routeAnnotations: {{ dig "ingress" "gatewayApi" "routes" "grpcWorker" "routeAnnotations" dict .Values | toYaml | nindent 8 | trim }} nats: enabled: {{ $natsRouteEnabled }} + name: {{ printf "%snats" $controlPlanePrefix }} + backend: + namespace: {{ $natsNamespace }} routeAnnotations: {{ dig "ingress" "gatewayApi" "routes" "nats" "routeAnnotations" dict .Values | toYaml | nindent 8 | trim }} llmWorker: enabled: {{ $llmWorkerRouteEnabled }} + name: {{ printf "%sllm-worker" $controlPlanePrefix }} {{- if $llmWorkerRouteEnabled }} backend: name: {{ dig "ingress" "gatewayApi" "routes" "llmWorker" "backend" "name" "llm-request-router-backend-router" .Values }} - namespace: {{ required "ingress.gatewayApi.routes.llmWorker.backend.namespace is required when ingress.gatewayApi.routes.llmWorker.enabled is true" .Values.ingress.gatewayApi.routes.llmWorker.backend.namespace }} + namespace: {{ dig "ingress" "gatewayApi" "routes" "llmWorker" "backend" "namespace" "" .Values | default $nvcfNamespace }} grpcPort: {{ dig "ingress" "gatewayApi" "routes" "llmWorker" "backend" "grpcPort" 50071 .Values }} quicPort: {{ dig "ingress" "gatewayApi" "routes" "llmWorker" "backend" "quicPort" 50072 .Values }} {{- end }} @@ -1603,7 +1871,12 @@ nvcfGatewayRoutes: # defaults); only the enable flag and annotations are environment-driven. ess: enabled: {{ $essRouteEnabled }} + name: {{ printf "%sess" $controlPlanePrefix }} + backend: + namespace: {{ $essNamespace }} routeAnnotations: {{ dig "ingress" "gatewayApi" "routes" "ess" "routeAnnotations" dict .Values | toYaml | nindent 8 | trim }} podMonitors: enabled: {{ .Values.global.observability.metrics.enabled }} + sharedName: {{ printf "%senvoy-gateway-proxy-shared" $controlPlanePrefix }} + grpcName: {{ printf "%senvoy-gateway-proxy-grpc" $controlPlanePrefix }} diff --git a/deploy/stacks/self-managed/helmfile.d/00-observability-infrastructure.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/00-observability-infrastructure.yaml.gotmpl index 9e50d3c41..ad8d917aa 100644 --- a/deploy/stacks/self-managed/helmfile.d/00-observability-infrastructure.yaml.gotmpl +++ b/deploy/stacks/self-managed/helmfile.d/00-observability-infrastructure.yaml.gotmpl @@ -6,12 +6,19 @@ environments: --- +{{- $controlPlaneID := dig "global" "controlPlane" "id" "" .Values | toString }} +{{- $sharedInfrastructure := dig "global" "controlPlane" "sharedInfrastructure" "bundled" .Values | toString }} +{{- if and $controlPlaneID (ne $sharedInfrastructure "external") }} +{{- fail "global.controlPlane.sharedInfrastructure must be external for a named control plane" }} +{{- end }} {{- $observability := dig "observability" (dict) .Values }} {{- $profile := dig "profile" "disabled" $observability }} {{- if not (has $profile (list "disabled" "control" "compute" "all")) }} {{- fail (printf "observability.profile must be disabled, control, compute, or all, got %q" $profile) }} {{- end }} -{{- if ne $profile "disabled" }} +{{- /* Named planes consume shared observability as an external prerequisite. + Excluding it from this Helmfile keeps plane destroy from uninstalling it. */}} +{{- if and (ne $profile "disabled") (ne $sharedInfrastructure "external") }} helmfiles: - path: ../../observability/helmfile.d/01-observability.yaml.gotmpl selectorsInherited: true diff --git a/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl index d7e298451..9f771c763 100644 --- a/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl +++ b/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl @@ -6,11 +6,31 @@ environments: --- +{{- $controlPlaneID := dig "global" "controlPlane" "id" "" .Values | toString -}} +{{- $sharedInfrastructure := dig "global" "controlPlane" "sharedInfrastructure" "bundled" .Values | toString -}} +{{- if and $controlPlaneID (or (gt (len $controlPlaneID) 20) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $controlPlaneID)) (eq $controlPlaneID "default")) -}} +{{- fail "global.controlPlane.id must be a DNS-1123 label of at most 20 characters and must not be 'default'" -}} +{{- end -}} +{{- if and $controlPlaneID (ne $sharedInfrastructure "external") -}} +{{- fail "global.controlPlane.sharedInfrastructure must be external for a named control plane" -}} +{{- end -}} +{{- if and (eq $sharedInfrastructure "external") (dig "certManager" "enabled" false .Values) -}} +{{- fail "certManager.enabled must be false when global.controlPlane.sharedInfrastructure is external" -}} +{{- end -}} +{{- $controlPlanePrefix := "" -}} +{{- if $controlPlaneID -}} +{{- $controlPlanePrefix = printf "%s-" $controlPlaneID -}} +{{- end -}} +{{- $natsNamespace := printf "%snats-system" $controlPlanePrefix -}} +{{- $vaultNamespace := printf "%svault-system" $controlPlanePrefix -}} +{{- $cassandraNamespace := printf "%scassandra-system" $controlPlanePrefix -}} +{{- $openbaoFullname := printf "%sopenbao-server" $controlPlanePrefix -}} + {{- $llmEnabled := dig "addons" "llm" "enabled" false .Values }} {{- $pkiEnabled := dig "addons" "llm" "pki" "enabled" false .Values }} {{- $llmPkiActive := and $llmEnabled $pkiEnabled }} {{- $issuerKind := dig "addons" "llm" "pki" "issuerKind" "ClusterIssuer" .Values }} -{{- $issuerName := dig "addons" "llm" "pki" "issuerName" "nvcf-openbao-pki" .Values }} +{{- $issuerName := dig "addons" "llm" "pki" "issuerName" (printf "%snvcf-openbao-pki" $controlPlanePrefix) .Values }} {{- $manageIssuer := false }} {{- /* existingSecret hands issuance to the operator, so the stack must not create an issuer or require OpenBao for one. Validate the mode here as @@ -46,7 +66,7 @@ environments: {{- if gt (len $issuerName) 253 }} {{- fail "addons.llm.pki.issuerName must be at most 253 characters" }} {{- end }} -{{- $manageIssuer = and (eq $issuerKind "ClusterIssuer") (eq $issuerName "nvcf-openbao-pki") }} +{{- $manageIssuer = and (eq $issuerKind "ClusterIssuer") (eq $issuerName (printf "%snvcf-openbao-pki" $controlPlanePrefix)) }} {{- $pkiValues := dig "addons" "llm" "pki" dict .Values }} {{- if kindIs "map" $pkiValues }} {{- $clusterIssuerValues := dig "clusterIssuer" dict $pkiValues }} @@ -65,6 +85,9 @@ environments: {{- if and $managedIssuer (ne $issuerKind "ClusterIssuer") }} {{- fail "addons.llm.pki.clusterIssuer management supports only issuerKind=ClusterIssuer" }} {{- end }} +{{- if and $controlPlaneID $managedIssuer (ne $issuerName (printf "%snvcf-openbao-pki" $controlPlanePrefix)) }} +{{- fail (printf "managed ClusterIssuer name %q must equal canonical name %q" $issuerName (printf "%snvcf-openbao-pki" $controlPlanePrefix)) }} +{{- end }} {{- if and $managedIssuer (not (dig "openbao" "enabled" true .Values)) }} {{- fail "openbao.enabled must be true when addons.llm.pki.clusterIssuer management is enabled" }} {{- end }} @@ -110,7 +133,7 @@ releases: - name: nats version: 0.8.1 condition: nats.enabled # From defaults.yaml or env overrides - namespace: nats-system + namespace: {{ $natsNamespace }} <<: *dependency # Inherits base values from the dependency template - name: cert-manager @@ -122,7 +145,7 @@ releases: - name: openbao-server # this name MUST not change version: 0.32.1 condition: openbao.enabled # From defaults.yaml or env overrides - namespace: vault-system + namespace: {{ $vaultNamespace }} <<: *dependency # Inherits base values from the dependency template # Secret values own migrations.env. Apply the generated LLM addition last # so it preserves that list while adding the addon migration switch. @@ -131,10 +154,14 @@ releases: - ../secrets/{{ requiredEnv "HELMFILE_ENV" }}-secrets.yaml - ../openbao-migrations-llm-env.yaml.gotmpl needs: - - nats-system/nats + - {{ $natsNamespace }}/nats {{- if $managedIssuer }} +{{- if $controlPlaneID }} + - name: {{ printf "%snvcf-pki" $controlPlanePrefix }} +{{- else }} - name: nvcf-pki +{{- end }} chart: nvcf/helm-nvcf-pki version: 0.1.0 namespace: cert-manager @@ -142,20 +169,23 @@ releases: - clusterIssuer: enabled: true name: {{ $issuerName | quote }} - server: "http://openbao-server.vault-system.svc.cluster.local:8200" +{{- if $controlPlaneID }} + controlPlaneID: {{ $controlPlaneID | quote }} +{{- end }} + server: {{ printf "http://%s.%s.svc.cluster.local:8200" $openbaoFullname $vaultNamespace | quote }} path: "services/all/pki/nvcf-service-issuing/sign/nvcf-service-server" auth: mountPath: "/v1/auth/jwt" role: "cert-manager" serviceAccount: name: "cert-manager" - audience: "http://openbao-server.vault-system.svc.cluster.local:8200" + audience: {{ printf "http://%s.%s.svc.cluster.local:8200" $openbaoFullname $vaultNamespace | quote }} wait: true waitForJobs: true labels: release-group: dependencies needs: - - vault-system/openbao-server + - {{ $vaultNamespace }}/openbao-server {{- if dig "certManager" "enabled" true .Values }} - cert-manager/cert-manager {{- end }} @@ -164,5 +194,5 @@ releases: - name: cassandra version: 0.20.2 condition: cassandra.enabled # From defaults.yaml or env overrides - namespace: cassandra-system + namespace: {{ $cassandraNamespace }} <<: *dependency # Inherits base values from the dependency template diff --git a/deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl index 3a0f5a37e..e4345b76e 100644 --- a/deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl +++ b/deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl @@ -6,6 +6,56 @@ environments: --- +{{- $controlPlaneID := dig "global" "controlPlane" "id" "" .Values | toString -}} +{{- $sharedInfrastructure := dig "global" "controlPlane" "sharedInfrastructure" "bundled" .Values | toString -}} +{{- if and $controlPlaneID (or (gt (len $controlPlaneID) 20) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $controlPlaneID)) (eq $controlPlaneID "default")) -}} +{{- fail "global.controlPlane.id must be a DNS-1123 label of at most 20 characters and must not be 'default'" -}} +{{- end -}} +{{- if and $controlPlaneID (ne $sharedInfrastructure "external") -}} +{{- fail "global.controlPlane.sharedInfrastructure must be external for a named control plane" -}} +{{- end -}} +{{- if and (eq $sharedInfrastructure "external") (dig "certManager" "enabled" false .Values) -}} +{{- fail "certManager.enabled must be false when global.controlPlane.sharedInfrastructure is external" -}} +{{- end -}} +{{- $gatewayEnabled := dig "ingress" "gatewayApi" "enabled" false .Values -}} +{{- if and $controlPlaneID $gatewayEnabled (eq (dig "global" "domain" "localhost" .Values | toString) "localhost") -}} +{{- fail "global.domain must be unique and non-localhost for a named control plane with Gateway API enabled" -}} +{{- end -}} +{{- if and $controlPlaneID (dig "addons" "nvcfUi" "enabled" false .Values) -}} +{{- fail "addons.nvcfUi.enabled must be false for a named control plane because the pinned chart is not namespace-isolated" -}} +{{- end -}} +{{- $controlPlanePrefix := "" -}} +{{- if $controlPlaneID -}} +{{- $controlPlanePrefix = printf "%s-" $controlPlaneID -}} +{{- end -}} +{{- $apiKeysNamespace := printf "%sapi-keys" $controlPlanePrefix -}} +{{- $sisNamespace := printf "%ssis" $controlPlanePrefix -}} +{{- $nvcfNamespace := printf "%snvcf" $controlPlanePrefix -}} +{{- $nvcfUiNamespace := printf "%snvcf-ui" $controlPlanePrefix -}} +{{- $essNamespace := printf "%sess" $controlPlanePrefix -}} +{{- $natsNamespace := printf "%snats-system" $controlPlanePrefix -}} +{{- $ingressNamespace := .Values.ingress.gatewayApi.controllerNamespace -}} +{{- if $controlPlaneID -}} +{{- $ingressNamespace = printf "%singress" $controlPlanePrefix -}} +{{- end -}} +{{- $ingressRelease := printf "%singress" $controlPlanePrefix -}} +{{- if and $controlPlaneID $gatewayEnabled -}} +{{- range $gatewayName := list (dig "ingress" "gatewayApi" "gateways" "shared" "name" "" $.Values | toString) (dig "ingress" "gatewayApi" "gateways" "grpc" "name" "" $.Values | toString) (dig "ingress" "gatewayApi" "gateways" "nats" "name" "" $.Values | toString) -}} +{{- if not (hasPrefix $controlPlanePrefix $gatewayName) -}} +{{- fail (printf "named control-plane Gateway %q must start with %q" $gatewayName $controlPlanePrefix) -}} +{{- end -}} +{{- end -}} +{{- if and $controlPlaneID (dig "ingress" "gatewayApi" "routes" "llmWorker" "enabled" false .Values) -}} +{{- $llmGrpcGatewayName := dig "ingress" "gatewayApi" "gateways" "llmGrpc" "name" "" .Values | default (printf "%sllm-grpc-gw" $controlPlanePrefix) | toString -}} +{{- $llmQuicGatewayName := dig "ingress" "gatewayApi" "gateways" "llmQuic" "name" "" .Values | default (printf "%sllm-quic-gw" $controlPlanePrefix) | toString -}} +{{- range $gatewayName := list $llmGrpcGatewayName $llmQuicGatewayName -}} +{{- if not (hasPrefix $controlPlanePrefix $gatewayName) -}} +{{- fail (printf "named control-plane Gateway %q must start with %q" $gatewayName $controlPlanePrefix) -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} + repositories: - name: nvcf {{- if dig "global" "helm" "sources" "url" "" .Values }} @@ -45,75 +95,75 @@ releases: # --- NVCF Services --- - name: api-keys version: 1.6.0 - namespace: api-keys + namespace: {{ $apiKeysNamespace }} inherit: - template: service - name: sis version: 2.0.0 - namespace: sis + namespace: {{ $sisNamespace }} inherit: - template: service - name: api version: 1.25.1 - namespace: nvcf + namespace: {{ $nvcfNamespace }} inherit: - template: service needs: # not needed to start, but needed for bootstrapping account which is done with a script in the api helm chart - - ess/ess-api + - {{ $essNamespace }}/ess-api - name: nvct-api version: 1.5.2 - namespace: nvcf + namespace: {{ $nvcfNamespace }} inherit: - template: service - name: invocation-service version: 1.6.0 - namespace: nvcf + namespace: {{ $nvcfNamespace }} inherit: - template: service needs: - - nvcf/api + - {{ $nvcfNamespace }}/api - name: grpc-proxy version: 1.7.1 - namespace: nvcf + namespace: {{ $nvcfNamespace }} inherit: - template: service needs: - - nvcf/api + - {{ $nvcfNamespace }}/api - name: ratelimiter chart: nvcf/helm-nvcf-rate-limiter version: 1.1.0 - namespace: nvcf + namespace: {{ $nvcfNamespace }} condition: rateLimiter.enabled values: - ../global.yaml.gotmpl labels: release-group: services needs: - - nvcf/api + - {{ $nvcfNamespace }}/api - name: ess-api version: 1.7.2 - namespace: ess + namespace: {{ $essNamespace }} inherit: - template: service - name: notary-service version: 1.4.2 - namespace: nvcf + namespace: {{ $nvcfNamespace }} inherit: - template: service - name: admin-issuer-proxy chart: nvcf/helm-admin-token-issuer-proxy version: 1.4.3 - namespace: api-keys + namespace: {{ $apiKeysNamespace }} values: - ../global.yaml.gotmpl # Finding #22 (E2E 2026-04-20): admin-issuer-proxy was not installed by @@ -145,7 +195,7 @@ releases: - name: reval chart: nvcf/helm-reval version: 1.3.8 - namespace: nvcf + namespace: {{ $nvcfNamespace }} values: - ../global.yaml.gotmpl labels: @@ -154,7 +204,7 @@ releases: - name: nats-auth-callout-service chart: nvcf/helm-nvcf-nats-auth-callout-service version: 1.1.3 - namespace: nats-system + namespace: {{ $natsNamespace }} values: - ../global.yaml.gotmpl {{- if .Values.global.imagePullSecrets }} @@ -170,42 +220,42 @@ releases: {{- if not $llmRequestRouterChartPath }} version: 1.12.2 {{- end }} - namespace: nvcf + namespace: {{ $nvcfNamespace }} condition: addons.llm.enabled values: - ../global.yaml.gotmpl needs: - - nvcf/api + - {{ $nvcfNamespace }}/api labels: release-group: services - name: llm-api-gateway chart: nvcf/helm-nvcf-llm-api-gateway version: 1.4.2 - namespace: nvcf + namespace: {{ $nvcfNamespace }} condition: addons.llm.enabled values: - ../global.yaml.gotmpl needs: - - nvcf/llm-request-router + - {{ $nvcfNamespace }}/llm-request-router labels: release-group: services - name: vanity-gateway version: 0.4.0 - namespace: nvcf + namespace: {{ $nvcfNamespace }} condition: addons.vanityGateway.enabled inherit: - template: service needs: - - nvcf/invocation-service + - {{ $nvcfNamespace }}/invocation-service labels: release-group: services - name: nvcf-ui chart: nvcf/helm-nvcf-ui version: 1.1.2 - namespace: nvcf-ui + namespace: {{ $nvcfUiNamespace }} condition: addons.nvcfUi.enabled values: - ../global.yaml.gotmpl @@ -213,19 +263,19 @@ releases: release-group: services # --- Gateway API Ingress --- - - name: ingress + - name: {{ $ingressRelease }} {{- $gatewayRoutesChartPath := dig "ingress" "gatewayApi" "chartPath" "" .Values }} chart: {{ $gatewayRoutesChartPath | default "nvcf/nvcf-gateway-routes" | quote }} {{- if not $gatewayRoutesChartPath }} version: 1.17.0 {{- end }} needs: - - nvcf/notary-service - - api-keys/api-keys + - {{ $nvcfNamespace }}/notary-service + - {{ $apiKeysNamespace }}/api-keys {{- if dig "addons" "nvcfUi" "enabled" false .Values }} - - nvcf-ui/nvcf-ui + - {{ $nvcfUiNamespace }}/nvcf-ui {{- end }} - namespace: {{ .Values.ingress.gatewayApi.controllerNamespace }} + namespace: {{ $ingressNamespace }} condition: ingress.gatewayApi.enabled values: - ../global.yaml.gotmpl diff --git a/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl index 3e3fb2974..87fe6ece7 100644 --- a/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl +++ b/deploy/stacks/self-managed/helmfile.d/03-observability.yaml.gotmpl @@ -6,6 +6,23 @@ environments: --- +{{- $controlPlaneID := dig "global" "controlPlane" "id" "" .Values | toString -}} +{{- $sharedInfrastructure := dig "global" "controlPlane" "sharedInfrastructure" "bundled" .Values | toString -}} +{{- if and $controlPlaneID (or (gt (len $controlPlaneID) 20) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $controlPlaneID)) (eq $controlPlaneID "default")) -}} +{{- fail "global.controlPlane.id must be a DNS-1123 label of at most 20 characters and must not be 'default'" -}} +{{- end -}} +{{- if and $controlPlaneID (ne $sharedInfrastructure "external") -}} +{{- fail "global.controlPlane.sharedInfrastructure must be external for a named control plane" -}} +{{- end -}} +{{- if and (eq $sharedInfrastructure "external") (dig "certManager" "enabled" false .Values) -}} +{{- fail "certManager.enabled must be false when global.controlPlane.sharedInfrastructure is external" -}} +{{- end -}} +{{- $controlPlanePrefix := "" -}} +{{- if $controlPlaneID -}} +{{- $controlPlanePrefix = printf "%s-" $controlPlaneID -}} +{{- end -}} +{{- $nvcfNamespace := printf "%snvcf" $controlPlanePrefix -}} + {{- $observabilityProfile := dig "observability" "profile" "disabled" .Values }} {{- if not (has $observabilityProfile (list "disabled" "control" "compute" "all")) }} {{- fail (printf "observability.profile must be disabled, control, compute, or all, got %q" $observabilityProfile) }} @@ -59,7 +76,7 @@ releases: # --- Observability Services --- - name: state-metrics version: 1.0.2 - namespace: nvcf + namespace: {{ $nvcfNamespace }} condition: stateMetrics.enabled # From global.yaml.gotmpl or env overrides inherit: - template: service @@ -69,9 +86,9 @@ releases: {{- if $functionAutoscalerEnabled }} - name: function-autoscaler version: 0.2.1 - namespace: nvcf + namespace: {{ $nvcfNamespace }} inherit: - template: functionAutoscaler needs: - - nvcf/state-metrics + - {{ $nvcfNamespace }}/state-metrics {{- end }} diff --git a/deploy/stacks/self-managed/scripts/control-plane-clusterissuers.sh b/deploy/stacks/self-managed/scripts/control-plane-clusterissuers.sh new file mode 100755 index 000000000..9da5837a3 --- /dev/null +++ b/deploy/stacks/self-managed/scripts/control-plane-clusterissuers.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [[ "$#" -ne 2 || "$1" != cleanup || -z "$2" ]]; then + echo "Usage: $0 cleanup " >&2 + exit 2 +fi + +control_plane_id="$2" +owner_label="nvcf.nvidia.com/control-plane-id" +owner_jsonpath='{.metadata.labels.nvcf\.nvidia\.com/control-plane-id}' + +# Helm retains managed ClusterIssuers, including prior names left by an +# upgrade. Select every issuer explicitly owned by this plane, then re-check +# both the label and name immediately before deleting to fail closed on stale +# list results or ownership changes. +resources="$(kubectl get clusterissuers \ + -l "${owner_label}=${control_plane_id}" \ + -o name)" +while IFS= read -r resource; do + [[ -n "$resource" ]] || continue + issuer="${resource##*/}" + if [[ "$issuer" != "${control_plane_id}-"* ]]; then + echo "Error: refusing to delete managed ClusterIssuer outside control plane ${control_plane_id}: ${issuer}" >&2 + exit 1 + fi + + owner="$(kubectl get clusterissuer "$issuer" -o "jsonpath=${owner_jsonpath}")" + if [[ "$owner" != "$control_plane_id" ]]; then + echo "Error: refusing to delete ClusterIssuer ${issuer}: expected owner ${control_plane_id}, found ${owner:-unset}." >&2 + exit 1 + fi + kubectl delete clusterissuer "$issuer" --wait=true +done <<<"$resources" diff --git a/deploy/stacks/self-managed/scripts/control-plane-namespaces.sh b/deploy/stacks/self-managed/scripts/control-plane-namespaces.sh new file mode 100755 index 000000000..cd164585d --- /dev/null +++ b/deploy/stacks/self-managed/scripts/control-plane-namespaces.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +usage() { + echo "Usage: $0 ..." >&2 + exit 2 +} + +[[ "$#" -ge 3 ]] || usage +action="$1" +control_plane_id="$2" +shift 2 + +if [[ "$action" != prepare && "$action" != verify && "$action" != cleanup ]]; then + usage +fi + +owner_label="nvcf.nvidia.com/control-plane-id" +owner_jsonpath='{.metadata.labels.nvcf\.nvidia\.com/control-plane-id}' + +for namespace in "$@"; do + if [[ "$namespace" != "${control_plane_id}-"* ]]; then + echo "Error: refusing to manage namespace outside control plane ${control_plane_id}: ${namespace}" >&2 + exit 1 + fi + + if [[ "$action" == prepare || "$action" == verify ]]; then + if kubectl get namespace "$namespace" >/dev/null 2>&1; then + owner="$(kubectl get namespace "$namespace" -o "jsonpath=${owner_jsonpath}")" + if [[ "$owner" != "$control_plane_id" ]]; then + echo "Error: namespace $namespace is not owned by control plane $control_plane_id (owner=${owner:-unset})." >&2 + exit 1 + fi + elif [[ "$action" == prepare ]]; then + kubectl create namespace "$namespace" + kubectl label namespace "$namespace" "${owner_label}=${control_plane_id}" + fi + continue + fi + + if ! kubectl get namespace "$namespace" >/dev/null 2>&1; then + continue + fi + owner="$(kubectl get namespace "$namespace" -o "jsonpath=${owner_jsonpath}")" + if [[ "$owner" != "$control_plane_id" ]]; then + echo "Skipping namespace $namespace: expected owner $control_plane_id, found ${owner:-unset}." + continue + fi + kubectl delete namespace "$namespace" --wait=true +done diff --git a/deploy/stacks/self-managed/scripts/validate-control-plane-config.sh b/deploy/stacks/self-managed/scripts/validate-control-plane-config.sh new file mode 100755 index 000000000..cd8aa7afb --- /dev/null +++ b/deploy/stacks/self-managed/scripts/validate-control-plane-config.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +control_plane_id="${CONTROL_PLANE_ID:-}" +control_plane_domain="${CONTROL_PLANE_DOMAIN:-}" + +# An empty ID selects the legacy single-control-plane behavior. +if [[ -z "$control_plane_id" ]]; then + exit 0 +fi + +dns_label_regex='^[a-z0-9]([-a-z0-9]*[a-z0-9])?$' +dns_name_regex='^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$' + +if [[ "$control_plane_id" == default ]] || + (( ${#control_plane_id} > 20 )) || + [[ ! "$control_plane_id" =~ $dns_label_regex ]]; then + echo "Error: CONTROL_PLANE_ID must be a DNS-1123 label of at most 20 characters and must not be 'default'." >&2 + exit 1 +fi + +if [[ -z "$control_plane_domain" ]]; then + echo "Error: CONTROL_PLANE_DOMAIN is required for a named control plane." >&2 + exit 1 +fi +if (( ${#control_plane_domain} > 253 )) || + [[ ! "$control_plane_domain" =~ $dns_name_regex ]]; then + echo "Error: CONTROL_PLANE_DOMAIN must be a lowercase DNS name." >&2 + exit 1 +fi + +for gateway in \ + "${CONTROL_PLANE_SHARED_GATEWAY:-${control_plane_id}-shared-gw}" \ + "${CONTROL_PLANE_GRPC_GATEWAY:-${control_plane_id}-grpc-gw}" \ + "${CONTROL_PLANE_NATS_GATEWAY:-${control_plane_id}-nats-gw}"; do + if (( ${#gateway} > 63 )) || + [[ ! "$gateway" =~ $dns_label_regex ]] || + [[ "$gateway" != "${control_plane_id}-"* ]]; then + echo "Error: named control-plane Gateway '$gateway' must be a DNS-1123 label starting with '${control_plane_id}-'." >&2 + exit 1 + fi +done diff --git a/deploy/stacks/self-managed/tests/control-plane-isolation.test.sh b/deploy/stacks/self-managed/tests/control-plane-isolation.test.sh new file mode 100755 index 000000000..8e97ba9e3 --- /dev/null +++ b/deploy/stacks/self-managed/tests/control-plane-isolation.test.sh @@ -0,0 +1,387 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Regression: two named control planes must render plane-owned releases into +# disjoint namespaces, while omitting the ID must preserve the legacy layout. +set -euo pipefail + +stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +stacks_dir="$(cd "$stack_dir/.." && pwd)" +gateway_chart_dir="$(cd "$stack_dir/../../helm/gateway-routes/chart" && pwd)" +work_dir="$(mktemp -d)" +test_stacks_dir="$work_dir/stacks" +test_stack_dir="$test_stacks_dir/self-managed" +trap 'rm -rf "$work_dir"' EXIT + +fail() { + echo "control-plane-isolation: $*" >&2 + exit 1 +} + +mkdir -p "$test_stacks_dir" +cp -R "$stacks_dir"/. "$test_stacks_dir" +printf '{}\n' >"$test_stack_dir/secrets/base-secrets.yaml" + +render_releases() { + local control_plane_id="$1" + local output_file="$2" + local state_file + local -a plane_args=() + local state_output state_log + local gateway_prefix="" + + if test -n "$control_plane_id"; then + gateway_prefix="$control_plane_id-" + plane_args=( + --state-values-set-string "global.controlPlane.id=$control_plane_id" + --state-values-set-string global.controlPlane.sharedInfrastructure=external + --state-values-set certManager.enabled=false + --state-values-set-string "global.domain=$control_plane_id.example.test" + --state-values-set-string "ingress.gatewayApi.gateways.nats.name=$control_plane_id-nats-gateway" + ) + fi + + : >"$output_file" + for state_file in 00-observability-infrastructure 01-dependencies 02-core 03-observability; do + state_output="$work_dir/$control_plane_id-$state_file.json" + state_log="$work_dir/$control_plane_id-$state_file.log" + if HELMFILE_ENV=base \ + HELMFILE_CACHE_HOME="$work_dir/helmfile-cache-$control_plane_id" \ + helmfile \ + --file "$test_stack_dir/helmfile.d/$state_file.yaml.gotmpl" \ + --environment default \ + --state-values-set ingress.gatewayApi.controllerNamespace=gateway-system \ + --state-values-set "ingress.gatewayApi.gateways.shared.name=${gateway_prefix}shared-gateway" \ + --state-values-set ingress.gatewayApi.gateways.shared.namespace=gateway-system \ + --state-values-set "ingress.gatewayApi.gateways.grpc.name=${gateway_prefix}grpc-gateway" \ + --state-values-set ingress.gatewayApi.gateways.grpc.namespace=gateway-system \ + "${plane_args[@]}" \ + list --skip-charts --output json >"$state_output" 2>"$state_log"; then + jq -c '.[] | select(.enabled != false and .installed != false)' "$state_output" >>"$output_file" + elif ! grep -Fq 'no releases found' "$state_log"; then + cat "$state_log" >&2 + fail "$state_file failed to render for ${control_plane_id:-legacy}" + fi + done +} + +assert_release_namespace() { + local releases_file="$1" release_name="$2" want_namespace="$3" + local got + got="$(jq -sr --arg name "$release_name" \ + '[.[] | select(.name == $name) | .namespace] | unique | if length == 1 then .[0] else "" end' \ + "$releases_file")" + test "$got" = "$want_namespace" || + fail "$release_name: expected namespace $want_namespace, got ${got:-}" +} + +render_values() { + local plane_id="$1" state_file="$2" release_name="$3" output_file="$4" + shift 4 + HELMFILE_ENV=base \ + HELMFILE_CACHE_HOME="$work_dir/helmfile-cache-values-$plane_id" \ + helmfile \ + --file "$test_stack_dir/helmfile.d/$state_file.yaml.gotmpl" \ + --environment default \ + --state-values-set-string "global.controlPlane.id=$plane_id" \ + --state-values-set-string global.controlPlane.sharedInfrastructure=external \ + --state-values-set certManager.enabled=false \ + --state-values-set-string "global.domain=$plane_id.example.test" \ + --state-values-set ingress.gatewayApi.controllerNamespace=gateway-system \ + --state-values-set-string "ingress.gatewayApi.gateways.shared.name=$plane_id-shared-gateway" \ + --state-values-set ingress.gatewayApi.gateways.shared.namespace=gateway-system \ + --state-values-set-string "ingress.gatewayApi.gateways.grpc.name=$plane_id-grpc-gateway" \ + --state-values-set ingress.gatewayApi.gateways.grpc.namespace=gateway-system \ + --state-values-set-string "ingress.gatewayApi.gateways.nats.name=$plane_id-nats-gateway" \ + "$@" \ + --selector "name=$release_name" \ + write-values --output-file-template "$output_file" >/dev/null +} + +assert_value() { + local values_file="$1" expression="$2" want="$3" label="$4" + local got + got="$(yq -r "$expression" "$values_file")" + test "$got" = "$want" || + fail "$label: expected $want, got ${got:-}" +} + +find_legacy_service_references() { + ruby -ryaml -e ' + legacy = /\.(?:nvcf|api-keys|sis|ess|nats-system|vault-system|cassandra-system)\.svc(?:\.cluster\.local)?/ + walk = lambda do |value, path| + case value + when Hash + return if value["name"] == "OPENBAO_JWT_AUDIENCE" + value.each do |key, child| + next if key.to_s == "audience" + walk.call(child, path + [key]) + end + when Array + value.each_with_index { |child, index| walk.call(child, path + [index]) } + when String + puts "#{path.join(".")}:#{value}" if value.match?(legacy) + end + end + ARGV.each do |file| + YAML.load_stream(File.read(file)).compact.each { |doc| walk.call(doc, []) } + end + ' "$@" +} + +render_releases alpha "$work_dir/alpha.jsonl" +render_releases beta "$work_dir/beta.jsonl" +render_releases '' "$work_dir/legacy.jsonl" + +# These releases own the control plane's data or services. A missing prefix on +# any one of them allows one plane to read, mutate, or delete the other's state. +for spec in \ + 'nats:nats-system' \ + 'openbao-server:vault-system' \ + 'cassandra:cassandra-system' \ + 'api-keys:api-keys' \ + 'sis:sis' \ + 'api:nvcf' \ + 'nvct-api:nvcf' \ + 'invocation-service:nvcf' \ + 'grpc-proxy:nvcf' \ + 'ess-api:ess' \ + 'notary-service:nvcf' \ + 'admin-issuer-proxy:api-keys' \ + 'reval:nvcf' \ + 'nats-auth-callout-service:nats-system' \ + 'state-metrics:nvcf' \ + 'function-autoscaler:nvcf'; do + release_name="${spec%%:*}" + legacy_namespace="${spec#*:}" + assert_release_namespace "$work_dir/alpha.jsonl" "$release_name" "alpha-$legacy_namespace" + assert_release_namespace "$work_dir/beta.jsonl" "$release_name" "beta-$legacy_namespace" + assert_release_namespace "$work_dir/legacy.jsonl" "$release_name" "$legacy_namespace" +done + +assert_release_namespace "$work_dir/legacy.jsonl" ingress gateway-system +assert_release_namespace "$work_dir/alpha.jsonl" alpha-ingress alpha-ingress +assert_release_namespace "$work_dir/beta.jsonl" beta-ingress beta-ingress + +alpha_owned="$(jq -r 'select(.namespace | startswith("alpha-")) | .namespace + "/" + .name' \ + "$work_dir/alpha.jsonl" | sort -u)" +beta_owned="$(jq -r 'select(.namespace | startswith("beta-")) | .namespace + "/" + .name' \ + "$work_dir/beta.jsonl" | sort -u)" +test -n "$alpha_owned" || fail 'alpha rendered no plane-owned releases' +test -n "$beta_owned" || fail 'beta rendered no plane-owned releases' +if comm -12 <(printf '%s\n' "$alpha_owned") <(printf '%s\n' "$beta_owned") | grep -q .; then + fail 'named control planes rendered overlapping plane-owned release identities' +fi + +for plane_id in alpha beta; do + if jq -e --arg prefix "$plane_id-" \ + 'select((.namespace | startswith($prefix)) | not)' \ + "$work_dir/$plane_id.jsonl" >/dev/null; then + jq -c --arg prefix "$plane_id-" \ + 'select((.namespace | startswith($prefix)) | not)' \ + "$work_dir/$plane_id.jsonl" >&2 + fail "$plane_id rendered a release outside its owned namespaces" + fi +done + +# The release namespace is only half of the isolation boundary. Values passed +# into the charts must point every in-cluster client, route, and bootstrap job +# back to the same plane. +for plane_id in alpha beta; do + core_values="$work_dir/$plane_id-core-values.yaml" + dependency_values="$work_dir/$plane_id-dependency-values.yaml" + render_values "$plane_id" 02-core nvct-api "$core_values" + render_values "$plane_id" 01-dependencies openbao-server "$dependency_values" + + assert_value "$dependency_values" '.openbao.fullnameOverride' \ + "$plane_id-openbao-server" "$plane_id OpenBao identity" + assert_value "$dependency_values" '.openbao.controlPlane.id' \ + "$plane_id" "$plane_id OpenBao owner" + assert_value "$core_values" '.api.accountBootstrap.openbaoServiceAddress' \ + "$plane_id-openbao-server.$plane_id-vault-system.svc.cluster.local:8200" \ + "$plane_id API bootstrap" + assert_value "$core_values" '.api.env.NVCF_NATS_URL' \ + "nats://nats.$plane_id-nats-system.svc.cluster.local:4222" \ + "$plane_id API NATS" + assert_value "$core_values" '.invocation.env.NVCF_API_ADDRESS' \ + "http://api.$plane_id-nvcf.svc.cluster.local:9090" \ + "$plane_id invocation API" + assert_value "$core_values" '.adminIssuerProxy.fullnameOverride' \ + "$plane_id-admin-token-issuer-proxy" "$plane_id admin issuer identity" + assert_value "$core_values" '.adminIssuerProxy.config.vaultAddr' \ + "http://$plane_id-openbao-server.$plane_id-vault-system.svc.cluster.local:8200" \ + "$plane_id admin issuer OpenBao" + assert_value "$core_values" '.nvcfGatewayRoutes.routeNamespace' \ + "$plane_id-ingress" "$plane_id route namespace" + assert_value "$core_values" '.nvcfGatewayRoutes.routes.nvcfApi.backend.namespace' \ + "$plane_id-nvcf" "$plane_id API route backend" + assert_value "$core_values" '.nvcfGatewayRoutes.routes.apiKeys.backend.namespace' \ + "$plane_id-api-keys" "$plane_id API keys route backend" + legacy_references="$(find_legacy_service_references "$core_values" "$dependency_values")" + if test -n "$legacy_references"; then + printf '%s\n' "$legacy_references" >&2 + fail "$plane_id values contain legacy cross-plane service references" + fi +done + +# A stack-managed ClusterIssuer is cluster-scoped and retained by Helm. Prefix +# matching alone is ambiguous: alpha-beta's canonical issuer also begins with +# alpha's prefix. Require alpha's exact canonical name at both render layers. +managed_issuer_error='managed ClusterIssuer name "alpha-beta-nvcf-openbao-pki" must equal canonical name "alpha-nvcf-openbao-pki"' +named_managed_args=( + --state-values-set addons.llm.enabled=true + --state-values-set addons.llm.pki.enabled=true + --state-values-set addons.llm.pki.clusterIssuer.enabled=true + --state-values-set-string addons.llm.pki.issuerName=alpha-beta-nvcf-openbao-pki +) +if HELMFILE_ENV=base helmfile \ + --file "$test_stack_dir/helmfile.d/01-dependencies.yaml.gotmpl" \ + --environment default \ + --state-values-set-string global.controlPlane.id=alpha \ + --state-values-set-string global.controlPlane.sharedInfrastructure=external \ + --state-values-set certManager.enabled=false \ + --state-values-set-string global.domain=alpha.example.test \ + "${named_managed_args[@]}" \ + list --skip-charts >"$work_dir/unprefixed-managed-dependency.log" 2>&1; then + fail 'dependency state accepted another plane canonical name through a shared prefix' +fi +grep -Fq "$managed_issuer_error" "$work_dir/unprefixed-managed-dependency.log" || + fail 'dependency state did not return the named managed ClusterIssuer ownership error' + +if render_values alpha 02-core nvct-api \ + "$work_dir/unprefixed-managed-global.yaml" \ + "${named_managed_args[@]}" \ + >"$work_dir/unprefixed-managed-global.log" 2>&1; then + fail 'global values accepted another plane canonical name through a shared prefix' +fi +grep -Fq "$managed_issuer_error" "$work_dir/unprefixed-managed-global.log" || + fail 'global values did not return the named managed ClusterIssuer ownership error' + +# The exact canonical issuer is owned by this plane. Both the dependency release +# and chart values must carry that stable name and owner identity. +canonical_managed_args=( + --state-values-set addons.llm.enabled=true + --state-values-set addons.llm.pki.enabled=true + --state-values-set addons.llm.pki.clusterIssuer.enabled=true + --state-values-set-string addons.llm.pki.issuerName=alpha-nvcf-openbao-pki +) +HELMFILE_ENV=base helmfile \ + --file "$test_stack_dir/helmfile.d/01-dependencies.yaml.gotmpl" \ + --environment default \ + --state-values-set-string global.controlPlane.id=alpha \ + --state-values-set-string global.controlPlane.sharedInfrastructure=external \ + --state-values-set certManager.enabled=false \ + --state-values-set-string global.domain=alpha.example.test \ + "${canonical_managed_args[@]}" \ + list --skip-charts --output json >"$work_dir/canonical-managed-dependency.json" +jq -e '.[] | select(.name == "alpha-nvcf-pki" and .enabled != false and .installed != false)' \ + "$work_dir/canonical-managed-dependency.json" >/dev/null || + fail 'dependency state omitted the canonical managed ClusterIssuer' +render_values alpha 01-dependencies alpha-nvcf-pki \ + "$work_dir/canonical-managed-dependency.yaml" \ + "${canonical_managed_args[@]}" +assert_value "$work_dir/canonical-managed-dependency.yaml" \ + '.clusterIssuer.name' alpha-nvcf-openbao-pki \ + 'named canonical managed ClusterIssuer name' +assert_value "$work_dir/canonical-managed-dependency.yaml" \ + '.clusterIssuer.controlPlaneID' alpha \ + 'named canonical managed ClusterIssuer owner' +render_values alpha 02-core nvct-api "$work_dir/canonical-managed-global.yaml" \ + "${canonical_managed_args[@]}" +assert_value "$work_dir/canonical-managed-global.yaml" \ + '.llmRequestRouter.certificate.issuerRef.name' alpha-nvcf-openbao-pki \ + 'named canonical managed Certificate issuer' + +# An explicitly external issuer is not owned or deleted by the stack and may +# therefore keep an unprefixed name in named mode. +external_issuer_args=( + --state-values-set addons.llm.enabled=true + --state-values-set addons.llm.pki.enabled=true + --state-values-set addons.llm.pki.clusterIssuer.enabled=false + --state-values-set-string addons.llm.pki.issuerName=external-shared-pki +) +HELMFILE_ENV=base helmfile \ + --file "$test_stack_dir/helmfile.d/01-dependencies.yaml.gotmpl" \ + --environment default \ + --state-values-set-string global.controlPlane.id=alpha \ + --state-values-set-string global.controlPlane.sharedInfrastructure=external \ + --state-values-set certManager.enabled=false \ + --state-values-set-string global.domain=alpha.example.test \ + "${external_issuer_args[@]}" \ + list --skip-charts --output json >"$work_dir/external-issuer-dependency.json" +if jq -e '.[] | select(.name == "alpha-nvcf-pki" and .enabled != false and .installed != false)' \ + "$work_dir/external-issuer-dependency.json" >/dev/null; then + fail 'dependency state managed an explicitly external ClusterIssuer' +fi +render_values alpha 02-core nvct-api "$work_dir/external-issuer-global.yaml" \ + "${external_issuer_args[@]}" +assert_value "$work_dir/external-issuer-global.yaml" \ + '.llmRequestRouter.certificate.issuerRef.name' external-shared-pki \ + 'named external ClusterIssuer' + +# The currently pinned UI chart has cluster-wide RBAC and therefore cannot be +# presented as isolated merely by changing its namespace. +if HELMFILE_ENV=base helmfile \ + --file "$test_stack_dir/helmfile.d/02-core.yaml.gotmpl" \ + --environment default \ + --state-values-set-string global.controlPlane.id=alpha \ + --state-values-set-string global.controlPlane.sharedInfrastructure=external \ + --state-values-set certManager.enabled=false \ + --state-values-set-string global.domain=alpha.example.test \ + --state-values-set-string ingress.gatewayApi.gateways.shared.name=alpha-shared-gateway \ + --state-values-set-string ingress.gatewayApi.gateways.grpc.name=alpha-grpc-gateway \ + --state-values-set-string ingress.gatewayApi.gateways.nats.name=alpha-nats-gateway \ + --state-values-set addons.nvcfUi.enabled=true \ + list --skip-charts >"$work_dir/named-ui.log" 2>&1; then + fail 'named control plane accepted the non-isolated NVCF UI addon' +fi +grep -Fq 'addons.nvcfUi.enabled must be false for a named control plane' \ + "$work_dir/named-ui.log" || + fail 'named NVCF UI rejection did not return the expected error' + +# Worker-facing TLS material is created in the shared Gateway namespace, so +# its name, SAN, issuer, and Gateway references must still be plane-specific. +llm_manifest="$work_dir/alpha-llm-gateway.yaml" +llm_hostname=alpha-llm-grpc.alpha.example.test +HELMFILE_ENV=base helmfile \ + --file "$test_stack_dir/helmfile.d/02-core.yaml.gotmpl" \ + --environment default \ + --state-values-set-string global.controlPlane.id=alpha \ + --state-values-set-string global.controlPlane.sharedInfrastructure=external \ + --state-values-set certManager.enabled=false \ + --state-values-set-string global.domain=alpha.example.test \ + --state-values-set-string ingress.gatewayApi.gateways.shared.name=alpha-shared-gateway \ + --state-values-set ingress.gatewayApi.gateways.shared.namespace=gateway-system \ + --state-values-set-string ingress.gatewayApi.gateways.grpc.name=alpha-grpc-gateway \ + --state-values-set ingress.gatewayApi.gateways.grpc.namespace=gateway-system \ + --state-values-set-string ingress.gatewayApi.gateways.nats.name=alpha-nats-gateway \ + --state-values-set addons.llm.enabled=true \ + --state-values-set ingress.gatewayApi.routes.llmWorker.enabled=true \ + --state-values-set addons.llm.requestRouter.grpcTls.enabled=true \ + --state-values-set-string "addons.llm.requestRouter.grpcTls.dnsNames[0]=$llm_hostname" \ + --state-values-set-string "global.workerEndpoints.llmRequestRouterAddress=https://$llm_hostname:443" \ + --state-values-set-string "addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress=https://$llm_hostname:443" \ + --state-values-set-string "addons.llm.requestRouter.backendRouter.pylonReverseTunnelDialAddress=alpha-llm-quic.alpha.example.test:443" \ + --selector name=alpha-ingress \ + --chart "$gateway_chart_dir" \ + --skip-deps template >"$llm_manifest" + +certificate="$(yq -o=json -I=0 \ + 'select(.kind == "Certificate" and .metadata.name == "alpha-llm-request-router-grpc-tls")' \ + "$llm_manifest")" +test -n "$certificate" || fail 'named LLM render omitted its gRPC Certificate' +test "$(jq -r '.metadata.namespace' <<<"$certificate")" = gateway-system || + fail 'named LLM Certificate was not rendered in the shared Gateway namespace' +test "$(jq -r '.spec.secretName' <<<"$certificate")" = alpha-llm-request-router-grpc-tls || + fail 'named LLM Certificate secret is not plane-prefixed' +test "$(jq -r '.spec.dnsNames[0]' <<<"$certificate")" = "$llm_hostname" || + fail 'named LLM Certificate SAN does not match its endpoint' +test "$(jq -r '.spec.issuerRef.name' <<<"$certificate")" = alpha-nvcf-openbao-pki || + fail 'named LLM Certificate does not use the plane-scoped issuer' +grep -Fq 'name: alpha-llm-grpc-gw' "$llm_manifest" || + fail 'named LLM gRPC route did not reference a plane-prefixed Gateway' +grep -Fq 'name: alpha-llm-quic-gw' "$llm_manifest" || + fail 'named LLM QUIC route did not reference a plane-prefixed Gateway' + +echo 'control-plane-isolation: all checks passed' diff --git a/deploy/stacks/self-managed/tests/control-plane-lifecycle.test.sh b/deploy/stacks/self-managed/tests/control-plane-lifecycle.test.sh new file mode 100755 index 000000000..724eeeab9 --- /dev/null +++ b/deploy/stacks/self-managed/tests/control-plane-lifecycle.test.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Regression: destroying one named control plane must delete only its owned +# namespaces. Another plane and intentionally shared prerequisites must remain. +set -euo pipefail + +stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +work_dir="$(mktemp -d)" +test_stack_dir="$work_dir/self-managed" +fake_bin="$work_dir/bin" +calls_file="$work_dir/kubectl.calls" +helmfile_calls_file="$work_dir/helmfile.calls" +prepared_file="$work_dir/prepared.namespaces" +trap 'rm -rf "$work_dir"' EXIT + +fail() { + echo "control-plane-lifecycle: $*" >&2 + exit 1 +} + +mkdir -p "$fake_bin" +cp -R "$stack_dir" "$test_stack_dir" + +printf '%s\n' '#!/usr/bin/env bash' \ + 'set -euo pipefail' \ + 'printf "%s\n" "$*" >>"$HELMFILE_CALLS_FILE"' \ + 'case " $* " in *" destroy "*|*" sync "*) exit 0 ;; *) exit 1 ;; esac' \ + >"$fake_bin/helmfile" +printf '%s\n' '#!/usr/bin/env bash' \ + 'set -euo pipefail' \ + 'printf "%s\n" "$*" >>"$KUBECTL_CALLS_FILE"' \ + 'if test "${1:-}" = get && test "${2:-}" = namespace; then' \ + ' test "${FAKE_EXISTS:-0}" = 1 || exit 1' \ + ' case " $* " in *" jsonpath="*) printf "%s" "${FAKE_OWNER:-}" ;; esac' \ + 'fi' \ + 'if test "${1:-}" = get && test "${2:-}" = clusterissuers; then' \ + ' case " $* " in' \ + ' *"control-plane-id=alpha "*) printf "%s\n" clusterissuer.cert-manager.io/alpha-nvcf-openbao-pki clusterissuer.cert-manager.io/alpha-retained-pki ;;' \ + ' *"control-plane-id=gamma "*) printf "%s\n" clusterissuer.cert-manager.io/gamma-nvcf-openbao-pki ;;' \ + ' esac' \ + 'fi' \ + 'if test "${1:-}" = get && test "${2:-}" = clusterissuer; then' \ + ' if test -n "${FAKE_ISSUER_OWNER_OVERRIDE:-}"; then printf "%s" "$FAKE_ISSUER_OWNER_OVERRIDE"; else' \ + ' case "${3:-}" in alpha-*) printf alpha ;; beta-*) printf beta ;; gamma-*) printf gamma ;; *) printf external ;; esac' \ + ' fi' \ + 'fi' \ + 'if test "${1:-}" = label && test "${2:-}" = namespace && test -n "${PREPARED_FILE:-}"; then' \ + ' printf "%s\n" "${3:-}" >>"$PREPARED_FILE"' \ + 'fi' \ + 'exit 0' \ + >"$fake_bin/kubectl" +chmod +x "$fake_bin/helmfile" "$fake_bin/kubectl" + +PATH="$fake_bin:$PATH" \ + KUBECTL_CALLS_FILE="$calls_file" \ + HELMFILE_CALLS_FILE="$helmfile_calls_file" \ + FAKE_EXISTS=1 \ + FAKE_OWNER=alpha \ + make --no-print-directory -f "$test_stack_dir/Makefile.dist" \ + destroy DEV_MODE=1 HELMFILE_ENV=base \ + CONTROL_PLANE_ID=alpha CONTROL_PLANE_DOMAIN=alpha.example.test >/dev/null + +helmfile_call="$(<"$helmfile_calls_file")" +for argument in \ + '--state-values-set-string global.controlPlane.id=alpha' \ + '--state-values-set-string global.controlPlane.sharedInfrastructure=external' \ + '--state-values-set certManager.enabled=false' \ + '--state-values-set-string global.domain=alpha.example.test'; do + grep -Fq -- "$argument" <<<"$helmfile_call" || + fail "destroy did not pass named-plane state argument: $argument" +done + +deleted_namespaces="$(sed -n 's/^delete namespace \([^ ]*\).*/\1/p' "$calls_file" | sort -u)" +for namespace in \ + alpha-api-keys \ + alpha-cassandra-system \ + alpha-ess \ + alpha-ingress \ + alpha-nats-system \ + alpha-nvcf \ + alpha-sis \ + alpha-vault-system; do + grep -Fxq "$namespace" <<<"$deleted_namespaces" || + fail "destroy did not delete owned namespace $namespace" +done + +for namespace in \ + beta-nvcf \ + beta-vault-system \ + cert-manager \ + gateway-system \ + nvcf \ + vault-system; do + if grep -Fxq "$namespace" <<<"$deleted_namespaces"; then + fail "destroy deleted unowned namespace $namespace" + fi +done + +expected_count=8 +actual_count="$(printf '%s\n' "$deleted_namespaces" | sed '/^$/d' | wc -l | tr -d ' ')" +test "$actual_count" = "$expected_count" || + fail "destroy deleted $actual_count namespaces, expected exactly $expected_count" + +# Managed ClusterIssuers carry the same plane owner label but are retained by +# Helm. Named destroy must explicitly remove all current/retained issuers owned +# by alpha without touching beta or an externally managed issuer. +deleted_clusterissuers="$(sed -n 's/^delete clusterissuer \([^ ]*\).*/\1/p' "$calls_file" | sort -u)" +for issuer in alpha-nvcf-openbao-pki alpha-retained-pki; do + grep -Fxq "$issuer" <<<"$deleted_clusterissuers" || + fail "destroy did not delete owned managed ClusterIssuer $issuer" +done +for issuer in beta-nvcf-openbao-pki external-shared-pki; do + if grep -Fxq "$issuer" <<<"$deleted_clusterissuers"; then + fail "destroy deleted unowned ClusterIssuer $issuer" + fi +done +grep -Fq 'get clusterissuers -l nvcf.nvidia.com/control-plane-id=alpha -o name' "$calls_file" || + fail 'destroy did not select managed ClusterIssuers by plane owner label' + +# Re-check ownership on each selected object so a stale list result or label +# race fails closed instead of deleting an issuer that is no longer alpha's. +: >"$calls_file" +if PATH="$fake_bin:$PATH" \ + KUBECTL_CALLS_FILE="$calls_file" \ + HELMFILE_CALLS_FILE="$helmfile_calls_file" \ + FAKE_EXISTS=1 \ + FAKE_OWNER=alpha \ + FAKE_ISSUER_OWNER_OVERRIDE=beta \ + make --no-print-directory -f "$test_stack_dir/Makefile.dist" \ + destroy DEV_MODE=1 HELMFILE_ENV=base \ + CONTROL_PLANE_ID=alpha CONTROL_PLANE_DOMAIN=alpha.example.test \ + >"$work_dir/foreign-issuer-owner.log" 2>&1; then + fail 'destroy accepted a managed ClusterIssuer whose owner changed' +fi +if grep -q '^delete clusterissuer ' "$calls_file"; then + fail 'destroy deleted a managed ClusterIssuer after its owner changed' +fi + +# Empty ID remains backward compatible with the historical namespace set. +: >"$calls_file" +PATH="$fake_bin:$PATH" \ + KUBECTL_CALLS_FILE="$calls_file" \ + HELMFILE_CALLS_FILE="$helmfile_calls_file" \ + FAKE_EXISTS=1 \ + make --no-print-directory -f "$test_stack_dir/Makefile.dist" \ + destroy DEV_MODE=1 HELMFILE_ENV=base CONTROL_PLANE_ID= >/dev/null +legacy_deleted="$(sed -n 's/^delete namespace \([^ ]*\).*/\1/p' "$calls_file" | sort -u)" +for namespace in api-keys cassandra-system ess nats-system ncp nvcf sis vault-system; do + grep -Fxq "$namespace" <<<"$legacy_deleted" || + fail "legacy destroy did not delete namespace $namespace" +done +if grep -q '^delete clusterissuer ' "$calls_file"; then + fail 'legacy destroy deleted a retained unlabeled ClusterIssuer' +fi + +# Existing unlabelled or foreign-owned namespaces must stop the destroy before +# Helmfile can uninstall releases from them. +for foreign_owner in '' beta; do + : >"$calls_file" + : >"$helmfile_calls_file" + if PATH="$fake_bin:$PATH" \ + KUBECTL_CALLS_FILE="$calls_file" \ + HELMFILE_CALLS_FILE="$helmfile_calls_file" \ + FAKE_EXISTS=1 \ + FAKE_OWNER="$foreign_owner" \ + make --no-print-directory -f "$test_stack_dir/Makefile.dist" \ + destroy DEV_MODE=1 HELMFILE_ENV=base \ + CONTROL_PLANE_ID=alpha CONTROL_PLANE_DOMAIN=alpha.example.test \ + >"$work_dir/foreign-owner.log" 2>&1; then + fail "destroy accepted namespace owner ${foreign_owner:-}" + fi + test ! -s "$helmfile_calls_file" || + fail "destroy invoked Helmfile before rejecting namespace owner ${foreign_owner:-}" + if grep -q '^delete namespace ' "$calls_file"; then + fail "destroy deleted a namespace owned by ${foreign_owner:-}" + fi +done + +# A stable identity may live in the selected environment instead of the Make +# command. Lifecycle ownership must resolve that value before choosing what to +# delete. +environment_name=lifecycle-plane +printf '%s\n' \ + 'global:' \ + ' controlPlane:' \ + ' id: gamma' \ + ' domain: gamma.example.test' \ + >"$test_stack_dir/environments/$environment_name.yaml" +: >"$calls_file" +: >"$helmfile_calls_file" +PATH="$fake_bin:$PATH" \ + KUBECTL_CALLS_FILE="$calls_file" \ + HELMFILE_CALLS_FILE="$helmfile_calls_file" \ + FAKE_EXISTS=1 \ + FAKE_OWNER=gamma \ + make --no-print-directory -f "$test_stack_dir/Makefile.dist" \ + destroy DEV_MODE=1 HELMFILE_ENV="$environment_name" >/dev/null +grep -Fq -- '--state-values-set-string global.controlPlane.id=gamma' \ + "$helmfile_calls_file" || + fail 'environment-configured identity did not reach Helmfile destroy' +grep -Fq 'delete namespace gamma-nvcf --wait=true' "$calls_file" || + fail 'environment-configured identity did not select owned namespace cleanup' + +if PATH="$fake_bin:$PATH" \ + make --no-print-directory -f "$test_stack_dir/Makefile.dist" \ + validate-control-plane-id CONTROL_PLANE_ID=alpha HELMFILE_ENV=base \ + >"$work_dir/missing-domain.log" 2>&1; then + fail 'named control plane accepted a missing unique domain' +fi +grep -Fq 'CONTROL_PLANE_DOMAIN is required' "$work_dir/missing-domain.log" || + fail 'missing domain did not return the expected error' + +# Namespace preparation and pre-install hooks are state mutations that must not +# race even when an operator invokes Make with -j. The hook sees all namespaces +# only if the install prerequisites are serialized in declaration order. +parallel_makefile="$work_dir/parallel.Makefile" +printf '%s\n' \ + "include $test_stack_dir/Makefile.dist" \ + '.PHONY: assert-namespaces-prepared' \ + 'assert-namespaces-prepared:' \ + $'\t@test "$$(wc -l < "$(PREPARED_FILE)")" -eq 8' \ + >"$parallel_makefile" +: >"$calls_file" +: >"$helmfile_calls_file" +: >"$prepared_file" +PATH="$fake_bin:$PATH" \ + KUBECTL_CALLS_FILE="$calls_file" \ + HELMFILE_CALLS_FILE="$helmfile_calls_file" \ + PREPARED_FILE="$prepared_file" \ + FAKE_EXISTS=0 \ + make --no-print-directory -j4 -f "$parallel_makefile" \ + install DEV_MODE=1 HELMFILE_ENV=base INSTALL_PRE_HOOKS=assert-namespaces-prepared \ + CONTROL_PLANE_ID=alpha CONTROL_PLANE_DOMAIN=alpha.example.test >/dev/null || + fail 'parallel install allowed namespace preparation and install hooks to race' + +echo 'control-plane-lifecycle: all checks passed' diff --git a/docs/user/helmfile-installation.md b/docs/user/helmfile-installation.md index c0f3d38dd..ad47b0b98 100644 --- a/docs/user/helmfile-installation.md +++ b/docs/user/helmfile-installation.md @@ -850,6 +850,45 @@ Deploy the self-managed stack: HELMFILE_ENV= helmfile sync ``` +#### Multiple isolated control planes in one cluster + +Use the Make targets when two or more control planes share a Kubernetes +cluster. Give each plane a stable DNS-1123 ID of at most 20 characters and a +unique external domain in its environment file: + +```yaml +global: + controlPlane: + id: plane-a + domain: plane-a.example.test +``` + +Install cert-manager, the Gateway API controller, and the shared observability +backend once, outside either plane's lifecycle. Named-plane installs treat +those components as external prerequisites and derive isolated service, data, +route, and ingress namespaces from the ID: + +```bash +make install HELMFILE_ENV=plane-a +make install HELMFILE_ENV=plane-b +``` + +The same selected environment must be used for updates and removal. Before +installing, updating, or deleting a named plane, the Make targets verify the +`nvcf.nvidia.com/control-plane-id=` namespace ownership label. Removing +one plane therefore leaves the other plane and shared prerequisites intact: + +```bash +make apply HELMFILE_ENV=plane-a +make destroy HELMFILE_ENV=plane-a +``` + +The optional NVCF UI is not yet supported in named-plane mode because its +published chart requires cluster-wide read access. Keep `addons.nvcfUi.enabled` +set to `false`; configuration validation rejects it rather than silently +weakening isolation. An empty control-plane ID preserves the existing +single-control-plane names and lifecycle behavior. + The initial deployment takes approximately 5-10 minutes for local development and 10-20 minutes for cloud deployments. diff --git a/migrations/cassandra/README.md b/migrations/cassandra/README.md index 2e50974d1..09535c7db 100644 --- a/migrations/cassandra/README.md +++ b/migrations/cassandra/README.md @@ -63,10 +63,11 @@ The container reads the following environment variables: | `CASSANDRA_PASSWORD` | Cassandra superuser password | | `SERVICE_ROLE_PASSWORD` | Substituted into the per-keyspace login role passwords (`02_init_roles.up.sql`) | | `REPLICA_COUNT` | Replication factor for service keyspaces (defaults to `3`) | +| `CONTROL_PLANE_ID` | Optional DNS-1123 control-plane ID. When set, derives isolated ESS, OpenBao, and Notary authorization URLs; empty preserves the legacy namespaces. | For each keyspace under `keyspaces/`, the container: -1. Pre-processes any `*.sql` and `*.cql` files via `envsubst` so that `${SERVICE_ROLE_PASSWORD}` and `${REPLICA_COUNT}` are substituted (other environment variables are intentionally not substituted). +1. Validates `CONTROL_PLANE_ID`, derives the authorization URLs without accepting raw CQL string inputs, and pre-processes `*.sql` and `*.cql` via an explicit `envsubst` allowlist. 2. Runs `migrate up` with the `cassandra://` driver, using the keyspace name as the migrations table name. A failure in any keyspace's migration set stops the run. @@ -138,6 +139,8 @@ spec: - name: SERVICE_ROLE_PASSWORD valueFrom: secretKeyRef: { name: cassandra-credentials, key: service-role-password } + - name: CONTROL_PLANE_ID + value: "plane-a" ``` ## Authoring migrations diff --git a/migrations/cassandra/execute_sqls.sh b/migrations/cassandra/execute_sqls.sh index 8a72419d2..d068ae964 100755 --- a/migrations/cassandra/execute_sqls.sh +++ b/migrations/cassandra/execute_sqls.sh @@ -15,6 +15,27 @@ # limitations under the License. REPLICA_COUNT=${REPLICA_COUNT:-3} +CONTROL_PLANE_ID=${CONTROL_PLANE_ID:-} +if [ -n "$CONTROL_PLANE_ID" ]; then + case "$CONTROL_PLANE_ID" in + *[!a-z0-9-]*|-*|*-) + echo "ERROR: CONTROL_PLANE_ID must be a DNS-1123 label, got: $CONTROL_PLANE_ID" >&2 + exit 1 + ;; + esac + if [ "${#CONTROL_PLANE_ID}" -gt 20 ] || [ "$CONTROL_PLANE_ID" = "default" ]; then + echo "ERROR: CONTROL_PLANE_ID must be at most 20 characters and must not be 'default', got: $CONTROL_PLANE_ID" >&2 + exit 1 + fi + NOTARY_BASE_URL="http://notary.${CONTROL_PLANE_ID}-nvcf.svc.cluster.local:8080" + ESS_JWKS_URL="http://${CONTROL_PLANE_ID}-openbao-server.${CONTROL_PLANE_ID}-vault-system.svc.cluster.local:8200/v1/services/ess-api/jwt/jwks" + ESS_ISSUER_URL="http://ess-api.${CONTROL_PLANE_ID}-ess.svc.cluster.local" +else + NOTARY_BASE_URL="http://notary.nvcf.svc.cluster.local:8080" + ESS_JWKS_URL="http://openbao-server.vault-system.svc.cluster.local:8200/v1/services/ess-api/jwt/jwks" + ESS_ISSUER_URL="http://ess-api.ess.svc.cluster.local" +fi +export NOTARY_BASE_URL ESS_JWKS_URL ESS_ISSUER_URL case "$REPLICA_COUNT" in ''|*[!0-9]*|0*) echo "ERROR: REPLICA_COUNT must be an integer from 1 to 2147483647, got: $REPLICA_COUNT" >&2 @@ -48,7 +69,7 @@ echo "Cassandra cqlsh superuser is available" # SECURITY: Only explicitly listed variables are substituted to prevent # unintended substitution of other environment variables # shellcheck disable=SC2016 -ENVSUBST_VARS='$SERVICE_ROLE_PASSWORD $REPLICA_COUNT' +ENVSUBST_VARS='$SERVICE_ROLE_PASSWORD $REPLICA_COUNT $NOTARY_BASE_URL $ESS_JWKS_URL $ESS_ISSUER_URL' TEMP_KEYSPACES="/tmp/keyspaces" echo "Pre-processing SQL files with environment variable substitution..." diff --git a/migrations/cassandra/keyspaces/ess_api/03_init_tables.up.sql b/migrations/cassandra/keyspaces/ess_api/03_init_tables.up.sql index 87910bcb8..b7c0ca82b 100644 --- a/migrations/cassandra/keyspaces/ess_api/03_init_tables.up.sql +++ b/migrations/cassandra/keyspaces/ess_api/03_init_tables.up.sql @@ -27,6 +27,7 @@ CREATE TYPE IF NOT EXISTS ess_api.entity_type ( CREATE TABLE IF NOT EXISTS ess_api.namespaces ( namespace text, oauth_authorizations map>, -- new primary column for tenant (non-notary) auths, oauth wins over ssa on read-merge + ssa_authorizations map>, -- compatibility for supported ESS images that still select this column notary_authorizations map>, entity_types map>, created_at timestamp, diff --git a/migrations/cassandra/keyspaces/ess_api/04_init_ncp_namespace.up.sql b/migrations/cassandra/keyspaces/ess_api/04_init_ncp_namespace.up.sql index 2075a87ea..b96451886 100644 --- a/migrations/cassandra/keyspaces/ess_api/04_init_ncp_namespace.up.sql +++ b/migrations/cassandra/keyspaces/ess_api/04_init_ncp_namespace.up.sql @@ -5,6 +5,7 @@ INSERT INTO ess_api.namespaces ( updated_at, entity_hash_size, require_lwt_for_secret_version_writes, + ssa_authorizations, notary_authorizations ) VALUES ( @@ -14,5 +15,12 @@ VALUES ( toTimestamp(now()), 10, False, - {'nvcf-api': {id: 'nvcf-api', name: 'nvcf notary client', jwks_url: 'http://notary.nvcf.svc.cluster.local:8080/.well-known/jwks.json', issuer: 'http://notary.nvcf.svc.cluster.local:8080', type: 'NOTARY'}} + { + 'nvcf-api': {id: 'nvcf-api', name: 'nvcf api service client', jwks_url: '${ESS_JWKS_URL}', issuer: '${ESS_ISSUER_URL}', type: 'SSA'}, + 'nvct-api': {id: 'nvct-api', name: 'nvct api service client', jwks_url: '${ESS_JWKS_URL}', issuer: '${ESS_ISSUER_URL}', type: 'SSA'} + }, + { + 'nvcf-api': {id: 'nvcf-api', name: 'nvcf notary client', jwks_url: '${NOTARY_BASE_URL}/.well-known/jwks.json', issuer: '${NOTARY_BASE_URL}', type: 'NOTARY'}, + 'nvct-api': {id: 'nvct-api', name: 'nvct api notary client', jwks_url: '${NOTARY_BASE_URL}/.well-known/jwks.json', issuer: '${NOTARY_BASE_URL}', type: 'NOTARY'} + } ); diff --git a/migrations/cassandra/keyspaces/ess_api/06_fix_nvcf_api_notary_issuer.up.sql b/migrations/cassandra/keyspaces/ess_api/06_fix_nvcf_api_notary_issuer.up.sql index 4b6a3ad84..0c0c3dc98 100644 --- a/migrations/cassandra/keyspaces/ess_api/06_fix_nvcf_api_notary_issuer.up.sql +++ b/migrations/cassandra/keyspaces/ess_api/06_fix_nvcf_api_notary_issuer.up.sql @@ -8,8 +8,8 @@ SET 'nvcf-api': { id: 'nvcf-api', name: 'nvcf notary client', - jwks_url: 'http://notary.nvcf.svc.cluster.local:8080/.well-known/jwks.json', - issuer: 'http://notary.nvcf.svc.cluster.local:8080', + jwks_url: '${NOTARY_BASE_URL}/.well-known/jwks.json', + issuer: '${NOTARY_BASE_URL}', type: 'NOTARY' } } diff --git a/migrations/cassandra/keyspaces/ess_api/08_add_oauth_authorizations.up.sql b/migrations/cassandra/keyspaces/ess_api/08_add_oauth_authorizations.up.sql index 4cc63aaf6..6c95c807b 100644 --- a/migrations/cassandra/keyspaces/ess_api/08_add_oauth_authorizations.up.sql +++ b/migrations/cassandra/keyspaces/ess_api/08_add_oauth_authorizations.up.sql @@ -5,15 +5,15 @@ SET 'nvcf-api': { id: 'nvcf-api', name: 'nvcf api service client', - jwks_url: 'http://openbao-server.vault-system.svc.cluster.local:8200/v1/services/ess-api/jwt/jwks', - issuer: 'http://ess-api.ess.svc.cluster.local', + jwks_url: '${ESS_JWKS_URL}', + issuer: '${ESS_ISSUER_URL}', type: null }, 'nvct-api': { id: 'nvct-api', name: 'nvct api service client', - jwks_url: 'http://openbao-server.vault-system.svc.cluster.local:8200/v1/services/ess-api/jwt/jwks', - issuer: 'http://ess-api.ess.svc.cluster.local', + jwks_url: '${ESS_JWKS_URL}', + issuer: '${ESS_ISSUER_URL}', type: null } } diff --git a/migrations/cassandra/keyspaces/ess_api/09_reconcile_control_plane_authorizations.up.sql b/migrations/cassandra/keyspaces/ess_api/09_reconcile_control_plane_authorizations.up.sql new file mode 100644 index 000000000..b8119c94f --- /dev/null +++ b/migrations/cassandra/keyspaces/ess_api/09_reconcile_control_plane_authorizations.up.sql @@ -0,0 +1,27 @@ +-- Reconcile both current and compatibility authorization maps. This forward +-- migration repairs clusters that already applied the legacy, single-plane +-- seeds in migrations 04, 06, or 08. + +ALTER TABLE ess_api.namespaces ADD IF NOT EXISTS ( + oauth_authorizations map>, + ssa_authorizations map>, + authorizations_version timeuuid +); + +UPDATE ess_api.namespaces +SET + updated_at = toTimestamp(now()), + authorizations_version = now(), + ssa_authorizations = ssa_authorizations + { + 'nvcf-api': {id: 'nvcf-api', name: 'nvcf api service client', jwks_url: '${ESS_JWKS_URL}', issuer: '${ESS_ISSUER_URL}', type: 'SSA'}, + 'nvct-api': {id: 'nvct-api', name: 'nvct api service client', jwks_url: '${ESS_JWKS_URL}', issuer: '${ESS_ISSUER_URL}', type: 'SSA'} + }, + oauth_authorizations = oauth_authorizations + { + 'nvcf-api': {id: 'nvcf-api', name: 'nvcf api service client', jwks_url: '${ESS_JWKS_URL}', issuer: '${ESS_ISSUER_URL}', type: null}, + 'nvct-api': {id: 'nvct-api', name: 'nvct api service client', jwks_url: '${ESS_JWKS_URL}', issuer: '${ESS_ISSUER_URL}', type: null} + }, + notary_authorizations = notary_authorizations + { + 'nvcf-api': {id: 'nvcf-api', name: 'nvcf notary client', jwks_url: '${NOTARY_BASE_URL}/.well-known/jwks.json', issuer: '${NOTARY_BASE_URL}', type: 'NOTARY'}, + 'nvct-api': {id: 'nvct-api', name: 'nvct api notary client', jwks_url: '${NOTARY_BASE_URL}/.well-known/jwks.json', issuer: '${NOTARY_BASE_URL}', type: 'NOTARY'} + } +WHERE namespace = 'nvcf'; diff --git a/migrations/cassandra/tests/test-execute-sqls.sh b/migrations/cassandra/tests/test-execute-sqls.sh index 79b6868ba..9bcc21538 100755 --- a/migrations/cassandra/tests/test-execute-sqls.sh +++ b/migrations/cassandra/tests/test-execute-sqls.sh @@ -43,8 +43,8 @@ fi envsubst_vars=$(sed -n "s/^ENVSUBST_VARS='\\(.*\\)'$/\\1/p" "${script}") # shellcheck disable=SC2016 -if [ "${envsubst_vars}" != '$SERVICE_ROLE_PASSWORD $REPLICA_COUNT' ]; then - fail "execute_sqls.sh does not allow REPLICA_COUNT substitution" +if [ "${envsubst_vars}" != '$SERVICE_ROLE_PASSWORD $REPLICA_COUNT $NOTARY_BASE_URL $ESS_JWKS_URL $ESS_ISSUER_URL' ]; then + fail "execute_sqls.sh does not allow the complete, explicit migration variable set" fi # shellcheck disable=SC2016 @@ -61,6 +61,92 @@ for replica_count in 1 3 2147483647; do fail "execute_sqls.sh rejects valid replica count ${replica_count}" fi done + +control_plane_derivation=$( + sed -n '/^CONTROL_PLANE_ID=${CONTROL_PLANE_ID:-}$/,/^export NOTARY_BASE_URL ESS_JWKS_URL ESS_ISSUER_URL$/p' "${script}" +) +derive_control_plane_endpoints() +{ + CONTROL_PLANE_ID="$1" sh -c "${control_plane_derivation} + printf '%s\n%s\n%s\n' \"\$NOTARY_BASE_URL\" \"\$ESS_JWKS_URL\" \"\$ESS_ISSUER_URL\"" +} + +expected_plane_a_endpoints='http://notary.plane-a-nvcf.svc.cluster.local:8080 +http://plane-a-openbao-server.plane-a-vault-system.svc.cluster.local:8200/v1/services/ess-api/jwt/jwks +http://ess-api.plane-a-ess.svc.cluster.local' +if [ "$(derive_control_plane_endpoints plane-a)" != "${expected_plane_a_endpoints}" ]; then + fail "execute_sqls.sh does not derive the exact plane-a authorization endpoints" +fi + +expected_legacy_endpoints='http://notary.nvcf.svc.cluster.local:8080 +http://openbao-server.vault-system.svc.cluster.local:8200/v1/services/ess-api/jwt/jwks +http://ess-api.ess.svc.cluster.local' +if [ "$(derive_control_plane_endpoints '')" != "${expected_legacy_endpoints}" ]; then + fail "execute_sqls.sh does not preserve the legacy authorization endpoints" +fi + +for control_plane_id in default Plane-A plane_a -plane-a plane-a- 'plane-a'';DROP' 123456789012345678901; do + if CONTROL_PLANE_ID="${control_plane_id}" sh -c "${control_plane_derivation}" 2>/dev/null; then + fail "execute_sqls.sh accepts invalid control-plane ID ${control_plane_id}" + fi +done + +notary_base_url='http://notary.plane-a-nvcf.svc.cluster.local:8080' +ess_jwks_url='http://plane-a-openbao-server.plane-a-vault-system.svc.cluster.local:8200/v1/services/ess-api/jwt/jwks' +ess_issuer_url='http://ess-api.plane-a-ess.svc.cluster.local' + +for migration in \ + "${keyspaces}/ess_api/04_init_ncp_namespace.up.sql" \ + "${keyspaces}/ess_api/06_fix_nvcf_api_notary_issuer.up.sql" \ + "${keyspaces}/ess_api/08_add_oauth_authorizations.up.sql" \ + "${keyspaces}/ess_api/09_reconcile_control_plane_authorizations.up.sql" +do + rendered=$( + REPLICA_COUNT=2 \ + SERVICE_ROLE_PASSWORD=test-password \ + NOTARY_BASE_URL="${notary_base_url}" \ + ESS_JWKS_URL="${ess_jwks_url}" \ + ESS_ISSUER_URL="${ess_issuer_url}" \ + envsubst "${envsubst_vars}" < "${migration}" + ) + + if printf '%s\n' "${rendered}" | grep -F -q '${'; then + fail "${migration} leaves an unsubstituted migration variable" + fi + + if grep -F -q '${NOTARY_BASE_URL}' "${migration}" && + ! printf '%s\n' "${rendered}" | grep -F -q "${notary_base_url}"; then + fail "${migration} does not substitute the plane-scoped Notary URL" + fi + if grep -F -q '${ESS_JWKS_URL}' "${migration}" && + { ! printf '%s\n' "${rendered}" | grep -F -q "${ess_jwks_url}" || + ! printf '%s\n' "${rendered}" | grep -F -q "${ess_issuer_url}"; }; then + fail "${migration} does not substitute the plane-scoped ESS authorization URLs" + fi +done + +reconcile_migration="${keyspaces}/ess_api/09_reconcile_control_plane_authorizations.up.sql" +reconcile_schema=$( + sed -n '/^ALTER TABLE ess_api.namespaces ADD IF NOT EXISTS (/,/^);$/p' "${reconcile_migration}" +) +for compatibility_column in oauth_authorizations ssa_authorizations authorizations_version; do + if ! printf '%s\n' "${reconcile_schema}" | grep -F -q "${compatibility_column}"; then + fail "forward migration does not create missing ${compatibility_column} upgrade schema" + fi +done +for authorization_map in ssa_authorizations oauth_authorizations notary_authorizations; do + if ! grep -F -q "${authorization_map} = ${authorization_map} +" "${reconcile_migration}"; then + fail "forward migration does not reconcile ${authorization_map}" + fi +done +if ! grep -F -q 'authorizations_version = now()' "${reconcile_migration}"; then + fail "forward migration does not bump authorizations_version" +fi +for service_id in nvcf-api nvct-api; do + if [ "$(grep -F -c "'${service_id}':" "${reconcile_migration}")" -ne 3 ]; then + fail "forward migration does not reconcile all three authorization maps for ${service_id}" + fi +done for replica_count in 0 01 -1 invalid 2147483648 99999999999; do if REPLICA_COUNT="${replica_count}" sh -c "${replica_count_validation}" 2>/dev/null; then fail "execute_sqls.sh accepts invalid replica count ${replica_count}" diff --git a/migrations/openbao/addons/lls/setup_lls.sh b/migrations/openbao/addons/lls/setup_lls.sh index b88716624..e950a8c0e 100644 --- a/migrations/openbao/addons/lls/setup_lls.sh +++ b/migrations/openbao/addons/lls/setup_lls.sh @@ -28,8 +28,8 @@ else source "${migrations_dir}/utils/encryption_setup.sh" fi -SERVICE_ACCOUNT_NAMESPACE="gdn-streaming" -SERVICE_ACCOUNT_NAME="turn" +SERVICE_ACCOUNT_NAMESPACE="${TURN_SERVICE_ACCOUNT_NAMESPACE:-gdn-streaming}" +SERVICE_ACCOUNT_NAME="${TURN_SERVICE_ACCOUNT_NAME:-turn}" #------------------------------------------- # Set defaults for secret paths and policies diff --git a/migrations/openbao/addons/nvcf-ui/README.md b/migrations/openbao/addons/nvcf-ui/README.md index 640eba8d2..76d7839ec 100644 --- a/migrations/openbao/addons/nvcf-ui/README.md +++ b/migrations/openbao/addons/nvcf-ui/README.md @@ -13,9 +13,10 @@ Creates, for the `nvcf-ui` ServiceAccount in the `nvcf-ui` namespace: (`services/sis-api/jwt`) - JWT secret sign role and read/write policy on the NVCF API mount (`services/nvcf-api/jwt`) -- The NVCT JWT secret engine (`services/nvct-api/jwt`) plus a sign role and - read/write policy. This mount exists only to let the UI mint NVCT tokens, so - the addon owns it rather than core migration `20_setup_nvct.sh`. +- A sign role and read/write policy on the core-owned NVCT JWT secret engine + (`services/nvct-api/jwt`). Core migration `20_setup_nvct.sh` owns the mount so + nvct-api can always serve its configured JWKS endpoint; the addon adds only + the optional UI credential-minting path. - JWT auth role `nvcf-ui` bound to the `nvcf-ui` namespace and service account, attached to the three sign policies above diff --git a/migrations/openbao/addons/nvcf-ui/setup_nvcf-ui.sh b/migrations/openbao/addons/nvcf-ui/setup_nvcf-ui.sh index 1417e97a9..e863b32f9 100755 --- a/migrations/openbao/addons/nvcf-ui/setup_nvcf-ui.sh +++ b/migrations/openbao/addons/nvcf-ui/setup_nvcf-ui.sh @@ -28,14 +28,14 @@ fi log_section "Setting up nvcf-ui service" -SERVICE_ACCOUNT_NAMESPACE="nvcf-ui" -SERVICE_ACCOUNT_NAME="nvcf-ui" +SERVICE_ACCOUNT_NAMESPACE="${NVCF_UI_NAMESPACE:-nvcf-ui}" +SERVICE_ACCOUNT_NAME="${NVCF_UI_SERVICE_ACCOUNT_NAME:-nvcf-ui}" #------------------------------------------- # Add Access to Spot Instance Service API via JWT Secret Role #------------------------------------------- -sis_namespace="sis" +sis_namespace="${SIS_NAMESPACE:-sis}" sis_service="api" sis_account="sis-api" sis_secret_base="services/${sis_account}" @@ -57,7 +57,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="${policy_name}" # Add Access to NVCF API via JWT Secret Role #------------------------------------------- -NVCF_API_SERVICE_ACCOUNT_NAMESPACE="nvcf" +NVCF_API_SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" NVCF_API_SERVICE_NAME="api" NVCF_API_SERVICE_ACCOUNT_NAME="nvcf-api" NVCF_API_SECRET_BASE_PATH="services/${NVCF_API_SERVICE_ACCOUNT_NAME}" @@ -79,24 +79,18 @@ VAULT_JWT_AUTH_ROLE_POLICIES="${VAULT_JWT_AUTH_ROLE_POLICIES},${policy_name}" #------------------------------------------- # Add Access to NVCT API via JWT Secret Role # -# The NVCT JWT secrets mount only exists to let the UI mint NVCT tokens, so the -# addon owns it. Enable it here (idempotent) instead of in core migration -# 20_setup_nvct.sh, keeping the NVCT signing path out of non-UI installs. +# Core migration 20 owns and configures the NVCT JWT mount because nvct-api's +# resource server always publishes that JWKS endpoint. This addon owns only the +# optional UI signing role and its policy. #------------------------------------------- -NVCT_API_SERVICE_ACCOUNT_NAMESPACE="nvcf" +NVCT_API_SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" NVCT_API_SERVICE_NAME="nvct-api" NVCT_API_SERVICE_ACCOUNT_NAME="nvct-api" NVCT_API_SECRET_BASE_PATH="services/${NVCT_API_SERVICE_ACCOUNT_NAME}" NVCT_API_SECRET_POLICY_PATH="services-${NVCT_API_SERVICE_ACCOUNT_NAME}" SCOPES="admin:cancel_task,admin:delete_task,admin:launch_task,admin:list_tasks,admin:task_details,admin:update_secrets,admin:list_events,admin:list_results" -# Create the NVCT JWT secret engine (mounted only when the UI addon runs) -enable_secrets_mount "${NVCT_API_SECRET_BASE_PATH}/jwt" "vault-plugin-secrets-jwt" - -jwt_secret_mount_config=$(generate_jwt_secret_mount_config) -config_jwt_secret_mount_config "${NVCT_API_SECRET_BASE_PATH}/jwt" "${jwt_secret_mount_config}" - # Issuer: http://nvct-api.nvcf.svc.cluster.local jwt_secret_role=$(generate_jwt_secret_role "${NVCT_API_SERVICE_ACCOUNT_NAMESPACE}" "${NVCT_API_SERVICE_NAME}" "${SERVICE_ACCOUNT_NAME}" "${SCOPES}") create_secret_jwt_role "${NVCT_API_SECRET_BASE_PATH}/jwt" "${SERVICE_ACCOUNT_NAME}" "${jwt_secret_role}" diff --git a/migrations/openbao/migrations/04_setup_reval.sh b/migrations/openbao/migrations/04_setup_reval.sh index 94c769ae6..682b7e012 100755 --- a/migrations/openbao/migrations/04_setup_reval.sh +++ b/migrations/openbao/migrations/04_setup_reval.sh @@ -26,7 +26,7 @@ else source "${curr_dir}/utils/functions.sh" fi -SERVICE_ACCOUNT_NAMESPACE="nvcf" +SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" SERVICE_ACCOUNT_NAME="reval" #------------------------------------------- diff --git a/migrations/openbao/migrations/05_setup_sis.sh b/migrations/openbao/migrations/05_setup_sis.sh index 179890af5..ccb42ea9f 100644 --- a/migrations/openbao/migrations/05_setup_sis.sh +++ b/migrations/openbao/migrations/05_setup_sis.sh @@ -26,7 +26,7 @@ else source "${curr_dir}/utils/functions.sh" fi -SERVICE_ACCOUNT_NAMESPACE="sis" +SERVICE_ACCOUNT_NAMESPACE="${SIS_NAMESPACE:-sis}" SERVICE_ACCOUNT_NAME="sis-api" #------------------------------------------- diff --git a/migrations/openbao/migrations/06_setup_notary-service.sh b/migrations/openbao/migrations/06_setup_notary-service.sh index c68fcf2ba..b8ac82d0b 100644 --- a/migrations/openbao/migrations/06_setup_notary-service.sh +++ b/migrations/openbao/migrations/06_setup_notary-service.sh @@ -28,7 +28,7 @@ else fi -SERVICE_ACCOUNT_NAMESPACE="nvcf" +SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" SERVICE_ACCOUNT_NAME="nvcf-notary" diff --git a/migrations/openbao/migrations/07_setup_api-keys.sh b/migrations/openbao/migrations/07_setup_api-keys.sh index 7dbf9954b..f1b05c887 100644 --- a/migrations/openbao/migrations/07_setup_api-keys.sh +++ b/migrations/openbao/migrations/07_setup_api-keys.sh @@ -27,7 +27,7 @@ else source "${curr_dir}/utils/encryption_setup.sh" fi -SERVICE_ACCOUNT_NAMESPACE="api-keys" +SERVICE_ACCOUNT_NAMESPACE="${API_KEYS_NAMESPACE:-api-keys}" SERVICE_ACCOUNT_NAME="api-keys-api" # 43-char service id for NVCT registration. Must match the value stored diff --git a/migrations/openbao/migrations/08_setup_ess.sh b/migrations/openbao/migrations/08_setup_ess.sh index 7980206f0..7d21beed0 100644 --- a/migrations/openbao/migrations/08_setup_ess.sh +++ b/migrations/openbao/migrations/08_setup_ess.sh @@ -26,7 +26,7 @@ else source "${curr_dir}/utils/functions.sh" fi -SERVICE_ACCOUNT_NAMESPACE="ess" +SERVICE_ACCOUNT_NAMESPACE="${ESS_NAMESPACE:-ess}" SERVICE_ACCOUNT_NAME="ess-api" #------------------------------------------- diff --git a/migrations/openbao/migrations/09_setup_nvcf-api.sh b/migrations/openbao/migrations/09_setup_nvcf-api.sh index fdbdc6228..9f405312d 100644 --- a/migrations/openbao/migrations/09_setup_nvcf-api.sh +++ b/migrations/openbao/migrations/09_setup_nvcf-api.sh @@ -27,7 +27,7 @@ else fi -SERVICE_ACCOUNT_NAMESPACE="nvcf" +SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" SERVICE_ACCOUNT_NAME="nvcf-api" SERVICE_NAME="api" @@ -102,7 +102,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="${VAULT_JWT_AUTH_ROLE_POLICIES},${policy_name}" # Add Access to SIS API via JWT Secret Role #------------------------------------------- -SIS_API_SERVICE_ACCOUNT_NAMESPACE="sis" +SIS_API_SERVICE_ACCOUNT_NAMESPACE="${SIS_NAMESPACE:-sis}" SIS_API_SERVICE_ACCOUNT_NAME="sis-api" SIS_API_SERVICE_NAME="api" SIS_API_SECRET_BASE_PATH="services/${SIS_API_SERVICE_ACCOUNT_NAME}" @@ -133,7 +133,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="${VAULT_JWT_AUTH_ROLE_POLICIES},${policy_name}" # Add Access to API-KEYS API via JWT Secret Role #-------------------------------------------- -API_KEYS_API_SERVICE_ACCOUNT_NAMESPACE="api-keys" +API_KEYS_API_SERVICE_ACCOUNT_NAMESPACE="${API_KEYS_NAMESPACE:-api-keys}" API_KEYS_API_SERVICE_ACCOUNT_NAME="api-keys-api" API_KEYS_API_SECRET_BASE_PATH="services/${API_KEYS_API_SERVICE_ACCOUNT_NAME}" API_KEYS_API_SECRET_POLICY_PATH="services-${API_KEYS_API_SERVICE_ACCOUNT_NAME}" @@ -161,7 +161,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="${VAULT_JWT_AUTH_ROLE_POLICIES},${policy_name}" # Add Access to ESS API via JWT Secret Role #------------------------------------------- -ESS_API_SERVICE_ACCOUNT_NAMESPACE="ess" +ESS_API_SERVICE_ACCOUNT_NAMESPACE="${ESS_NAMESPACE:-ess}" ESS_API_SERVICE_ACCOUNT_NAME="ess-api" ESS_API_SECRET_BASE_PATH="services/${ESS_API_SERVICE_ACCOUNT_NAME}" ESS_API_SECRET_POLICY_PATH="services-${ESS_API_SERVICE_ACCOUNT_NAME}" @@ -187,7 +187,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="${VAULT_JWT_AUTH_ROLE_POLICIES},${policy_name}" # Add Access to API Keys API via JWT Secret Role #------------------------------------------- -API_KEYS_API_SERVICE_ACCOUNT_NAMESPACE="api-keys" +API_KEYS_API_SERVICE_ACCOUNT_NAMESPACE="${API_KEYS_NAMESPACE:-api-keys}" API_KEYS_API_SERVICE_ACCOUNT_NAME="api-keys-api" API_KEYS_API_SERVICE_NAME="api-keys" API_KEYS_API_SECRET_BASE_PATH="services/${API_KEYS_API_SERVICE_ACCOUNT_NAME}" @@ -217,7 +217,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="${VAULT_JWT_AUTH_ROLE_POLICIES},${policy_name}" # Add Access to Reval API via JWT Secret Role #------------------------------------------- -REVAL_SERVICE_ACCOUNT_NAMESPACE="nvcf" +REVAL_SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" REVAL_SERVICE_ACCOUNT_NAME="reval" REVAL_SERVICE_NAME="reval" REVAL_SECRET_BASE_PATH="services/${REVAL_SERVICE_ACCOUNT_NAME}" diff --git a/migrations/openbao/migrations/10_setup_admin-issuer.sh b/migrations/openbao/migrations/10_setup_admin-issuer.sh index 5baf7fc91..7e6bb89ae 100644 --- a/migrations/openbao/migrations/10_setup_admin-issuer.sh +++ b/migrations/openbao/migrations/10_setup_admin-issuer.sh @@ -24,7 +24,7 @@ else source "${curr_dir}/utils/functions.sh" fi -SERVICE_ACCOUNT_NAMESPACE="api-keys" +SERVICE_ACCOUNT_NAMESPACE="${API_KEYS_NAMESPACE:-api-keys}" SERVICE_ACCOUNT_NAME="admin-issuer-proxy" SERVICE_NAME="admin-issuer-proxy" @@ -49,7 +49,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="" # Issuer will be: http://api.nvcf.svc.cluster.local (same as NVCF API) # Client will be: admin-issuer-proxy (the proxy's identity) -NVCF_API_NAMESPACE="nvcf" +NVCF_API_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" NVCF_API_SERVICE_NAME="api" # Full admin-level scopes for NVCF operations diff --git a/migrations/openbao/migrations/11_setup_ratelimiter.sh b/migrations/openbao/migrations/11_setup_ratelimiter.sh index 08ee163b1..f0eff7d51 100644 --- a/migrations/openbao/migrations/11_setup_ratelimiter.sh +++ b/migrations/openbao/migrations/11_setup_ratelimiter.sh @@ -26,7 +26,7 @@ else source "${curr_dir}/utils/functions.sh" fi -SERVICE_ACCOUNT_NAMESPACE="nvcf" +SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" SERVICE_ACCOUNT_NAME="ratelimiter-api" #------------------------------------------- @@ -51,7 +51,7 @@ config_jwt_secret_mount_config "${VAULT_SECRET_BASE_PATH}/jwt" "${jwt_secret_mou # Add Access to NVCF API via JWT Secret Role #------------------------------------------- -NVCF_API_SERVICE_ACCOUNT_NAMESPACE="nvcf" +NVCF_API_SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" NVCF_API_SERVICE_ACCOUNT_NAME="nvcf-api" NVCF_API_SERVICE_NAME="api" NVCF_API_SECRET_BASE_PATH="services/${NVCF_API_SERVICE_ACCOUNT_NAME}" diff --git a/migrations/openbao/migrations/12_setup_grpc-proxy.sh b/migrations/openbao/migrations/12_setup_grpc-proxy.sh index 9c390f705..0a184c5b5 100644 --- a/migrations/openbao/migrations/12_setup_grpc-proxy.sh +++ b/migrations/openbao/migrations/12_setup_grpc-proxy.sh @@ -26,7 +26,7 @@ else source "${curr_dir}/utils/functions.sh" fi -SERVICE_ACCOUNT_NAMESPACE="nvcf" +SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" SERVICE_ACCOUNT_NAME="grpc-proxy-proxy" #------------------------------------------- @@ -40,7 +40,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="services-all-kv-ro" # Add Access to NVCF API via JWT Secret Role #------------------------------------------- -NVCF_API_SERVICE_ACCOUNT_NAMESPACE="nvcf" +NVCF_API_SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" NVCF_API_SERVICE_ACCOUNT_NAME="nvcf-api" NVCF_API_SERVICE_NAME="api" NVCF_API_SECRET_BASE_PATH="services/${NVCF_API_SERVICE_ACCOUNT_NAME}" @@ -69,7 +69,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="${VAULT_JWT_AUTH_ROLE_POLICIES},${policy_name}" # Add Access to Ratelimiter API via JWT Secret Role #------------------------------------------- -RATELIMITER_API_SERVICE_ACCOUNT_NAMESPACE="nvcf" +RATELIMITER_API_SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" RATELIMITER_API_SERVICE_ACCOUNT_NAME="ratelimiter-api" RATELIMITER_API_SERVICE_NAME="ratelimiter" RATELIMITER_API_SECRET_BASE_PATH="services/${RATELIMITER_API_SERVICE_ACCOUNT_NAME}" diff --git a/migrations/openbao/migrations/13_setup_invocation.sh b/migrations/openbao/migrations/13_setup_invocation.sh index 0940e0f37..43f77cd89 100644 --- a/migrations/openbao/migrations/13_setup_invocation.sh +++ b/migrations/openbao/migrations/13_setup_invocation.sh @@ -27,7 +27,7 @@ else fi -SERVICE_ACCOUNT_NAMESPACE="nvcf" +SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" SERVICE_ACCOUNT_NAME="invocation-api" #------------------------------------------- @@ -78,7 +78,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="${VAULT_JWT_AUTH_ROLE_POLICIES},${policy_name}" # Add Access to NVCF API via JWT Secret Role #------------------------------------------- -NVCF_API_SERVICE_ACCOUNT_NAMESPACE="nvcf" +NVCF_API_SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" NVCF_API_SERVICE_ACCOUNT_NAME="nvcf-api" NVCF_API_SERVICE_NAME="api" NVCF_API_SECRET_BASE_PATH="services/${NVCF_API_SERVICE_ACCOUNT_NAME}" @@ -107,7 +107,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="${VAULT_JWT_AUTH_ROLE_POLICIES},${policy_name}" # Add Access to Ratelimiter API via JWT Secret Role #------------------------------------------- -RATELIMITER_API_SERVICE_ACCOUNT_NAMESPACE="nvcf" +RATELIMITER_API_SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" RATELIMITER_API_SERVICE_ACCOUNT_NAME="ratelimiter-api" RATELIMITER_API_SERVICE_NAME="ratelimiter" RATELIMITER_API_SECRET_BASE_PATH="services/${RATELIMITER_API_SERVICE_ACCOUNT_NAME}" diff --git a/migrations/openbao/migrations/14_setup_nvca.sh b/migrations/openbao/migrations/14_setup_nvca.sh index 3d6d26e32..138ccd11b 100644 --- a/migrations/openbao/migrations/14_setup_nvca.sh +++ b/migrations/openbao/migrations/14_setup_nvca.sh @@ -26,7 +26,7 @@ else source "${curr_dir}/utils/functions.sh" fi -SERVICE_ACCOUNT_NAMESPACE="nvca-system" +SERVICE_ACCOUNT_NAMESPACE="${NVCA_NAMESPACE:-nvca-system}" SERVICE_ACCOUNT_NAME="nvca" #------------------------------------------- @@ -40,7 +40,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="services-all-kv-ro" # Add Access to SIS via JWT Secret Role #------------------------------------------- -SIS_API_SERVICE_ACCOUNT_NAMESPACE="sis" +SIS_API_SERVICE_ACCOUNT_NAMESPACE="${SIS_NAMESPACE:-sis}" SIS_API_SERVICE_ACCOUNT_NAME="sis-api" SIS_API_SERVICE_NAME="api" SIS_API_SECRET_BASE_PATH="services/${SIS_API_SERVICE_ACCOUNT_NAME}" @@ -70,7 +70,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="${VAULT_JWT_AUTH_ROLE_POLICIES},${policy_name}" # Add Access to ReVal via JWT Secret Role #------------------------------------------- -REVAL_SERVICE_ACCOUNT_NAMESPACE="nvcf" +REVAL_SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" REVAL_SERVICE_ACCOUNT_NAME="reval" REVAL_SERVICE_NAME="reval" REVAL_SECRET_BASE_PATH="services/${REVAL_SERVICE_ACCOUNT_NAME}" diff --git a/migrations/openbao/migrations/15_setup_nvca-operator.sh b/migrations/openbao/migrations/15_setup_nvca-operator.sh index 32c467b44..031c5dfb8 100644 --- a/migrations/openbao/migrations/15_setup_nvca-operator.sh +++ b/migrations/openbao/migrations/15_setup_nvca-operator.sh @@ -26,7 +26,7 @@ else source "${curr_dir}/utils/functions.sh" fi -SERVICE_ACCOUNT_NAMESPACE="nvca-operator" +SERVICE_ACCOUNT_NAMESPACE="${NVCA_OPERATOR_NAMESPACE:-nvca-operator}" SERVICE_ACCOUNT_NAME="nvca-operator" #------------------------------------------- @@ -40,7 +40,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="services-all-kv-ro" # Add Access to SIS via JWT Secret Role #------------------------------------------- -SIS_API_SERVICE_ACCOUNT_NAMESPACE="sis" +SIS_API_SERVICE_ACCOUNT_NAMESPACE="${SIS_NAMESPACE:-sis}" SIS_API_SERVICE_ACCOUNT_NAME="sis-api" SIS_API_SERVICE_NAME="api" SIS_API_SECRET_BASE_PATH="services/${SIS_API_SERVICE_ACCOUNT_NAME}" diff --git a/migrations/openbao/migrations/16_setup_nvcf-state-metrics.sh b/migrations/openbao/migrations/16_setup_nvcf-state-metrics.sh index 2df027d5b..b42004249 100644 --- a/migrations/openbao/migrations/16_setup_nvcf-state-metrics.sh +++ b/migrations/openbao/migrations/16_setup_nvcf-state-metrics.sh @@ -26,7 +26,7 @@ else source "${curr_dir}/utils/functions.sh" fi -SERVICE_ACCOUNT_NAMESPACE="nvcf" +SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" SERVICE_ACCOUNT_NAME="nvcf-state-metrics" #------------------------------------------- @@ -40,7 +40,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="services-all-kv-ro" # Add Access to NVCF API via JWT Secret Role #------------------------------------------- -NVCF_API_SERVICE_ACCOUNT_NAMESPACE="nvcf" +NVCF_API_SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" NVCF_API_SERVICE_ACCOUNT_NAME="nvcf-api" NVCF_API_SERVICE_NAME="api" NVCF_API_SECRET_BASE_PATH="services/${NVCF_API_SERVICE_ACCOUNT_NAME}" diff --git a/migrations/openbao/migrations/17_setup_llm-api-gateway.sh b/migrations/openbao/migrations/17_setup_llm-api-gateway.sh index 81c5e4d88..4f527015c 100755 --- a/migrations/openbao/migrations/17_setup_llm-api-gateway.sh +++ b/migrations/openbao/migrations/17_setup_llm-api-gateway.sh @@ -26,7 +26,7 @@ else source "${curr_dir}/utils/functions.sh" fi -SERVICE_ACCOUNT_NAMESPACE="nvcf" +SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" SERVICE_ACCOUNT_NAME="llm-api-gateway" #------------------------------------------- @@ -40,7 +40,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="services-all-kv-ro" # Add Access to NVCF API via JWT Secret Role #------------------------------------------- -NVCF_API_SERVICE_ACCOUNT_NAMESPACE="nvcf" +NVCF_API_SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" NVCF_API_SERVICE_ACCOUNT_NAME="nvcf-api" NVCF_API_SERVICE_NAME="api" NVCF_API_SECRET_BASE_PATH="services/${NVCF_API_SERVICE_ACCOUNT_NAME}" diff --git a/migrations/openbao/migrations/18_setup_llm-request-router.sh b/migrations/openbao/migrations/18_setup_llm-request-router.sh index 366b1238f..9ff73c002 100755 --- a/migrations/openbao/migrations/18_setup_llm-request-router.sh +++ b/migrations/openbao/migrations/18_setup_llm-request-router.sh @@ -26,7 +26,7 @@ else source "${curr_dir}/utils/functions.sh" fi -SERVICE_ACCOUNT_NAMESPACE="nvcf" +SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" SERVICE_ACCOUNT_NAME="llm-request-router" #------------------------------------------- @@ -40,7 +40,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="services-all-kv-ro" # Add Access to NVCF API via JWT Secret Role #------------------------------------------- -NVCF_API_SERVICE_ACCOUNT_NAMESPACE="nvcf" +NVCF_API_SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" NVCF_API_SERVICE_ACCOUNT_NAME="nvcf-api" NVCF_API_SERVICE_NAME="api" NVCF_API_SECRET_BASE_PATH="services/${NVCF_API_SERVICE_ACCOUNT_NAME}" diff --git a/migrations/openbao/migrations/19_setup_nats-auth-callout.sh b/migrations/openbao/migrations/19_setup_nats-auth-callout.sh index 041e9b56f..b9c381bc3 100755 --- a/migrations/openbao/migrations/19_setup_nats-auth-callout.sh +++ b/migrations/openbao/migrations/19_setup_nats-auth-callout.sh @@ -30,7 +30,7 @@ fi # the NATS data plane), not in the standard `nvcf` ns the other per-service # scripts use. The `services-all-kv-ro` policy is enough — auth-callout only # reads its own KV path written by 03_setup_shared_secrets.sh. -SERVICE_ACCOUNT_NAMESPACE="nats-system" +SERVICE_ACCOUNT_NAMESPACE="${NATS_NAMESPACE:-nats-system}" SERVICE_ACCOUNT_NAME="nats-auth-callout" VAULT_JWT_AUTH_ROLE_POLICIES="services-all-kv-ro" diff --git a/migrations/openbao/migrations/20_setup_nvct.sh b/migrations/openbao/migrations/20_setup_nvct.sh index db74de92a..9178208d4 100755 --- a/migrations/openbao/migrations/20_setup_nvct.sh +++ b/migrations/openbao/migrations/20_setup_nvct.sh @@ -26,7 +26,7 @@ else source "${curr_dir}/utils/functions.sh" fi -SERVICE_ACCOUNT_NAMESPACE="nvcf" +SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" SERVICE_ACCOUNT_NAME="nvct-api" #------------------------------------------- @@ -44,6 +44,14 @@ VAULT_JWT_AUTH_ROLE_POLICIES="services-all-kv-ro" #------------------------------------------- enable_secrets_mount "${VAULT_SECRET_BASE_PATH}/kv" "kv-v2" +# NVCT is deployed by the core stack and its resource server always resolves +# keys from this mount. The optional UI addon adds a signing role, but must not +# own the mount itself or core/no-UI installs expose a permanently failing JWKS +# endpoint. +enable_secrets_mount "${VAULT_SECRET_BASE_PATH}/jwt" "vault-plugin-secrets-jwt" +jwt_secret_mount_config=$(generate_jwt_secret_mount_config) +config_jwt_secret_mount_config "${VAULT_SECRET_BASE_PATH}/jwt" "${jwt_secret_mount_config}" + #------------------------------------------- # Create default service paths and secrets #------------------------------------------- @@ -63,7 +71,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="${VAULT_JWT_AUTH_ROLE_POLICIES},${VAULT_POLICY_BAS # Add JWT sign access to NVCF API for nvct-api (notary + account_setup) #------------------------------------------- -NVCF_API_SERVICE_ACCOUNT_NAMESPACE="nvcf" +NVCF_API_SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" NVCF_API_SERVICE_ACCOUNT_NAME="nvcf-api" NVCF_API_SERVICE_NAME="api" NVCF_API_SECRET_BASE_PATH="services/${NVCF_API_SERVICE_ACCOUNT_NAME}" @@ -115,7 +123,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="${VAULT_JWT_AUTH_ROLE_POLICIES},${policy_name}" # Add Access to SIS API via JWT Secret Role #------------------------------------------- -SIS_API_SERVICE_ACCOUNT_NAMESPACE="sis" +SIS_API_SERVICE_ACCOUNT_NAMESPACE="${SIS_NAMESPACE:-sis}" SIS_API_SERVICE_ACCOUNT_NAME="sis-api" SIS_API_SERVICE_NAME="api" SIS_API_SECRET_BASE_PATH="services/${SIS_API_SERVICE_ACCOUNT_NAME}" @@ -146,7 +154,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="${VAULT_JWT_AUTH_ROLE_POLICIES},${policy_name}" # Add Access to API-KEYS API via JWT Secret Role #-------------------------------------------- -API_KEYS_API_SERVICE_ACCOUNT_NAMESPACE="api-keys" +API_KEYS_API_SERVICE_ACCOUNT_NAMESPACE="${API_KEYS_NAMESPACE:-api-keys}" API_KEYS_API_SERVICE_ACCOUNT_NAME="api-keys-api" API_KEYS_API_SECRET_BASE_PATH="services/${API_KEYS_API_SERVICE_ACCOUNT_NAME}" API_KEYS_API_SECRET_POLICY_PATH="services-${API_KEYS_API_SERVICE_ACCOUNT_NAME}" @@ -176,7 +184,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="${VAULT_JWT_AUTH_ROLE_POLICIES},${policy_name}" # Add Access to ESS API via JWT Secret Role #------------------------------------------- -ESS_API_SERVICE_ACCOUNT_NAMESPACE="ess" +ESS_API_SERVICE_ACCOUNT_NAMESPACE="${ESS_NAMESPACE:-ess}" ESS_API_SERVICE_ACCOUNT_NAME="ess-api" ESS_API_SECRET_BASE_PATH="services/${ESS_API_SERVICE_ACCOUNT_NAME}" ESS_API_SECRET_POLICY_PATH="services-${ESS_API_SERVICE_ACCOUNT_NAME}" @@ -205,7 +213,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="${VAULT_JWT_AUTH_ROLE_POLICIES},${policy_name}" # Add Access to Reval API via JWT Secret Role #------------------------------------------- -REVAL_SERVICE_ACCOUNT_NAMESPACE="nvcf" +REVAL_SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" REVAL_SERVICE_ACCOUNT_NAME="reval" REVAL_SERVICE_NAME="reval" REVAL_SECRET_BASE_PATH="services/${REVAL_SERVICE_ACCOUNT_NAME}" diff --git a/migrations/openbao/migrations/21_setup_autoscaler.sh b/migrations/openbao/migrations/21_setup_autoscaler.sh index 4d51e409f..0bbd8dc52 100755 --- a/migrations/openbao/migrations/21_setup_autoscaler.sh +++ b/migrations/openbao/migrations/21_setup_autoscaler.sh @@ -27,7 +27,7 @@ else fi -SERVICE_ACCOUNT_NAMESPACE="nvcf" +SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" SERVICE_ACCOUNT_NAME="nvcf-autoscaler-service" #------------------------------------------- @@ -64,7 +64,7 @@ VAULT_JWT_AUTH_ROLE_POLICIES="${VAULT_JWT_AUTH_ROLE_POLICIES},${VAULT_POLICY_BAS # is deprecated and should not be requested by the function autoscaler. #------------------------------------------- -NVCF_API_SERVICE_ACCOUNT_NAMESPACE="nvcf" +NVCF_API_SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" NVCF_API_SERVICE_ACCOUNT_NAME="nvcf-api" NVCF_API_SERVICE_NAME="api" NVCF_API_SECRET_BASE_PATH="services/${NVCF_API_SERVICE_ACCOUNT_NAME}" diff --git a/migrations/openbao/migrations/22_setup_event-ledger.sh b/migrations/openbao/migrations/22_setup_event-ledger.sh index be3d49d8f..b79314a98 100644 --- a/migrations/openbao/migrations/22_setup_event-ledger.sh +++ b/migrations/openbao/migrations/22_setup_event-ledger.sh @@ -26,7 +26,7 @@ else source "${curr_dir}/utils/functions.sh" fi -SERVICE_ACCOUNT_NAMESPACE="nvcf" +SERVICE_ACCOUNT_NAMESPACE="${NVCF_NAMESPACE:-nvcf}" SERVICE_ACCOUNT_NAME="event-ledger" #------------------------------------------- @@ -65,7 +65,7 @@ config_jwt_secret_mount_config "${VAULT_SECRET_BASE_PATH}/jwt" "${jwt_secret_mou # Issuer: http://event-ledger.nvcf.svc.cluster.local #------------------------------------------- -SIS_SERVICE_ACCOUNT_NAMESPACE="sis" +SIS_SERVICE_ACCOUNT_NAMESPACE="${SIS_NAMESPACE:-sis}" SIS_SERVICE_ACCOUNT_NAME="sis-api" SCOPES="fnds:createEvent,fnds:archiveEvents" @@ -85,7 +85,7 @@ create_auth_jwt_role "${SIS_SERVICE_ACCOUNT_NAME}" "${sis_jwt_auth_role}" # Issuer: http://event-ledger.nvcf.svc.cluster.local #------------------------------------------- -NVCA_SERVICE_ACCOUNT_NAMESPACE="nvca-system" +NVCA_SERVICE_ACCOUNT_NAMESPACE="${NVCA_NAMESPACE:-nvca-system}" NVCA_SERVICE_ACCOUNT_NAME="nvca" jwt_secret_role=$(generate_jwt_secret_role "${SERVICE_ACCOUNT_NAMESPACE}" "${SERVICE_ACCOUNT_NAME}" "${NVCA_SERVICE_ACCOUNT_NAME}" "${SCOPES}") diff --git a/migrations/openbao/migrations/utils/functions.sh b/migrations/openbao/migrations/utils/functions.sh index 2d688dc9d..33f5b9299 100644 --- a/migrations/openbao/migrations/utils/functions.sh +++ b/migrations/openbao/migrations/utils/functions.sh @@ -36,7 +36,12 @@ function initialize_mount_lists() { log_success "Mount lists initialized." } -OPENBAO_SERVER_INTERNAL_URL="http://openbao-server.vault-system.svc.cluster.local:8200" +OPENBAO_SERVER_INTERNAL_URL="${OPENBAO_SERVER_INTERNAL_URL:-http://openbao-server.vault-system.svc.cluster.local:8200}" +# JWT audiences are an opaque trust-domain identifier, not the network address +# used to reach a particular OpenBao instance. Keep the legacy audience by +# default so every projected service-account token continues to match its role +# when OPENBAO_SERVER_INTERNAL_URL is plane-scoped. +OPENBAO_JWT_AUDIENCE="${OPENBAO_JWT_AUDIENCE:-http://openbao-server.vault-system.svc.cluster.local:8200}" ## # Enable a auth engine @@ -474,8 +479,8 @@ function generate_jwt_auth_role() { local service_name=$1 local service_account_namespace=$2 local policies=$3 - # Allow audience to be overridden, but default to the server's internal URL - local audience=${4:-"${OPENBAO_SERVER_INTERNAL_URL}"} + # Allow audience to be overridden, but default to the shared trust-domain. + local audience=${4:-"${OPENBAO_JWT_AUDIENCE}"} local quoted_policies=$(sed 's/\([^,]*\)/"\1"/g' <<< "$policies") local quoted_audiences=$(sed 's/\([^,]*\)/"\1"/g' <<< "$audience") diff --git a/migrations/openbao/tests/namespace-isolation-test.sh b/migrations/openbao/tests/namespace-isolation-test.sh new file mode 100755 index 000000000..1c49433b2 --- /dev/null +++ b/migrations/openbao/tests/namespace-isolation-test.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +OPENBAO_SERVER_INTERNAL_URL="http://plane-a-openbao-server.plane-a-vault-system.svc.cluster.local:8200" +export OPENBAO_SERVER_INTERNAL_URL +# shellcheck source=../migrations/utils/functions.sh +source "$root/migrations/utils/functions.sh" +if [[ "$OPENBAO_SERVER_INTERNAL_URL" != "http://plane-a-openbao-server.plane-a-vault-system.svc.cluster.local:8200" ]]; then + echo "FAIL: functions.sh overwrote OPENBAO_SERVER_INTERNAL_URL" >&2 + exit 1 +fi +if [[ "$OPENBAO_JWT_AUDIENCE" != "http://openbao-server.vault-system.svc.cluster.local:8200" ]]; then + echo "FAIL: JWT role audience followed the plane-specific network address" >&2 + exit 1 +fi +OPENBAO_JWT_ISSUER="https://kubernetes.default.svc" +role_json="$(generate_jwt_auth_role test-service plane-a-nvcf test-policy)" +role_audience="$(printf '%s' "$role_json" | jq -r '.bound_audiences[0]')" +if [[ "$role_audience" != "http://openbao-server.vault-system.svc.cluster.local:8200" ]]; then + echo "FAIL: generated JWT role does not use the shared projected-token audience" >&2 + exit 1 +fi +if [[ "$role_audience" == "$OPENBAO_SERVER_INTERNAL_URL" ]]; then + echo "FAIL: generated JWT role uses the plane-specific OpenBao network address as its audience" >&2 + exit 1 +fi + +for namespace_pair in \ + "nvcf:NVCF_NAMESPACE" \ + "sis:SIS_NAMESPACE" \ + "api-keys:API_KEYS_NAMESPACE" \ + "ess:ESS_NAMESPACE" \ + "nats-system:NATS_NAMESPACE" \ + "nvca-system:NVCA_NAMESPACE" \ + "nvca-operator:NVCA_OPERATOR_NAMESPACE"; do + namespace="${namespace_pair%%:*}" + env_name="${namespace_pair#*:}" + if grep -R -E "^[A-Za-z_][A-Za-z0-9_]*(NAMESPACE|namespace)=\"${namespace}\"$" \ + "$root/migrations" "$root/addons" >/dev/null; then + echo "FAIL: hard-coded service-account namespace remains: ${namespace}" >&2 + grep -R -n -E "^[A-Za-z_][A-Za-z0-9_]*(NAMESPACE|namespace)=\"${namespace}\"$" \ + "$root/migrations" "$root/addons" >&2 + exit 1 + fi + expected='${'"${env_name}:-${namespace}"'}' + if ! grep -R -F -q "$expected" "$root/migrations" "$root/addons"; then + echo "FAIL: migration scripts do not consume ${env_name}" >&2 + exit 1 + fi +done + +if ! grep -Fq 'SERVICE_ACCOUNT_NAMESPACE="${TURN_SERVICE_ACCOUNT_NAMESPACE:-gdn-streaming}"' \ + "$root/addons/lls/setup_lls.sh"; then + echo "FAIL: LLS migration does not consume TURN_SERVICE_ACCOUNT_NAMESPACE" >&2 + exit 1 +fi +if ! grep -Fq 'SERVICE_ACCOUNT_NAMESPACE="${NVCF_UI_NAMESPACE:-nvcf-ui}"' \ + "$root/addons/nvcf-ui/setup_nvcf-ui.sh"; then + echo "FAIL: NVCF UI migration does not consume NVCF_UI_NAMESPACE" >&2 + exit 1 +fi + +# NVCT is part of the core stack, so its resource-server JWKS mount must exist +# even when the optional UI addon (and its signing role) is disabled. +if ! grep -Fq 'enable_secrets_mount "${VAULT_SECRET_BASE_PATH}/jwt" "vault-plugin-secrets-jwt"' \ + "$root/migrations/20_setup_nvct.sh"; then + echo "FAIL: core NVCT migration does not own the NVCT JWT mount" >&2 + exit 1 +fi +if ! grep -Fq 'config_jwt_secret_mount_config "${VAULT_SECRET_BASE_PATH}/jwt"' \ + "$root/migrations/20_setup_nvct.sh"; then + echo "FAIL: core NVCT migration does not configure the NVCT JWT mount" >&2 + exit 1 +fi +if grep -Fq 'enable_secrets_mount "${NVCT_API_SECRET_BASE_PATH}/jwt"' \ + "$root/addons/nvcf-ui/setup_nvcf-ui.sh"; then + echo "FAIL: optional UI addon still owns the core NVCT JWT mount" >&2 + exit 1 +fi + +echo "OpenBao migration namespace-isolation checks passed." diff --git a/src/clis/nvcf-cli/cmd/self_hosted_compute_plane.go b/src/clis/nvcf-cli/cmd/self_hosted_compute_plane.go index 5eb7c47c3..801f44d74 100644 --- a/src/clis/nvcf-cli/cmd/self_hosted_compute_plane.go +++ b/src/clis/nvcf-cli/cmd/self_hosted_compute_plane.go @@ -665,10 +665,21 @@ func isHelmfileTemplate(name string) bool { } func parseComputePlaneChart(body string) (string, string) { + const defaultNVCAChart = "nvcf/helm-nvca-operator" var chart, version string inNVCARelease := false for _, line := range strings.Split(body, "\n") { line = strings.TrimSpace(line) + if strings.HasPrefix(line, "chart:") && containsQuotedChartDefault(line, defaultNVCAChart) { + // Named control planes use a templated release name and a source-chart + // override with this registry chart as the default. Preserve the + // registry reference for the CLI handoff summary without requiring a + // full Helm template evaluation. + inNVCARelease = true + chart = defaultNVCAChart + version = "" + continue + } switch { case strings.HasPrefix(line, "- name:"): inNVCARelease = strings.TrimSpace(strings.TrimPrefix(line, "- name:")) == "nvca-operator" @@ -688,6 +699,10 @@ func parseComputePlaneChart(body string) (string, string) { return chart, version } +func containsQuotedChartDefault(line, chart string) bool { + return strings.Contains(line, `"`+chart+`"`) || strings.Contains(line, `'`+chart+`'`) +} + func shellCommand(args ...string) string { quoted := make([]string, 0, len(args)) for _, arg := range args { diff --git a/src/clis/nvcf-cli/cmd/self_hosted_compute_plane_test.go b/src/clis/nvcf-cli/cmd/self_hosted_compute_plane_test.go index ffcb2e0c5..fa4204b78 100644 --- a/src/clis/nvcf-cli/cmd/self_hosted_compute_plane_test.go +++ b/src/clis/nvcf-cli/cmd/self_hosted_compute_plane_test.go @@ -850,6 +850,43 @@ releases: assert.Equal(t, "2.0.0", version) }) + t.Run("reads chart default from dynamically named nvca release", func(t *testing.T) { + stackDir := t.TempDir() + helmfileDir := filepath.Join(stackDir, "helmfile.d") + require.NoError(t, os.MkdirAll(helmfileDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(helmfileDir, "02-nvca.yaml.gotmpl"), []byte(` +{{- $operatorReleaseName := "nvca-operator" }} +releases: + - name: {{ $operatorReleaseName }} + chart: {{ $nvcaOperatorChartPath | default "nvcf/helm-nvca-operator" | quote }} + version: 1.21.3 +`), 0o644)) + + chart, version, err := computePlaneChartFromStack(stackDir) + require.NoError(t, err) + assert.Equal(t, "nvcf/helm-nvca-operator", chart) + assert.Equal(t, "1.21.3", version) + }) + + t.Run("does not mistake a similarly named dynamic chart for the default", func(t *testing.T) { + stackDir := t.TempDir() + helmfileDir := filepath.Join(stackDir, "helmfile.d") + require.NoError(t, os.MkdirAll(helmfileDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(helmfileDir, "02-nvca.yaml.gotmpl"), []byte(` +{{- $operatorReleaseName := "nvca-operator" }} +releases: + - name: {{ $operatorReleaseName }} + chart: {{ $nvcaOperatorChartPath | default "example/nvcf/helm-nvca-operator-backup" | quote }} + version: 1.21.3 +`), 0o644)) + + chart, version, err := computePlaneChartFromStack(stackDir) + require.Error(t, err) + assert.Empty(t, chart) + assert.Empty(t, version) + assert.Contains(t, err.Error(), "compute-plane chart reference not found in stack") + }) + t.Run("errors when helmfile directory is missing", func(t *testing.T) { stackDir := t.TempDir() diff --git a/src/clis/nvcf-cli/cmd/self_hosted_control_plane_profile.go b/src/clis/nvcf-cli/cmd/self_hosted_control_plane_profile.go index d2172e58a..53e5cbc31 100644 --- a/src/clis/nvcf-cli/cmd/self_hosted_control_plane_profile.go +++ b/src/clis/nvcf-cli/cmd/self_hosted_control_plane_profile.go @@ -51,17 +51,23 @@ type controlPlaneProfileWriteRequest struct { ICMSURL string NATSURL string StackDomain string + ControlPlaneID string SourceRootCA bool RootCAPEM string } func writeControlPlaneProfile(req controlPlaneProfileWriteRequest) (string, error) { - if req.StackDomain == "" { - domain, err := loadControlPlaneStackDomain(req.StackPath, req.Env) + if req.StackDomain == "" || req.ControlPlaneID == "" { + settings, err := loadControlPlaneStackProfileSettings(req.StackPath, req.Env) if err != nil { return "", err } - req.StackDomain = domain + if req.StackDomain == "" { + req.StackDomain = settings.Domain + } + if req.ControlPlaneID == "" { + req.ControlPlaneID = settings.ControlPlaneID + } } doc := buildControlPlaneProfile(req) rootCAPEM := strings.TrimSpace(req.RootCAPEM) @@ -71,7 +77,7 @@ func writeControlPlaneProfile(req controlPlaneProfileWriteRequest) (string, erro ctx = context.Background() } var err error - rootCAPEM, err = fetchControlPlaneRootCAPEM(ctx, req.ControlPlaneContext) + rootCAPEM, err = fetchControlPlaneRootCAPEM(ctx, req.ControlPlaneContext, req.ControlPlaneID) if err != nil { return "", err } @@ -92,8 +98,8 @@ func controlPlaneProfilePath(stackPath string) string { return filepath.Join(stackPath, "out", controlPlaneProfileFileName) } -var fetchControlPlaneRootCAPEM = func(ctx context.Context, kctx string) (string, error) { - cfg := controlPlaneRootCAOpenBaoConfig(kctx) +var fetchControlPlaneRootCAPEM = func(ctx context.Context, kctx, controlPlaneID string) (string, error) { + cfg := controlPlaneRootCAOpenBaoConfig(kctx, controlPlaneID) pem, err := openbao.NewClient(cfg, nil).ReadPKICertificatePEM(ctx, controlPlaneRootPKIPath()) if errors.Is(err, openbao.ErrPKICertificateNotFound) { return "", nil @@ -104,13 +110,19 @@ var fetchControlPlaneRootCAPEM = func(ctx context.Context, kctx string) (string, return strings.TrimSpace(pem), nil } -func controlPlaneRootCAOpenBaoConfig(kctx string) *openbao.Config { +func controlPlaneRootCAOpenBaoConfig(kctx, controlPlaneID string) *openbao.Config { + prefix := "" + if id := strings.TrimSpace(controlPlaneID); id != "" { + prefix = id + "-" + } + vaultNamespace := prefix + "vault-system" return &openbao.Config{ - OpenBaoURL: defaultString(firstNonEmptyEnv("NVCF_OPENBAO_URL", "OPENBAO_URL", "VAULT_ADDR", "BAO_ADDR"), "http://openbao-server.vault-system.svc.cluster.local:8200"), - OpenBaoNamespace: defaultString(os.Getenv("NVCF_OPENBAO_NAMESPACE"), "vault-system"), - OpenBaoSecretName: defaultString(os.Getenv("NVCF_OPENBAO_SECRET_NAME"), "openbao-server-root-token"), + OpenBaoURL: defaultString(firstNonEmptyEnv("NVCF_OPENBAO_URL", "OPENBAO_URL", "VAULT_ADDR", "BAO_ADDR"), + fmt.Sprintf("http://%sopenbao-server.%s.svc.cluster.local:8200", prefix, vaultNamespace)), + OpenBaoNamespace: defaultString(os.Getenv("NVCF_OPENBAO_NAMESPACE"), vaultNamespace), + OpenBaoSecretName: defaultString(os.Getenv("NVCF_OPENBAO_SECRET_NAME"), prefix+"openbao-server-root-token"), KubeContext: kctx, - ClusterNamespace: defaultString(os.Getenv("NVCF_CLUSTER_NAMESPACE"), "nvcf"), + ClusterNamespace: defaultString(os.Getenv("NVCF_CLUSTER_NAMESPACE"), prefix+"nvcf"), UtilityImage: defaultString(os.Getenv("NVCF_CLUSTER_UTILITY_IMAGE"), "curlimages/curl:latest"), } } @@ -148,6 +160,7 @@ func buildControlPlaneProfile(req controlPlaneProfileWriteRequest) controlplanep sisHost := firstNonEmpty(os.Getenv("NVCF_ICMS_HOST"), viper.GetString("icms_host"), "sis."+domain) revalHost := firstNonEmpty(os.Getenv("NVCF_REVAL_HOST"), viper.GetString("reval_host"), "reval."+domain) natsHost := firstNonEmpty(os.Getenv("NVCF_NATS_HOST"), viper.GetString("nats_host"), "nats."+domain) + inClusterEndpoints := profileInClusterEndpointScope(req.ControlPlaneID) if strings.EqualFold(req.Env, "local") { computeEndpoints.ICMSServiceURL = rewriteURLHost(computeEndpoints.ICMSServiceURL, sisHost) computeEndpoints.ReValServiceURL = rewriteURLHost(computeEndpoints.ReValServiceURL, revalHost) @@ -162,11 +175,7 @@ func buildControlPlaneProfile(req controlPlaneProfileWriteRequest) controlplanep NCAID: defaultString(req.NCAID, "nvcf-default"), Region: defaultString(req.Region, "us-west-1"), Endpoints: controlplaneprofile.Endpoints{ - InCluster: controlplaneprofile.EndpointScope{ - ICMSURL: "http://api.sis.svc.cluster.local:8080", - ReValURL: "http://reval.nvcf.svc.cluster.local:8080", - NATSURL: "nats://nats.nats-system.svc.cluster.local:4222", - }, + InCluster: inClusterEndpoints, ComputeReachable: controlplaneprofile.EndpointScope{ ICMSURL: computeEndpoints.ICMSServiceURL, ReValURL: computeEndpoints.ReValServiceURL, @@ -189,11 +198,28 @@ func buildControlPlaneProfile(req controlPlaneProfileWriteRequest) controlplanep } } -func loadControlPlaneStackDomain(stackPath, env string) (string, error) { +func profileInClusterEndpointScope(controlPlaneID string) controlplaneprofile.EndpointScope { + prefix := "" + if id := strings.TrimSpace(controlPlaneID); id != "" { + prefix = id + "-" + } + return controlplaneprofile.EndpointScope{ + ICMSURL: fmt.Sprintf("http://api.%ssis.svc.cluster.local:8080", prefix), + ReValURL: fmt.Sprintf("http://reval.%snvcf.svc.cluster.local:8080", prefix), + NATSURL: fmt.Sprintf("nats://nats.%snats-system.svc.cluster.local:4222", prefix), + } +} + +type controlPlaneStackProfileSettings struct { + Domain string + ControlPlaneID string +} + +func loadControlPlaneStackProfileSettings(stackPath, env string) (controlPlaneStackProfileSettings, error) { if stackPath == "" { - return "", nil + return controlPlaneStackProfileSettings{}, nil } - domain := "" + settings := controlPlaneStackProfileSettings{} for _, name := range []string{"base.yaml", env + ".yaml"} { path := filepath.Join(stackPath, "environments", name) body, err := os.ReadFile(path) @@ -201,21 +227,32 @@ func loadControlPlaneStackDomain(stackPath, env string) (string, error) { continue } if err != nil { - return "", fmt.Errorf("reading control-plane stack values %q: %w", path, err) + return controlPlaneStackProfileSettings{}, fmt.Errorf("reading control-plane stack values %q: %w", path, err) } var values struct { Global struct { - Domain string `yaml:"domain"` + Domain string `yaml:"domain"` + ControlPlane struct { + ID string `yaml:"id"` + } `yaml:"controlPlane"` } `yaml:"global"` } if err := yaml.Unmarshal(body, &values); err != nil { - return "", fmt.Errorf("parsing control-plane stack values %q: %w", path, err) + return controlPlaneStackProfileSettings{}, fmt.Errorf("parsing control-plane stack values %q: %w", path, err) } if value := strings.TrimSpace(values.Global.Domain); value != "" { - domain = value + settings.Domain = value + } + if value := strings.TrimSpace(values.Global.ControlPlane.ID); value != "" { + settings.ControlPlaneID = value } } - return domain, nil + return settings, nil +} + +func loadControlPlaneStackDomain(stackPath, env string) (string, error) { + settings, err := loadControlPlaneStackProfileSettings(stackPath, env) + return settings.Domain, err } func resolveProfileICMSURL(flagValue, env, stackDomain string) string { diff --git a/src/clis/nvcf-cli/cmd/self_hosted_control_plane_test.go b/src/clis/nvcf-cli/cmd/self_hosted_control_plane_test.go index 4ae0841d3..9bb6c4178 100644 --- a/src/clis/nvcf-cli/cmd/self_hosted_control_plane_test.go +++ b/src/clis/nvcf-cli/cmd/self_hosted_control_plane_test.go @@ -131,7 +131,7 @@ func TestControlPlaneProfileExportCommandRecreatesProfileFromOpenBao(t *testing. require.NoError(t, err) prevFetch := fetchControlPlaneRootCAPEM - fetchControlPlaneRootCAPEM = func(_ context.Context, kctx string) (string, error) { + fetchControlPlaneRootCAPEM = func(_ context.Context, kctx, _ string) (string, error) { assert.Equal(t, "cp-context", kctx) return rootCA, nil } @@ -179,15 +179,16 @@ func TestControlPlaneProfileExportCommandUsesSelectedEnvironmentDomain(t *testin require.NoError(t, os.MkdirAll(filepath.Join(stackDir, "helmfile.d"), 0o755)) require.NoError(t, os.MkdirAll(filepath.Join(stackDir, "environments"), 0o755)) require.NoError(t, os.WriteFile(filepath.Join(stackDir, "environments", "base.yaml"), []byte("global:\n domain: base.example.test\n"), 0o600)) - require.NoError(t, os.WriteFile(filepath.Join(stackDir, "environments", "alpha.yaml"), []byte("global:\n domain: alpha.example.test\n"), 0o600)) - require.NoError(t, os.WriteFile(filepath.Join(stackDir, "environments", "beta.yaml"), []byte("global:\n domain: beta.example.test\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stackDir, "environments", "alpha.yaml"), []byte("global:\n domain: alpha.example.test\n controlPlane:\n id: plane-a\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stackDir, "environments", "beta.yaml"), []byte("global:\n domain: beta.example.test\n controlPlane:\n id: plane-b\n"), 0o600)) for _, tc := range []struct { - env string - domain string + env string + domain string + controlPlaneID string }{ - {env: "alpha", domain: "alpha.example.test"}, - {env: "beta", domain: "beta.example.test"}, + {env: "alpha", domain: "alpha.example.test", controlPlaneID: "plane-a"}, + {env: "beta", domain: "beta.example.test", controlPlaneID: "plane-b"}, } { t.Run(tc.env, func(t *testing.T) { resetControlPlaneProfileValidateCommand(t) @@ -208,7 +209,7 @@ func TestControlPlaneProfileExportCommandUsesSelectedEnvironmentDomain(t *testin } prevFetch := fetchControlPlaneRootCAPEM - fetchControlPlaneRootCAPEM = func(context.Context, string) (string, error) { + fetchControlPlaneRootCAPEM = func(context.Context, string, string) (string, error) { return "", nil } t.Cleanup(func() { fetchControlPlaneRootCAPEM = prevFetch }) @@ -241,6 +242,9 @@ func TestControlPlaneProfileExportCommandUsesSelectedEnvironmentDomain(t *testin assert.Equal(t, "https://sis."+tc.domain, profile.Endpoints.ComputeReachable.ICMSURL) assert.Equal(t, "https://reval."+tc.domain, profile.Endpoints.ComputeReachable.ReValURL) assert.Equal(t, "nats://nats."+tc.domain+":4222", profile.Endpoints.ComputeReachable.NATSURL) + assert.Equal(t, "http://api."+tc.controlPlaneID+"-sis.svc.cluster.local:8080", profile.Endpoints.InCluster.ICMSURL) + assert.Equal(t, "http://reval."+tc.controlPlaneID+"-nvcf.svc.cluster.local:8080", profile.Endpoints.InCluster.ReValURL) + assert.Equal(t, "nats://nats."+tc.controlPlaneID+"-nats-system.svc.cluster.local:4222", profile.Endpoints.InCluster.NATSURL) }) } } @@ -278,7 +282,7 @@ icms_host: sis.config.example.test require.NoError(t, os.WriteFile(filepath.Join(stackDir, "environments", "qa.yaml"), []byte("global:\n domain: stack.example.test\n"), 0o600)) prevFetch := fetchControlPlaneRootCAPEM - fetchControlPlaneRootCAPEM = func(context.Context, string) (string, error) { + fetchControlPlaneRootCAPEM = func(context.Context, string, string) (string, error) { return "", nil } t.Cleanup(func() { fetchControlPlaneRootCAPEM = prevFetch }) @@ -445,7 +449,7 @@ func TestWriteControlPlaneProfileSourcesOpenBaoRootCA(t *testing.T) { require.NoError(t, err) prevFetch := fetchControlPlaneRootCAPEM - fetchControlPlaneRootCAPEM = func(_ context.Context, kctx string) (string, error) { + fetchControlPlaneRootCAPEM = func(_ context.Context, kctx, _ string) (string, error) { assert.Equal(t, "cp-context", kctx) return rootCA, nil } @@ -475,6 +479,33 @@ func TestWriteControlPlaneProfileSourcesOpenBaoRootCA(t *testing.T) { assert.Equal(t, wantFingerprint, result.Profile.TransportTLS.TrustBundleFingerprint) } +func TestControlPlaneRootCAOpenBaoConfigUsesNamedScope(t *testing.T) { + for _, name := range []string{ + "NVCF_OPENBAO_URL", + "OPENBAO_URL", + "VAULT_ADDR", + "BAO_ADDR", + "NVCF_OPENBAO_NAMESPACE", + "NVCF_OPENBAO_SECRET_NAME", + "NVCF_CLUSTER_NAMESPACE", + } { + t.Setenv(name, "") + } + + legacy := controlPlaneRootCAOpenBaoConfig("legacy-context", "") + assert.Equal(t, "http://openbao-server.vault-system.svc.cluster.local:8200", legacy.OpenBaoURL) + assert.Equal(t, "vault-system", legacy.OpenBaoNamespace) + assert.Equal(t, "openbao-server-root-token", legacy.OpenBaoSecretName) + assert.Equal(t, "nvcf", legacy.ClusterNamespace) + + named := controlPlaneRootCAOpenBaoConfig("plane-a-context", "plane-a") + assert.Equal(t, "http://plane-a-openbao-server.plane-a-vault-system.svc.cluster.local:8200", named.OpenBaoURL) + assert.Equal(t, "plane-a-vault-system", named.OpenBaoNamespace) + assert.Equal(t, "plane-a-openbao-server-root-token", named.OpenBaoSecretName) + assert.Equal(t, "plane-a-nvcf", named.ClusterNamespace) + assert.Equal(t, "plane-a-context", named.KubeContext) +} + func TestRewriteURLHost(t *testing.T) { cases := []struct { name string diff --git a/src/clis/nvcf-cli/cmd/self_hosted_install_test.go b/src/clis/nvcf-cli/cmd/self_hosted_install_test.go index a3de40b7b..1a46fe8d3 100644 --- a/src/clis/nvcf-cli/cmd/self_hosted_install_test.go +++ b/src/clis/nvcf-cli/cmd/self_hosted_install_test.go @@ -50,7 +50,7 @@ func resetInstallFlags(t *testing.T) { return selfhosted.HelmRuntimeHelm3Legacy, nil } prevFetchRootCA := fetchControlPlaneRootCAPEM - fetchControlPlaneRootCAPEM = func(context.Context, string) (string, error) { + fetchControlPlaneRootCAPEM = func(context.Context, string, string) (string, error) { return "", nil } t.Cleanup(func() { diff --git a/src/clis/nvcf-cli/cmd/self_hosted_up_test.go b/src/clis/nvcf-cli/cmd/self_hosted_up_test.go index e59bcda39..4759f300e 100644 --- a/src/clis/nvcf-cli/cmd/self_hosted_up_test.go +++ b/src/clis/nvcf-cli/cmd/self_hosted_up_test.go @@ -70,7 +70,7 @@ func resetUpFlags(t *testing.T) { selfHostedUpCurrentKubeContext = func() (string, error) { return "k3d-ncp-local", nil } - fetchControlPlaneRootCAPEM = func(context.Context, string) (string, error) { + fetchControlPlaneRootCAPEM = func(context.Context, string, string) (string, error) { return "", nil } t.Cleanup(func() { diff --git a/src/clis/nvcf-cli/internal/openbao/client.go b/src/clis/nvcf-cli/internal/openbao/client.go index 1f3d24618..744772220 100644 --- a/src/clis/nvcf-cli/internal/openbao/client.go +++ b/src/clis/nvcf-cli/internal/openbao/client.go @@ -102,6 +102,21 @@ type pkiCertificateHTTPResponse struct { Body string } +type pkiCertificateRequestError struct { + response pkiCertificateHTTPResponse +} + +func (e *pkiCertificateRequestError) Error() string { + message := fmt.Sprintf("OpenBao PKI certificate request failed with HTTP %d", e.response.StatusCode) + if e.response.ContentType != "" { + message += fmt.Sprintf(" (content type %q)", e.response.ContentType) + } + if body := boundedOpenBaoHTTPErrorBody(e.response.Body); body != "" { + message += ": " + body + } + return message +} + // NewClient creates a new OpenBao client func NewClient(config *Config, k8sClient *k8s.Client) *Client { return &Client{ @@ -412,11 +427,38 @@ func (c *Client) generateUserJWTTokenWithSubject(ctx context.Context, vaultToken // text suitable for a public trust bundle. func (c *Client) ReadPKICertificatePEM(ctx context.Context, pkiPath string) (string, error) { readURL := strings.TrimRight(c.config.OpenBaoURL, "/") + "/v1/" + strings.Trim(pkiPath, "/") + "/cert/ca" + pem, err := c.readPKICertificatePEMWithToken(ctx, readURL, "") + var requestErr *pkiCertificateRequestError + if err == nil || !errors.As(err, &requestErr) || requestErr.response.StatusCode != http.StatusForbidden { + return pem, err + } + + // OpenBao returns 403 for an unauthenticated request when the optional PKI + // mount is absent, hiding the 404 that distinguishes "not enabled" from an + // authorization problem. Retry only that response with the configured root + // token so profile export can classify the missing optional mount correctly. + rootToken, tokenErr := c.getOpenBaoRootToken() + if tokenErr != nil { + return "", fmt.Errorf("%w; authenticated retry unavailable: %v", err, tokenErr) + } + pem, err = c.readPKICertificatePEMWithToken(ctx, readURL, rootToken) + if errors.As(err, &requestErr) && requestErr.response.StatusCode == http.StatusNotFound { + return "", fmt.Errorf("%w: authenticated OpenBao response returned HTTP 404", ErrPKICertificateNotFound) + } + return pem, err +} + +func (c *Client) readPKICertificatePEMWithToken(ctx context.Context, readURL, token string) (string, error) { writeOut := "\n" + curlHTTPStatusMarker + "%{http_code}\n" + curlHTTPContentTypeMarker + "%{content_type}\n" curlArgs := []string{"curl", "-sS", "--write-out", writeOut, readURL} + stdin := "" + if token != "" { + curlArgs = append(curlArgs, "-H", "@-") + stdin = "X-Vault-Token: " + token + "\n" + } return readPKICertificatePEM(ctx, 3, 2*time.Second, func(ctx context.Context) (pkiCertificateHTTPResponse, error) { - output, err := c.executeKubectlRun(ctx, "openbao-pki-root-ca", curlArgs) + output, err := c.executeKubectlRunWithInput(ctx, "openbao-pki-root-ca", curlArgs, stdin) if err != nil { return pkiCertificateHTTPResponse{}, err } @@ -497,14 +539,7 @@ func pkiCertificateHTTPResponseFromOutput(output string) (pkiCertificateHTTPResp } func pkiCertificateHTTPError(response pkiCertificateHTTPResponse) error { - message := fmt.Sprintf("OpenBao PKI certificate request failed with HTTP %d", response.StatusCode) - if response.ContentType != "" { - message += fmt.Sprintf(" (content type %q)", response.ContentType) - } - if body := boundedOpenBaoHTTPErrorBody(response.Body); body != "" { - message += ": " + body - } - return errors.New(message) + return &pkiCertificateRequestError{response: response} } func retryablePKICertificateHTTPStatus(statusCode int) bool { @@ -577,6 +612,10 @@ func (c *Client) kubectlBaseArgs() []string { // executeKubectlRun executes a kubectl run command with the utility image func (c *Client) executeKubectlRun(ctx context.Context, name string, args []string) (string, error) { + return c.executeKubectlRunWithInput(ctx, name, args, "") +} + +func (c *Client) executeKubectlRunWithInput(ctx context.Context, name string, args []string, stdin string) (string, error) { if ctx == nil { ctx = context.Background() } @@ -606,6 +645,9 @@ func (c *Client) executeKubectlRun(ctx context.Context, name string, args []stri // Execute the command cmd := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...) + if stdin != "" { + cmd.Stdin = strings.NewReader(stdin) + } output, err := cmd.CombinedOutput() if err != nil { if c.config.Debug { diff --git a/src/clis/nvcf-cli/internal/openbao/client_test.go b/src/clis/nvcf-cli/internal/openbao/client_test.go index de33bf242..7575f1084 100644 --- a/src/clis/nvcf-cli/internal/openbao/client_test.go +++ b/src/clis/nvcf-cli/internal/openbao/client_test.go @@ -335,3 +335,147 @@ printf '%s\n' '__NVCF_HTTP_CONTENT_TYPE__:application/json' assert.Contains(t, commands, "/v1/services/all/pki/root/cert/ca") assert.Contains(t, commands, "--write-out") } + +func TestReadPKICertificatePEMRetriesForbiddenPublicEndpointWithRootToken(t *testing.T) { + testDir := t.TempDir() + commandLog := filepath.Join(testDir, "kubectl.log") + kubectlPath := filepath.Join(testDir, "kubectl") + kubectlScript := `#!/bin/sh +printf '%s\n' "$*" >> "$KUBECTL_COMMAND_LOG" +case " $* " in + *" get secret "*) printf '%s\n' 'c3VwZXItc2VjcmV0LXRva2Vu' ;; + *" -H @- "*) + header=$(cat) + [ "$header" = "X-Vault-Token: super-secret-token" ] || exit 93 + printf '%s\n' '{"data":{"certificate":"-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n"}}' + printf '%s\n' '__NVCF_HTTP_STATUS__:200' + printf '%s\n' '__NVCF_HTTP_CONTENT_TYPE__:application/json' + ;; + *) + printf '%s\n' '{"errors":["permission denied"]}' + printf '%s\n' '__NVCF_HTTP_STATUS__:403' + printf '%s\n' '__NVCF_HTTP_CONTENT_TYPE__:application/json' + ;; +esac +` + require.NoError(t, os.WriteFile(kubectlPath, []byte(kubectlScript), 0o755)) + t.Setenv("PATH", testDir+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("KUBECTL_COMMAND_LOG", commandLog) + + client := NewClient(&Config{ + OpenBaoURL: "http://openbao-openbao.nvcf.svc.cluster.local:8200", + OpenBaoNamespace: "openbao", + OpenBaoSecretName: "openbao-root-token", + ClusterNamespace: "nvcf", + UtilityImage: "curlimages/curl:latest", + }, nil) + + got, err := client.ReadPKICertificatePEM(context.Background(), "services/all/pki/root") + require.NoError(t, err) + assert.Equal(t, openBaoTestCertPEM, got) + + logBody, err := os.ReadFile(commandLog) + require.NoError(t, err) + commands := string(logBody) + assert.Contains(t, commands, "get secret openbao-root-token ") + assert.Contains(t, commands, "-H @-") + assert.NotContains(t, commands, "super-secret-token") + assert.Less(t, strings.Index(commands, "/cert/ca"), strings.Index(commands, "get secret openbao-root-token")) + assert.Less(t, strings.Index(commands, "get secret openbao-root-token"), strings.LastIndex(commands, "/cert/ca")) +} + +func TestReadPKICertificatePEMUsesAuthenticatedNotFoundAfterPublicForbidden(t *testing.T) { + testDir := t.TempDir() + kubectlPath := filepath.Join(testDir, "kubectl") + kubectlScript := `#!/bin/sh +case " $* " in + *" get secret "*) printf '%s\n' 'c3VwZXItc2VjcmV0LXRva2Vu' ;; + *" -H @- "*) + header=$(cat) + [ "$header" = "X-Vault-Token: super-secret-token" ] || exit 93 + printf '%s\n' '{"errors":["route unavailable"]}' + printf '%s\n' '__NVCF_HTTP_STATUS__:404' + printf '%s\n' '__NVCF_HTTP_CONTENT_TYPE__:application/json' + ;; + *) + printf '%s\n' '{"errors":["permission denied"]}' + printf '%s\n' '__NVCF_HTTP_STATUS__:403' + printf '%s\n' '__NVCF_HTTP_CONTENT_TYPE__:application/json' + ;; +esac +` + require.NoError(t, os.WriteFile(kubectlPath, []byte(kubectlScript), 0o755)) + t.Setenv("PATH", testDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + client := NewClient(&Config{ + OpenBaoURL: "http://openbao-openbao.nvcf.svc.cluster.local:8200", + OpenBaoNamespace: "openbao", + OpenBaoSecretName: "openbao-root-token", + ClusterNamespace: "nvcf", + UtilityImage: "curlimages/curl:latest", + }, nil) + + _, err := client.ReadPKICertificatePEM(context.Background(), "services/all/pki/root") + require.Error(t, err) + assert.ErrorIs(t, err, ErrPKICertificateNotFound) +} + +func TestReadPKICertificatePEMPreservesForbiddenWhenRootSecretUnavailable(t *testing.T) { + testDir := t.TempDir() + kubectlPath := filepath.Join(testDir, "kubectl") + kubectlScript := `#!/bin/sh +case " $* " in + *" get secret "*) exit 91 ;; + *) + printf '%s\n' '{"errors":["permission denied"]}' + printf '%s\n' '__NVCF_HTTP_STATUS__:403' + printf '%s\n' '__NVCF_HTTP_CONTENT_TYPE__:application/json' + ;; +esac +` + require.NoError(t, os.WriteFile(kubectlPath, []byte(kubectlScript), 0o755)) + t.Setenv("PATH", testDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + client := NewClient(&Config{ + OpenBaoURL: "http://openbao-openbao.nvcf.svc.cluster.local:8200", + OpenBaoNamespace: "openbao", + OpenBaoSecretName: "openbao-root-token", + ClusterNamespace: "nvcf", + UtilityImage: "curlimages/curl:latest", + }, nil) + + _, err := client.ReadPKICertificatePEM(context.Background(), "services/all/pki/root") + require.Error(t, err) + assert.ErrorContains(t, err, "HTTP 403") + assert.ErrorContains(t, err, "authenticated retry unavailable") + assert.NotContains(t, err.Error(), "super-secret-token") +} + +func TestReadPKICertificatePEMReturnsAuthenticatedForbidden(t *testing.T) { + testDir := t.TempDir() + kubectlPath := filepath.Join(testDir, "kubectl") + kubectlScript := `#!/bin/sh +case " $* " in + *" get secret "*) printf '%s\n' 'c3VwZXItc2VjcmV0LXRva2Vu' ;; + *" -H @- "*) cat >/dev/null ;; +esac +printf '%s\n' '{"errors":["permission denied"]}' +printf '%s\n' '__NVCF_HTTP_STATUS__:403' +printf '%s\n' '__NVCF_HTTP_CONTENT_TYPE__:application/json' +` + require.NoError(t, os.WriteFile(kubectlPath, []byte(kubectlScript), 0o755)) + t.Setenv("PATH", testDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + client := NewClient(&Config{ + OpenBaoURL: "http://openbao-openbao.nvcf.svc.cluster.local:8200", + OpenBaoNamespace: "openbao", + OpenBaoSecretName: "openbao-root-token", + ClusterNamespace: "nvcf", + UtilityImage: "curlimages/curl:latest", + }, nil) + + _, err := client.ReadPKICertificatePEM(context.Background(), "services/all/pki/root") + require.Error(t, err) + assert.ErrorContains(t, err, "HTTP 403") + assert.NotContains(t, err.Error(), "super-secret-token") +} diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/README.md b/src/compute-plane-services/nvca/deployments/nvca-operator/README.md index f1135082c..6b5e9a6a7 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/README.md +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/README.md @@ -29,6 +29,7 @@ used in Kubernetes Clusters to run NVCF Workloads. | `generateImagePullSecret` | Use the ngcConfig.serviceKey to generate an image pull secret for nvca and nvca-operator Pods | `true` | | `imagePullSecretName` | Name of the image pull secret to use for nvca and nvca-operator Pods. | `nvca-operator-image-pull` | | `imagePullSecrets` | List of pre-existing imagePullSecret objects in the nvca-operator namespace to use for nvca and nvca-operator Pods. Each object must have a 'name' field. Example: [{name: "foo-bar"}, {name: "baz"}] | `[]` | +| `controlPlane.id` | Optional lowercase DNS label identifying an isolated control plane. Empty preserves legacy names; a value such as `plane-a` requires release and namespace `plane-a-nvca-operator`. | `""` | | `serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | | `serviceAccount.annotations` | Additional custom annotations for the ServiceAccount | `{}` | | `serviceAccount.name` | The name of the ServiceAccount to use. | `""` | @@ -77,7 +78,7 @@ used in Kubernetes Clusters to run NVCF Workloads. | `agent.cacheMountOptionsEnabled` | Enable or disable CSI volume mount options for NVCA caches | `true` | | `agent.cacheMountOptions` | Comma-separated string of CSI volume mount options (e.g., "ro,noatime,nouuid") used when cacheMountOptionsEnabled is true | `ro,norecovery,nouuid` | | `agent.workerDegradationPeriod` | Duration for determining if a worker is degraded (e.g., "90m", "1h30m") | `""` | -| `agent.secretMirrorNamespace` | Default namespace to mirror custom secrets for nvcf workloads | `nvca-operator` | +| `agent.secretMirrorNamespace` | Namespace to source mirrored workload secrets from; empty derives this control plane's operator namespace. A custom namespace is an intentional shared/alternate source and must be access-controlled. | `""` | | `agent.secretMirrorLabelSelector` | Label selector on the secrets in the sourceNamespace | `""` | | `agent.customAnnotations` | Map of custom annotations to add to the agent pod | `{}` | | `agent.gpuProfiling.functionIds` | Comma/space/newline-separated NVCF function IDs (or "*" for all) whose pods NVCA labels for NVIDIA Nsight GPU profiling. Empty disables profiling. | `""` | diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/_helpers.tpl b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/_helpers.tpl index 1e093ec48..6c4ff1b77 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/_helpers.tpl +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/_helpers.tpl @@ -22,13 +22,64 @@ Expand the name of the chart. {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Validated control-plane identity. Empty is the legacy compatibility mode. +*/}} +{{- define "nvcaop.controlPlaneID" -}} +{{- $controlPlane := .Values.controlPlane | default dict -}} +{{- $id := $controlPlane.id | default "" -}} +{{- if eq $id "default" -}} +{{- fail "controlPlane.id \"default\" is reserved" -}} +{{- end -}} +{{- if and $id (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $id)) -}} +{{- fail (printf "controlPlane.id must be a lowercase RFC 1123 DNS label, got %q" $id) -}} +{{- end -}} +{{- if gt (len $id) 20 -}} +{{- fail "controlPlane.id must be at most 20 characters" -}} +{{- end -}} +{{- if $id -}} +{{- $expectedOperatorName := printf "%s-nvca-operator" $id -}} +{{- if ne .Release.Name $expectedOperatorName -}} +{{- fail (printf "controlPlane.id=%q requires Helm release name %q" $id $expectedOperatorName) -}} +{{- end -}} +{{- if ne .Release.Namespace $expectedOperatorName -}} +{{- fail (printf "controlPlane.id=%q requires Helm release namespace %q" $id $expectedOperatorName) -}} +{{- end -}} +{{- end -}} +{{- $id -}} +{{- end -}} + +{{/* Namespaces derived from the control-plane identity. */}} +{{- define "nvcaop.operatorNamespace" -}} +{{- $id := include "nvcaop.controlPlaneID" . -}} +{{- if $id -}}{{ printf "%s-nvca-operator" $id }}{{- else -}}nvca-operator{{- end -}} +{{- end -}} + +{{- define "nvcaop.systemNamespace" -}} +{{- $id := include "nvcaop.controlPlaneID" . -}} +{{- if $id -}}{{ printf "%s-nvca-system" $id }}{{- else -}}nvca-system{{- end -}} +{{- end -}} + +{{- define "nvcaop.requestsNamespace" -}} +{{- $id := include "nvcaop.controlPlaneID" . -}} +{{- if $id -}}{{ printf "%s-nvcf-backend" $id }}{{- else -}}nvcf-backend{{- end -}} +{{- end -}} + +{{/* Secret mirror source defaults to this control plane's operator namespace. */}} +{{- define "nvcaop.secretMirrorNamespace" -}} +{{- .Values.agent.secretMirrorNamespace | default (include "nvcaop.operatorNamespace" .) -}} +{{- end -}} + {{/* Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} {{- define "nvcaop.fullname" -}} -{{- if .Values.fullnameOverride -}} +{{- $id := include "nvcaop.controlPlaneID" . -}} +{{- if $id -}} +{{- printf "%s-nvca-operator" $id -}} +{{- else if .Values.fullnameOverride -}} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} {{- else -}} {{- $name := default .Chart.Name .Values.nameOverride -}} @@ -54,6 +105,10 @@ Common labels helm.sh/chart: {{ include "nvcaop.chart" . }} app.kubernetes.io/name: {{ include "nvcaop.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} +{{- $controlPlaneID := include "nvcaop.controlPlaneID" . }} +{{- if $controlPlaneID }} +nvcf.nvidia.com/control-plane-id: {{ $controlPlaneID | quote }} +{{- end }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/crds/nvidia.io_nvcfbackends_crd.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/crds/nvidia.io_nvcfbackends_crd.yaml index cb0d433ab..c85df2a5f 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/crds/nvidia.io_nvcfbackends_crd.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/crds/nvidia.io_nvcfbackends_crd.yaml @@ -13,10 +13,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +{{- $controlPlaneID := include "nvcaop.controlPlaneID" . -}} +{{- if not $controlPlaneID -}} +{{- $existing := lookup "apiextensions.k8s.io/v1" "CustomResourceDefinition" "" "nvcfbackends.nvcf.nvidia.io" -}} +{{- $ownedByThisRelease := false -}} +{{- with $existing -}} +{{- $annotations := .metadata.annotations | default dict -}} +{{- $ownedByThisRelease = and + (eq (get $annotations "meta.helm.sh/release-name") $.Release.Name) + (eq (get $annotations "meta.helm.sh/release-namespace") $.Release.Namespace) -}} +{{- end -}} +{{- if or (not $existing) $ownedByThisRelease }} apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: nvcfbackends.nvcf.nvidia.io + annotations: + helm.sh/resource-policy: keep labels: {{- include "nvcaop.labels" . | nindent 4 }} spec: @@ -58,3 +71,5 @@ spec: storage: true subresources: status: {} +{{- end }} +{{- end }} diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/deployment.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/deployment.yaml index df774a2e6..9a56cfbe9 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/deployment.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/deployment.yaml @@ -129,6 +129,11 @@ spec: value: /var/run/secrets/ngc-service-key/{{ default "ngcServiceKey" .Values.ngcConfig.serviceKeySecretKeyName }} - name: NVCA_CLUSTER_SOURCE value: {{ .Values.ngcConfig.clusterSource | default "ngc-managed" }} + {{- $controlPlaneID := include "nvcaop.controlPlaneID" . }} + {{- if $controlPlaneID }} + - name: NVCF_CONTROL_PLANE_ID + value: {{ $controlPlaneID | quote }} + {{- end }} {{- if and .Values.vaultConfig .Values.vaultConfig.oAuthClientMountPathTemplate }} - name: VAULT_OAUTH_CLIENT_MOUNT_PATH_TEMPLATE value: {{ .Values.vaultConfig.oAuthClientMountPathTemplate | quote }} @@ -216,6 +221,10 @@ spec: - "{{ .Values.nvcaHelmRepositoryPrefix}}" - --cluster-id - "{{ .Values.clusterID }}" + {{- if $controlPlaneID }} + - --control-plane-id + - {{ $controlPlaneID | quote }} + {{- end }} {{- if .Values.enableGXCache }} - --enable-gxcache {{- end}} @@ -232,11 +241,14 @@ spec: - --nvca-worker-degradation-period - {{ .Values.agent.workerDegradationPeriod | quote }} {{- end }} - {{- if and ((.Values.agent).secretMirrorLabelSelector) ((.Values.agent).secretMirrorNamespace) }} + {{- $secretMirrorNamespace := include "nvcaop.secretMirrorNamespace" . }} + {{- if $secretMirrorNamespace }} - --nvca-secret-mirror-source-namespace - - "{{ .Values.agent.secretMirrorNamespace }}" + - {{ $secretMirrorNamespace | quote }} + {{- end }} + {{- if ((.Values.agent).secretMirrorLabelSelector) }} - --nvca-secret-mirror-label-selector - - "{{ .Values.agent.secretMirrorLabelSelector }}" + - {{ .Values.agent.secretMirrorLabelSelector | quote }} {{- end }} {{- $agent := .Values.agent | default dict }} {{- $byooOtelCollectorImage := include "nvcaop.byooOtelCollectorImage" . }} @@ -315,7 +327,7 @@ spec: - /usr/bin/nvca-mirror - run - --target-namespace - - "nvca-system" + - {{ include "nvcaop.systemNamespace" . | quote }} - --log-level - "{{ .Values.logLevel }}" env: diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/image-pull-secret.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/image-pull-secret.yaml index cc3d9128d..c5f037110 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/image-pull-secret.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/image-pull-secret.yaml @@ -18,6 +18,7 @@ apiVersion: v1 kind: Secret metadata: name: {{ default "nvca-operator-image-pull" .Values.imagePullSecretName }} + namespace: {{ .Release.Namespace }} labels: {{- include "nvcaop.labels" . | nindent 4 }} type: kubernetes.io/dockerconfigjson diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/ngc-service-key.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/ngc-service-key.yaml index 3468ffe67..ee75390ed 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/ngc-service-key.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/ngc-service-key.yaml @@ -18,6 +18,7 @@ apiVersion: v1 kind: Secret metadata: name: ngc-service-key + namespace: {{ .Release.Namespace }} labels: {{- include "nvcaop.labels" . | nindent 4 }} data: diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/nvca-operator_rq.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/nvca-operator_rq.yaml index 24f09df69..339b55ea7 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/nvca-operator_rq.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/nvca-operator_rq.yaml @@ -16,8 +16,8 @@ apiVersion: v1 kind: ResourceQuota metadata: - name: nvca-operator - namespace: nvca-operator + name: {{ include "nvcaop.fullname" . }} + namespace: {{ .Release.Namespace }} spec: scopeSelector: matchExpressions: diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/pre-delete-cleanup-rbac.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/pre-delete-cleanup-rbac.yaml index 51c2e07ca..915fe1f21 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/pre-delete-cleanup-rbac.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/pre-delete-cleanup-rbac.yaml @@ -24,7 +24,7 @@ metadata: annotations: "helm.sh/hook": pre-delete "helm.sh/hook-weight": "-20" - "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded automountServiceAccountToken: false --- apiVersion: rbac.authorization.k8s.io/v1 @@ -37,7 +37,7 @@ metadata: annotations: "helm.sh/hook": pre-delete "helm.sh/hook-weight": "-20" - "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded rules: - apiGroups: ["nvcf.nvidia.io"] resources: ["nvcfbackends", "nvcfbackends/finalizers", "nvcfbackends/status"] diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/self-managed-nvcfbackend-cm.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/self-managed-nvcfbackend-cm.yaml index 9e304f860..43acf639f 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/self-managed-nvcfbackend-cm.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/self-managed-nvcfbackend-cm.yaml @@ -22,6 +22,12 @@ metadata: {{- if eq .Values.ngcConfig.clusterSource "self-managed" }} data: cluster-dto.yaml: | + {{- $controlPlaneID := include "nvcaop.controlPlaneID" . }} + {{- if $controlPlaneID }} + controlPlaneID: {{ $controlPlaneID | quote }} + systemNamespace: {{ include "nvcaop.systemNamespace" . | quote }} + requestsNamespace: {{ include "nvcaop.requestsNamespace" . | quote }} + {{- end }} clusterId: {{ .Values.clusterID | quote }} clusterGroupId: {{ .Values.clusterGroupID | quote }} clusterName: {{ .Values.clusterName | default "nvcf-default" | quote }} diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/values.schema.json b/src/compute-plane-services/nvca/deployments/nvca-operator/values.schema.json index 4989f9a22..8c9530044 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/values.schema.json +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/values.schema.json @@ -108,6 +108,22 @@ "default": [], "items": {} }, + "controlPlane": { + "type": "object", + "description": "Optional identity used to isolate multiple control-plane compute agents in one Kubernetes cluster.", + "properties": { + "id": { + "type": "string", + "description": "Lowercase DNS label prefix. Empty preserves legacy resource names and namespaces.", + "default": "", + "maxLength": 20, + "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", + "not": { + "const": "default" + } + } + } + }, "serviceAccount": { "type": "object", "properties": { @@ -392,8 +408,8 @@ }, "secretMirrorNamespace": { "type": "string", - "description": "Default namespace to mirror custom secrets for nvcf workloads", - "default": "nvca-operator" + "description": "Namespace to source mirrored workload secrets from. Empty derives this control plane's operator namespace.", + "default": "" }, "secretMirrorLabelSelector": { "type": "string", diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml index df8e99acd..b08e66687 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/values.yaml @@ -55,6 +55,10 @@ imagePullSecretName: "nvca-operator-image-pull" ## @param imagePullSecrets List of pre-existing imagePullSecret objects in the nvca-operator namespace to use for nvca and nvca-operator Pods. Each object must have a 'name' field. Example: [{name: "foo-bar"}, {name: "baz"}] imagePullSecrets: [] +## @param controlPlane.id Optional control-plane identity for running multiple isolated NVCA operators in one Kubernetes cluster. Empty preserves the legacy resource names and namespaces. +controlPlane: + id: "" + ## Service Account configuration serviceAccount: ## @param serviceAccount.create Specifies whether a ServiceAccount should be created @@ -186,7 +190,7 @@ resources: ## @param agent.cacheMountOptionsEnabled Enable or disable CSI volume mount options for NVCA caches ## @param agent.cacheMountOptions Comma-separated string of CSI volume mount options (e.g., "ro,noatime,nouuid") used when cacheMountOptionsEnabled is true ## @param agent.workerDegradationPeriod Duration for determining if a worker is degraded (e.g., "90m", "1h30m") -## @param agent.secretMirrorNamespace Default namespace to mirror custom secrets for nvcf workloads +## @param agent.secretMirrorNamespace Namespace to source mirrored workload secrets from. Empty derives this control plane's operator namespace. ## @param agent.secretMirrorLabelSelector Label selector on the secrets in the sourceNamespace ## @param agent.customAnnotations Map of custom annotations to add to the agent pod ## @param agent.gpuProfiling.functionIds Comma/space/newline-separated NVCF function IDs (or "*" for all) whose pods NVCA labels for NVIDIA Nsight GPU profiling. Empty disables profiling. The operator creates and mirrors the nvca-gpu-profiling-config ConfigMap from this value at deploy/upgrade time. @@ -203,7 +207,7 @@ agent: cacheMountOptionsEnabled: true cacheMountOptions: "ro,norecovery,nouuid" workerDegradationPeriod: "" - secretMirrorNamespace: nvca-operator + secretMirrorNamespace: "" secretMirrorLabelSelector: "" customAnnotations: {} ## GPU (Nsight) profiling opt-in: NVCF function IDs to profile ("*" = all) and an optional diff --git a/src/compute-plane-services/nvca/internal/miniservice/BUILD.bazel b/src/compute-plane-services/nvca/internal/miniservice/BUILD.bazel index b903ca03b..5289daa1b 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/BUILD.bazel +++ b/src/compute-plane-services/nvca/internal/miniservice/BUILD.bazel @@ -109,6 +109,7 @@ go_library( "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/controller-runtime/pkg/handler", "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/controller-runtime/pkg/log", "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/controller-runtime/pkg/manager", + "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/controller-runtime/pkg/predicate", "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/controller-runtime/pkg/reconcile", "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/yaml", ], @@ -124,6 +125,7 @@ go_test( name = "miniservice_test", srcs = [ "controller_test.go", + "control_plane_isolation_test.go", "gvkcache_test.go", "metadata_configmap_test.go", "mutate_test.go", diff --git a/src/compute-plane-services/nvca/internal/miniservice/control_plane_isolation_test.go b/src/compute-plane-services/nvca/internal/miniservice/control_plane_isolation_test.go new file mode 100644 index 000000000..b0471c873 --- /dev/null +++ b/src/compute-plane-services/nvca/internal/miniservice/control_plane_isolation_test.go @@ -0,0 +1,42 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package mscontroller + +import ( + "testing" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" +) + +func TestMiniServiceEventHandlerScopesControlPlane(t *testing.T) { + owned := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{ + miniserviceNameLabel: "plane-a-request-miniservice", + nvcatypes.ControlPlaneIDLabel: "plane-a", + }}} + foreign := owned.DeepCopy() + foreign.Labels[nvcatypes.ControlPlaneIDLabel] = "plane-b" + + assert.Len(t, miniServiceRequestsForObject(owned, "plane-a"), 1) + assert.Empty(t, miniServiceRequestsForObject(foreign, "plane-a")) +} + +func TestControlPlaneObjectPredicate(t *testing.T) { + p := controlPlaneObjectPredicate("plane-a") + owned := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{ + nvcatypes.ControlPlaneIDLabel: "plane-a", + }}} + foreign := owned.DeepCopy() + foreign.Labels[nvcatypes.ControlPlaneIDLabel] = "plane-b" + + assert.True(t, p(client.Object(owned))) + assert.False(t, p(client.Object(foreign))) + assert.True(t, controlPlaneObjectPredicate("")(client.Object(foreign))) +} diff --git a/src/compute-plane-services/nvca/internal/miniservice/controller.go b/src/compute-plane-services/nvca/internal/miniservice/controller.go index 1e744691b..c6844f71d 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/controller.go +++ b/src/compute-plane-services/nvca/internal/miniservice/controller.go @@ -51,6 +51,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/metrics" @@ -104,6 +105,7 @@ func getKartaAddToScheme() func(*runtime.Scheme) error { } type ControllerOptions struct { + ControlPlaneID string SystemNamespace string ICMSRequestNamespace string K8sVersion string @@ -259,10 +261,13 @@ func BuildController(ctx context.Context, return fmt.Errorf("failed to set status event indices on manager: %w", err) } + controlPlanePredicate := predicate.NewPredicateFuncs(controlPlaneObjectPredicate(r.ControlPlaneID)) + miniserviceLabelEventHandler := newMiniServiceLabelEventHandler(r.ControlPlaneID) b := builder. ControllerManagedBy(mgr). Named("miniservice_controller"). - For(&nvcav1alpha1.MiniService{}). + For(&nvcav1alpha1.MiniService{}, builder.WithPredicates(controlPlanePredicate)). + WithEventFilter(controlPlanePredicate). // Labels will be set on these objects and their children // in order to capture events by un-owned objects by the same type. Watches(&nvcav1.StorageRequest{}, miniserviceLabelEventHandler). @@ -300,7 +305,16 @@ func getMiniServiceNameFromLabel(obj client.Object) string { return labels[miniserviceNameLabel] } -var miniserviceLabelEventHandler = handler.EnqueueRequestsFromMapFunc(func(_ context.Context, obj client.Object) []reconcile.Request { +func controlPlaneObjectPredicate(controlPlaneID string) func(client.Object) bool { + return func(obj client.Object) bool { + return types.IsOwnedByControlPlane(obj, controlPlaneID) + } +} + +func miniServiceRequestsForObject(obj client.Object, controlPlaneID string) []reconcile.Request { + if !types.IsOwnedByControlPlane(obj, controlPlaneID) { + return nil + } msName := getMiniServiceNameFromLabel(obj) if msName == "" { return nil @@ -308,7 +322,13 @@ var miniserviceLabelEventHandler = handler.EnqueueRequestsFromMapFunc(func(_ con req := reconcile.Request{} req.Name = msName return []reconcile.Request{req} -}) +} + +func newMiniServiceLabelEventHandler(controlPlaneID string) handler.EventHandler { + return handler.EnqueueRequestsFromMapFunc(func(_ context.Context, obj client.Object) []reconcile.Request { + return miniServiceRequestsForObject(obj, controlPlaneID) + }) +} const ( eventInvObjPrefix = "involvedObject." diff --git a/src/compute-plane-services/nvca/internal/miniservice/prereqs.go b/src/compute-plane-services/nvca/internal/miniservice/prereqs.go index b49b366ce..871bb4cd3 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/prereqs.go +++ b/src/compute-plane-services/nvca/internal/miniservice/prereqs.go @@ -352,6 +352,7 @@ func (r *Reconciler) ensureImageCredentialUpdaterObjects( jobLabels := map[string]string{ miniserviceNameLabel: ms.Name, } + jobLabels = nvcatypes.AddControlPlaneLabel(jobLabels, r.ControlPlaneID) tprUpdaterInitJob.Labels = jobLabels tprUpdaterInitJob.Spec.Template.Labels = jobLabels // Use NVCA's service account to run the job for API access and image pull secrets. diff --git a/src/compute-plane-services/nvca/internal/miniservice/reconcile.go b/src/compute-plane-services/nvca/internal/miniservice/reconcile.go index d0d5625fa..c91403432 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/reconcile.go +++ b/src/compute-plane-services/nvca/internal/miniservice/reconcile.go @@ -164,6 +164,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco } return reconcile.Result{}, err } + if !nvcatypes.IsOwnedByControlPlane(ms, r.ControlPlaneID) { + log.V(1).Info("MiniService belongs to another control plane; ignoring", "controlPlaneID", r.ControlPlaneID) + return reconcile.Result{}, nil + } if ms.Spec.Namespace == "" { return reconcile.Result{}, reconcile.TerminalError( @@ -1108,6 +1112,9 @@ func (r *Reconciler) ensureInstanceNamespace( existingNamespace := &corev1.Namespace{} if err := r.Client.Get(ctx, client.ObjectKeyFromObject(namespace), existingNamespace); err == nil { + if !nvcatypes.IsOwnedByControlPlane(existingNamespace, r.ControlPlaneID) { + return fmt.Errorf("namespace %s is not owned by control plane %q", namespaceName, r.ControlPlaneID) + } if existingNamespace.Status.Phase == corev1.NamespaceTerminating { log.V(1).Info("Namespace is terminating, will requeue and wait for deletion to complete", "namespace", namespaceName) return nil @@ -1116,6 +1123,7 @@ func (r *Reconciler) ensureInstanceNamespace( if !mapContainsAll(namespace.Labels, existingNamespace.Labels) || !mapContainsAll(namespace.Annotations, existingNamespace.Annotations) { log.Info("Updating namespace with missing metadata") + namespace.ResourceVersion = existingNamespace.ResourceVersion if err := r.Client.Update(ctx, namespace); err != nil { return err } diff --git a/src/compute-plane-services/nvca/internal/miniservice/reconcile_storagerequests.go b/src/compute-plane-services/nvca/internal/miniservice/reconcile_storagerequests.go index 49d5760b2..cab033a65 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/reconcile_storagerequests.go +++ b/src/compute-plane-services/nvca/internal/miniservice/reconcile_storagerequests.go @@ -148,7 +148,10 @@ func (r *Reconciler) makeStorageRequests( // injection, so a request either caches on every backend or on none. switch backend { case nvcastorage.HelmCacheBackendNVMesh, nvcastorage.HelmCacheBackendSharedFS, nvcastorage.HelmCacheBackendSamba: - if cacheInitJob != nil && cacheInitPVC != nil && cacheLaunchRequested(icmsReq) { + // Model-cache plumbing still owns cluster-scoped PVs and a historical + // singleton init namespace. Disable it in named multi-control-plane mode + // until those resources can be isolated without cross-plane data risk. + if r.ControlPlaneID == "" && cacheInitJob != nil && cacheInitPVC != nil && cacheLaunchRequested(icmsReq) { st, err := nvcastorage.NewModelCacheStorageRequest(icmsReq, r.FeatureFlagFetcher) if err != nil { // Invalid cache spec is not retryable; keep it terminal (the diff --git a/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/generated.openapi.go b/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/generated.openapi.go index 6173881e4..ed0281757 100644 --- a/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/generated.openapi.go +++ b/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/generated.openapi.go @@ -131,6 +131,13 @@ func schema_pkg_apis_nvcf_v1_ClusterConfig(ref common.ReferenceCallback) common. SchemaProps: spec.SchemaProps{ Type: []string{"object"}, Properties: map[string]spec.Schema{ + "controlPlaneId": { + SchemaProps: spec.SchemaProps{ + Description: "ControlPlaneID is the stable identity used to isolate multiple NVCF control planes sharing one Kubernetes cluster. Empty preserves legacy names.", + Type: []string{"string"}, + Format: "", + }, + }, "clusterID": { SchemaProps: spec.SchemaProps{ Description: "required", diff --git a/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/nvcfbackend_types.go b/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/nvcfbackend_types.go index afed44f30..7e303397e 100644 --- a/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/nvcfbackend_types.go +++ b/src/compute-plane-services/nvca/pkg/apis/nvcf/v1/nvcfbackend_types.go @@ -74,6 +74,9 @@ type AccountConfig struct { // +k8s:openapi-gen=true type ClusterConfig struct { + // ControlPlaneID is the stable identity used to isolate multiple NVCF + // control planes sharing one Kubernetes cluster. Empty preserves legacy names. + ControlPlaneID string `json:"controlPlaneId,omitempty"` // required ClusterID string `json:"clusterID,omitempty"` ClusterName string `json:"clusterName"` diff --git a/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel b/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel index 84ab93aad..3a5db89d2 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel @@ -180,6 +180,7 @@ go_test( "backendk8scache_gxcache_test.go", "backendk8scache_test.go", "cli_test.go", + "control_plane_isolation_test.go", "encrypt_modelcache_test.go", "gpumonitor_test.go", "gpu_registration_manager_test.go", @@ -315,6 +316,7 @@ go_test( "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/kubernetes/fake", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/listers/core/v1:core", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/rest", + "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/testing", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/tools/cache", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/tools/record", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/util/workqueue", diff --git a/src/compute-plane-services/nvca/pkg/nvca/agent.go b/src/compute-plane-services/nvca/pkg/nvca/agent.go index adaaf3f3b..b0f0799c4 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/agent.go +++ b/src/compute-plane-services/nvca/pkg/nvca/agent.go @@ -168,6 +168,7 @@ type AgentOptions struct { NCAId string ClusterName string + ControlPlaneID string ClusterID string ClusterDescription string ClusterGroupName string @@ -503,6 +504,22 @@ func (o *AgentOptions) GetOTelAttributes() []otelattr.KeyValue { } func NewAgent(ctx context.Context, opts *AgentOptions) (*Agent, error) { + if err := types.ValidateControlPlaneID(opts.ControlPlaneID); err != nil { + return nil, fmt.Errorf("invalid control plane ID: %w", err) + } + if opts.ControlPlaneID != "" { + if opts.SystemNamespace == "" { + opts.SystemNamespace = types.ControlPlaneResourceName(opts.ControlPlaneID, SystemNamespace) + } + if opts.RequestsNamespace == "" { + opts.RequestsNamespace = types.ControlPlaneResourceName(opts.ControlPlaneID, RequestsNamespace) + } + labelsCopy := labels.Set{} + for key, value := range opts.NamespaceLabels { + labelsCopy[key] = value + } + opts.NamespaceLabels = types.AddControlPlaneLabel(labelsCopy, opts.ControlPlaneID) + } log := core.GetLogger(ctx) if opts.EffectiveICMSURL() == "" { return nil, errors.New("ICMSURL required for Agent") @@ -1142,6 +1159,7 @@ func (a *Agent) Start(ctx context.Context) error { log.Info("Configuring backendk8scache") backendk8scache, _, err := a.newBackendK8sCacheBuilder(). WithConfig(a.Config). + WithControlPlaneID(a.ControlPlaneID). WithClusterProvider(a.CloudProvider). WithClusterRegion(a.ClusterRegion). WithClusterName(a.ClusterName). diff --git a/src/compute-plane-services/nvca/pkg/nvca/agent_manager.go b/src/compute-plane-services/nvca/pkg/nvca/agent_manager.go index 477bbc9c9..7f5c04ae6 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/agent_manager.go +++ b/src/compute-plane-services/nvca/pkg/nvca/agent_manager.go @@ -29,6 +29,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" ctrl "sigs.k8s.io/controller-runtime" @@ -47,6 +48,8 @@ import ( mscontroller "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/miniservice" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/util/k8sutil" nvcav1new "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v1" + nvcav1alpha1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v1alpha1" + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/featureflag" nvcaerrors "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/errors" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" @@ -62,6 +65,12 @@ func init() { utilruntime.Must(mscontroller.SchemeBuilder.AddToScheme(mgrScheme)) } +// storageControllerTypes preserves the legacy helper contract used by tests. +// Named deployments call storage.ControllerTypes directly with their ID. +func storageControllerTypes(cachingEnabled bool) []nvcav1new.StorageRequestType { + return storage.ControllerTypes(cachingEnabled, "") +} + // startControllerManagerForAgent creates and starts a controller-runtime manager // for auxiliary CRD controllers. // storageControllerTypes returns the StorageRequest controller types to @@ -70,17 +79,6 @@ func init() { // selected cache backends (NVMesh / shared-FS / Samba). Callers pass the stable // agent-level caching flag (a.CachingSupportEnabled), not a live feature-flag // lookup, so the set is deterministic for the lifetime of the agent. -func storageControllerTypes(cachingEnabled bool) []nvcav1new.StorageRequestType { - sts := []nvcav1new.StorageRequestType{ - nvcav1new.SharedStorageRequest, - nvcav1new.InternalPersistentStorageRequest, - } - if cachingEnabled { - sts = append(sts, nvcav1new.ModelCacheRequest) - } - return sts -} - func startControllerManagerForAgent( ctx context.Context, a *Agent, @@ -89,25 +87,33 @@ func startControllerManagerForAgent( metrics *nvcametrics.Metrics, ) error { log := core.GetLogger(ctx) + byObject := map[client.Object]cache.ByObject{ + &corev1.Event{}: { + // Exclude Kyverno events from the cache entirely. + Field: fields.AndSelectors( + fields.OneTermNotEqualSelector("reportingComponent", "kyverno-admission"), + fields.OneTermNotEqualSelector("reportingComponent", "kyverno-scan"), + fields.OneTermNotEqualSelector("reportingComponent", "kyverno-generate"), + ), + }, + } + if a.ControlPlaneID != "" { + selector := labels.SelectorFromSet(labels.Set{types.ControlPlaneIDLabel: a.ControlPlaneID}) + requestNamespaces := map[string]cache.Config{a.RequestsNamespace: {}} + byObject[&nvcav2beta1.ICMSRequest{}] = cache.ByObject{Namespaces: requestNamespaces, Label: selector} + byObject[&nvcav1new.StorageRequest{}] = cache.ByObject{Label: selector} + byObject[&nvcav2beta1.StorageRequest{}] = cache.ByObject{Label: selector} + byObject[&nvcav1alpha1.MiniService{}] = cache.ByObject{Label: selector} + } + // Node, GPU, DRA, and KAI scheduler objects intentionally remain shared, + // cluster-wide inputs. They describe common cluster capacity rather than + // resources owned by one NVCF control plane. mgr, err := ctrl.NewManager(clients.Config, manager.Options{ Scheme: mgrScheme, WebhookServer: fakeWebhookServer{}, Cache: cache.Options{ - ByObject: map[client.Object]cache.ByObject{ - &corev1.Event{}: { - // Exclude Kyverno events from the cache entirely. - // These PolicyViolation events are not actionable by users and can - // be very numerous, contributing to memory pressure. This filter - // is applied server-side so events never come over the wire. - // See DGXCINC-3086. - Field: fields.AndSelectors( - fields.OneTermNotEqualSelector("reportingComponent", "kyverno-admission"), - fields.OneTermNotEqualSelector("reportingComponent", "kyverno-scan"), - fields.OneTermNotEqualSelector("reportingComponent", "kyverno-generate"), - ), - }, - }, + ByObject: byObject, }, }) if err != nil { @@ -119,7 +125,11 @@ func startControllerManagerForAgent( // backs every storage-class-selected cache backend (NVMesh / shared-FS / // Samba), so it is registered whenever caching is enabled. cachingEnabled := a.CachingSupportEnabled - sts := storageControllerTypes(cachingEnabled) + sts := storage.ControllerTypes(cachingEnabled, a.ControlPlaneID) + if cachingEnabled && a.ControlPlaneID != "" { + log.WithField("control_plane_id", a.ControlPlaneID). + Warn("Model-cache controller disabled: named multi-control-plane mode does not support legacy cluster-scoped model-cache resources") + } log.WithField("controllers", sts). WithField("caching_support_enabled", cachingEnabled). WithField("caching_support_flag", a.FeatureFlagFetcher.IsFeatureFlagEnabled(featureflag.CachingSupport)). @@ -130,6 +140,7 @@ func startControllerManagerForAgent( a.ClusterRegion, a.K8sTimeConfig, storage.ControllerOptions{ + ControlPlaneID: a.ControlPlaneID, ICMSRequestNamespace: a.RequestsNamespace, CSIVolumeMountOptions: a.CSIVolumeMountOptions, Metrics: metrics, @@ -144,7 +155,7 @@ func startControllerManagerForAgent( // Seed the model cache mount option defaults once, after the manager cache // has started. A failure is logged rather than fatal: without the ConfigMap // the reconciler falls back to the configured mount options. - if cachingEnabled { + if cachingEnabled && a.ControlPlaneID == "" { if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { if err := storage.EnsureCacheMountOptionsConfigMap(ctx, mgr.GetClient(), ""); err != nil { log.WithError(err).Error("Failed to seed the cache mount option defaults") @@ -187,6 +198,7 @@ func startControllerManagerForAgent( a.backendk8scache.regITCache, a.ClusterAttributes, mscontroller.ControllerOptions{ + ControlPlaneID: a.ControlPlaneID, SystemNamespace: a.SystemNamespace, ICMSRequestNamespace: a.RequestsNamespace, K8sVersion: a.K8sVersion, @@ -224,12 +236,17 @@ func startControllerManagerForAgent( } // Add GC cleaners to the manager - gcRunnable := gc.NewRunnable(clients, metrics, gc.DefaultInterval, a.RequestsNamespace) - if err := mgr.Add(gcRunnable); err != nil { - log.WithError(err).Error("Failed to add GC controller to controller manager") - return fmt.Errorf("add GC controller to controller manager: %v", err) + if a.ControlPlaneID == "" { + gcRunnable := gc.NewRunnable(clients, metrics, gc.DefaultInterval, a.RequestsNamespace) + if err := mgr.Add(gcRunnable); err != nil { + log.WithError(err).Error("Failed to add GC controller to controller manager") + return fmt.Errorf("add GC controller to controller manager: %v", err) + } + log.Info("Added GC controller to controller manager") + } else { + log.WithField("control_plane_id", a.ControlPlaneID). + Warn("Legacy cluster-wide garbage collectors disabled in named mode; owner reconciliation remains active") } - log.Info("Added GC controller to controller manager") // Don't need a select statement since the context will cancel // the blocking manager start call diff --git a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go index c3107211c..aae43b39d 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go +++ b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go @@ -138,6 +138,8 @@ type CacheAccessObj struct { // thread-safe. type BackendK8sCache struct { cfg nvcaconfig.Config + // controlPlaneID is empty for the legacy single-control-plane deployment. + controlPlaneID string // Namespace for all BART app resources. // No ICMS request or request-derived resources will be created here. @@ -302,6 +304,14 @@ func (b *BackendK8sCacheBuilder) WithRequestsNamespace(requestsNamespace string) return &next } +// WithControlPlaneID scopes all namespaced and cluster-scoped runtime objects +// to one stable control-plane identity. Empty preserves legacy behavior. +func (b *BackendK8sCacheBuilder) WithControlPlaneID(controlPlaneID string) *BackendK8sCacheBuilder { + next := *b + next.controlPlaneID = controlPlaneID + return &next +} + func (b *BackendK8sCacheBuilder) WithNamespaceLabels(nsl labels.Set) *BackendK8sCacheBuilder { next := *b next.namespaceLabels = nsl @@ -508,6 +518,7 @@ func (b *BackendK8sCacheBuilder) Start(ctx context.Context) (*BackendK8sCache, < c := &BackendK8sCache{ cfg: b.cfg, + controlPlaneID: b.controlPlaneID, systemNamespace: b.systemNamespace, requestsNamespace: b.requestsNamespace, namespaceLabels: b.namespaceLabels, @@ -587,9 +598,14 @@ func (b *BackendK8sCacheBuilder) Start(ctx context.Context) (*BackendK8sCache, < // Initialize ICMSRequest and StorageRequest informers but do not start. // Must be started after dependencies are initialized and started. + informerOptions := []nvcainformers.SharedInformerOption{} + if c.controlPlaneID != "" { + informerOptions = append(informerOptions, nvcainformers.WithNamespace(c.requestsNamespace)) + } nvcaInformerFactory := nvcainformers.NewSharedInformerFactoryWithOptions( b.clients.BART, ResyncInterval, + informerOptions..., ) icmsReqGenInf, err := nvcaInformerFactory.ForResource(nvcav2beta1new.SchemeGroupVersion.WithResource("icmsrequests")) if err != nil { @@ -642,10 +658,16 @@ func (b *BackendK8sCacheBuilder) Start(ctx context.Context) (*BackendK8sCache, < } } } - podInfFactory := k8sinformers.NewSharedInformerFactoryWithOptions( - k8sClient, - resyncPeriod, - ) + podInformerOptions := []k8sinformers.SharedInformerOption{} + if c.controlPlaneID != "" { + controlPlaneSelector := labels.SelectorFromSet(labels.Set{ + nvcatypes.ControlPlaneIDLabel: c.controlPlaneID, + }) + podInformerOptions = append(podInformerOptions, k8sinformers.WithTweakListOptions(func(lo *metav1.ListOptions) { + lo.LabelSelector = controlPlaneSelector.String() + })) + } + podInfFactory := k8sinformers.NewSharedInformerFactoryWithOptions(k8sClient, resyncPeriod, podInformerOptions...) podInf := podInfFactory.Core().V1().Pods() _, err := podInf.Informer().AddEventHandler(&cache.ResourceEventHandlerFuncs{ UpdateFunc: func(oldObj, newObj any) { @@ -727,24 +749,26 @@ func (b *BackendK8sCacheBuilder) Start(ctx context.Context) (*BackendK8sCache, < } } - // Helm model cache initialization namespace may not exist on startup. - mcInitNamespace := storage.NewModelCacheInitNamespace() - _, err := c.clients.K8s.CoreV1().Namespaces().Create(ctx, mcInitNamespace, metav1.CreateOptions{}) - - if err != nil && !k8serrors.IsAlreadyExists(err) { - return nil, nil, fmt.Errorf("failed to create model cache init namespace: %w", err) - } - // Patch WorkloadInstanceTypeLabel onto the namespace so the Kyverno - // add-unbound-dns policy injects nvcf-unbound nameservers into writer - // job pods. Done here (not only in Create) so pre-existing namespaces - // on upgraded clusters receive the label immediately at startup. - if err := ensureModelCacheNamespaceLabel(ctx, c.clients.K8s.CoreV1().Namespaces(), mcInitNamespace.Name); err != nil { - return nil, nil, fmt.Errorf("failed to patch model cache init namespace labels: %w", err) + // Network policies must exist in all workload namespaces; the Helm + // handler methods do this for each new namespace. The historical model + // cache namespace is a shared singleton and is therefore created only in + // legacy mode; named mode disables durable model-cache resources. + workloadNamespaces := []string{c.podInstanceNamespace} + if legacyModelCacheResourcesEnabled(c.controlPlaneID) { + mcInitNamespace := storage.NewModelCacheInitNamespace() + _, err := c.clients.K8s.CoreV1().Namespaces().Create(ctx, mcInitNamespace, metav1.CreateOptions{}) + if err != nil && !k8serrors.IsAlreadyExists(err) { + return nil, nil, fmt.Errorf("failed to create model cache init namespace: %w", err) + } + // Patch WorkloadInstanceTypeLabel onto the namespace so the Kyverno + // add-unbound-dns policy injects nvcf-unbound nameservers into writer + // job pods. Done here so upgraded legacy clusters are patched too. + if err := ensureModelCacheNamespaceLabel(ctx, c.clients.K8s.CoreV1().Namespaces(), mcInitNamespace.Name); err != nil { + return nil, nil, fmt.Errorf("failed to patch model cache init namespace labels: %w", err) + } + workloadNamespaces = append(workloadNamespaces, mcInitNamespace.Name) } - - // Network policies must exist in all workload namespaces; - // the Helm handler methods will do this for each new namespace. - for _, namespace := range []string{c.podInstanceNamespace, mcInitNamespace.Name} { + for _, namespace := range workloadNamespaces { err := k8sArtHelper.(K8sComputeBackend).ensureNetworkPolicies(ctx, namespace) if err != nil { return nil, nil, fmt.Errorf("create NetworkPolicies in namespace %s: %v", @@ -895,7 +919,12 @@ func (b *BackendK8sCacheBuilder) Start(ctx context.Context) (*BackendK8sCache, < // Start NVCA object informers after all setup is complete so dependencies // can start first. nvcaInformerFactory.Start(ctx.Done()) - cache.WaitForCacheSync(ctx.Done(), c.syncedFuncs...) + if ok := cache.WaitForCacheSync(ctx.Done(), c.syncedFuncs...); !ok { + return nil, nil, fmt.Errorf("failed to sync Kubernetes informer caches") + } + if err := c.reconcileExistingMirroredSecrets(ctx); err != nil { + return nil, nil, fmt.Errorf("reconcile existing mirrored secrets: %w", err) + } log.Infof("Starting %d ICMS request sync workers", b.icmsRequestSyncConcurrency) for i := 0; i < b.icmsRequestSyncConcurrency; i++ { @@ -955,6 +984,36 @@ func (c *BackendK8sCache) startSecretMirroringInformer(ctx context.Context) erro return nil } +// reconcileExistingMirroredSecrets closes the startup race between the Secret +// and instance-namespace informers. Add events can run before the namespace +// lister is initialized or synced, so replay the cached source Secrets after +// all informer caches have synced. +func (c *BackendK8sCache) reconcileExistingMirroredSecrets(ctx context.Context) error { + if c.secretMirrorLabelSelector == "" { + return nil + } + if c.secretNamespaceLister == nil { + return fmt.Errorf("secret mirror lister is not initialized") + } + if c.instanceNamespaceLister == nil { + return fmt.Errorf("instance namespace lister is not initialized") + } + + secrets, err := c.secretNamespaceLister.List(labels.Everything()) + if err != nil { + return fmt.Errorf("list existing source secrets: %w", err) + } + + var reconcileErrors []error + for _, secret := range secrets { + if err := c.mirrorSecret(ctx, secret); err != nil { + reconcileErrors = append(reconcileErrors, + fmt.Errorf("mirror existing secret %s/%s: %w", secret.Namespace, secret.Name, err)) + } + } + return errors.Join(reconcileErrors...) +} + // initCustomAnnotationsCache initializes the custom annotations cache // The cache is populated by the ConfigMap informer in addConfigMapInformers func (c *BackendK8sCache) initCustomAnnotationsCache() { @@ -979,8 +1038,7 @@ func (c *BackendK8sCache) mirrorSecret(ctx context.Context, sourceSecret *corev1 } if c.instanceNamespaceLister == nil { - log.Debug("instanceNamespaceLister not initialized to mirror secrets") - return nil + return fmt.Errorf("instance namespace lister is not initialized") } // Get all function namespaces @@ -989,8 +1047,14 @@ func (c *BackendK8sCache) mirrorSecret(ctx context.Context, sourceSecret *corev1 return fmt.Errorf("failed to list function namespaces: %w", err) } - // Create a new secret object for each namespace + // Create a new secret object for each namespace. Continue processing other + // namespaces after a target failure, but report every failure to the caller + // so startup reconciliation cannot silently succeed with missing mirrors. + var mirrorErrors []error for _, ns := range namespaces { + if !nvcatypes.IsOwnedByControlPlane(ns, c.controlPlaneID) { + continue + } // Skip if source namespace is the same as target if ns.Name == sourceSecret.Namespace { continue @@ -1013,6 +1077,7 @@ func (c *BackendK8sCache) mirrorSecret(ctx context.Context, sourceSecret *corev1 } // add mirrored from label newSecret.Labels[SecretMirroredFromLabelKey] = sourceSecret.Namespace + newSecret.Labels = nvcatypes.AddControlPlaneLabel(newSecret.Labels, c.controlPlaneID) // Copy the secret data maps.Copy(newSecret.Data, sourceSecret.Data) @@ -1021,15 +1086,32 @@ func (c *BackendK8sCache) mirrorSecret(ctx context.Context, sourceSecret *corev1 _, err = c.clients.K8s.CoreV1().Secrets(ns.Name).Create(ctx, newSecret, metav1.CreateOptions{}) if err != nil { if k8serrors.IsAlreadyExists(err) { - // Update existing secret - _, err = c.clients.K8s.CoreV1().Secrets(ns.Name).Update(ctx, newSecret, metav1.UpdateOptions{}) - if err != nil { - log.WithError(err).Errorf("failed to update mirrored secret %s in namespace %s", newSecret.Name, ns.Name) + // The informer handler and startup replay can update the same + // mirror concurrently. Re-read ownership and resourceVersion on + // every retry so normal conflicts cannot fail agent startup. + updateErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { + existing, getErr := c.clients.K8s.CoreV1().Secrets(ns.Name).Get(ctx, newSecret.Name, metav1.GetOptions{}) + if getErr != nil { + return fmt.Errorf("get mirrored secret %s/%s: %w", ns.Name, newSecret.Name, getErr) + } + if !nvcatypes.IsOwnedByControlPlane(existing, c.controlPlaneID) { + return fmt.Errorf( + "refusing to update mirrored secret %s/%s owned by another control plane", ns.Name, newSecret.Name, + ) + } + newSecret.ResourceVersion = existing.ResourceVersion + _, updateErr := c.clients.K8s.CoreV1().Secrets(ns.Name).Update(ctx, newSecret, metav1.UpdateOptions{}) + return updateErr + }) + if updateErr != nil { + mirrorErrors = append(mirrorErrors, + fmt.Errorf("update mirrored secret %s/%s: %w", ns.Name, newSecret.Name, updateErr)) continue } log.Debugf("Updated mirrored secret %s in namespace %s", newSecret.Name, ns.Name) } else { - log.WithError(err).Errorf("failed to create mirrored secret %s in namespace %s", newSecret.Name, ns.Name) + mirrorErrors = append(mirrorErrors, + fmt.Errorf("create mirrored secret %s/%s: %w", ns.Name, newSecret.Name, err)) continue } } else { @@ -1037,7 +1119,7 @@ func (c *BackendK8sCache) mirrorSecret(ctx context.Context, sourceSecret *corev1 } } - return nil + return errors.Join(mirrorErrors...) } func (c *BackendK8sCache) processICMSRequestWork(ctx context.Context) bool { @@ -1079,9 +1161,14 @@ func (c *BackendK8sCache) processICMSRequestWork(ctx context.Context) bool { // additionally delete it's associated MiniService if it exists. if k8serrors.IsNotFound(err) { c.icmsRequestWQ.Forget(obj) - miniserviceName := getMiniServiceInstanceID(nn.Name) - if err := c.clients.HelmV2.Get(ctx, client.ObjectKey{Name: miniserviceName}, &v1alpha1.MiniService{}); err == nil { - err = c.clients.HelmV2.Delete(ctx, &v1alpha1.MiniService{ObjectMeta: metav1.ObjectMeta{Name: miniserviceName}}) + miniserviceName := getMiniServiceInstanceID(nn.Name, c.controlPlaneID) + miniService := &v1alpha1.MiniService{} + if err := c.clients.HelmV2.Get(ctx, client.ObjectKey{Name: miniserviceName}, miniService); err == nil { + if !nvcatypes.IsOwnedByControlPlane(miniService, c.controlPlaneID) { + log.Errorf("Refusing to delete MiniService %s owned by another control plane", miniserviceName) + return + } + err = c.clients.HelmV2.Delete(ctx, miniService) if !k8serrors.IsNotFound(err) { log.WithError(err).Error("Failed to delete MiniService workload, requeuing to try again") c.icmsRequestWQ.AddRateLimited(obj) @@ -1107,7 +1194,7 @@ func (c *BackendK8sCache) processICMSRequestWork(ctx context.Context) bool { if len(sr.Status.Instances) == 0 { if helmutil.IsMiniServiceCreateRequest(sr) { // Helm chart request - create single placeholder instance - instanceID := getMiniServiceInstanceID(sr.Name) + instanceID := getMiniServiceInstanceID(sr.Name, c.controlPlaneID) sr.Status.Instances = map[string]nvcav2beta1new.InstanceStatus{ instanceID: { ID: instanceID, @@ -1337,8 +1424,10 @@ func configMapInformerHandler(ctx context.Context, c *BackendK8sCache) func(obj // return true if the namespace is not active which will remove it from the list return ns.Status.Phase != corev1.NamespaceActive }) - // Include the model cache init namespace since it has workload pods running (cache init jobs). - instanceNamespaces = append(instanceNamespaces, storage.NewModelCacheInitNamespace()) + // Legacy model-cache init jobs run in a shared singleton namespace. + if legacyModelCacheResourcesEnabled(c.controlPlaneID) { + instanceNamespaces = append(instanceNamespaces, storage.NewModelCacheInitNamespace()) + } switch cm.Name { case k8sutil.NetworkPoliciesConfigMapName: @@ -1494,18 +1583,38 @@ func (c *BackendK8sCache) processNodeWork(ctx context.Context, return true } -func (c *BackendK8sCache) initInstanceNamespaceInformer(ctx context.Context) error { +func legacyModelCacheResourcesEnabled(controlPlaneID string) bool { + return controlPlaneID == "" +} + +func instanceNamespaceSelector(controlPlaneID string) (labels.Selector, error) { // Ensure only instance namespaces are selected namespaceLabelSel, err := labels.NewRequirement(nvcatypes.WorkloadInstanceTypeLabel, selection.Exists, nil) if err != nil { - return fmt.Errorf("failed to create namespace label requirement: %w", err) + return nil, fmt.Errorf("failed to create namespace label requirement: %w", err) + } + selector := labels.NewSelector().Add(*namespaceLabelSel) + if controlPlaneID == "" { + return selector, nil + } + controlPlaneSel, err := labels.NewRequirement(nvcatypes.ControlPlaneIDLabel, selection.Equals, []string{controlPlaneID}) + if err != nil { + return nil, fmt.Errorf("failed to create control plane label requirement: %w", err) + } + return selector.Add(*controlPlaneSel), nil +} + +func (c *BackendK8sCache) initInstanceNamespaceInformer(ctx context.Context) error { + selector, err := instanceNamespaceSelector(c.controlPlaneID) + if err != nil { + return err } infFactory := k8sinformers.NewSharedInformerFactoryWithOptions( c.clients.K8s, c.resyncPeriod, k8sinformers.WithTweakListOptions(func(lo *metav1.ListOptions) { - lo.LabelSelector = namespaceLabelSel.String() + lo.LabelSelector = selector.String() }), ) namespaceInf := infFactory.Core().V1().Namespaces() @@ -2024,6 +2133,13 @@ func getICMSRequestObjectMeta(depInfo types.DeploymentInfo) metav1.ObjectMeta { return om } +func ownsICMSRequest(req *nvcav2beta1new.ICMSRequest, requestsNamespace, controlPlaneID string) bool { + if controlPlaneID == "" { + return true + } + return req != nil && req.Namespace == requestsNamespace && nvcatypes.IsOwnedByControlPlane(req, controlPlaneID) +} + func (c *BackendK8sCache) applyICMSRequestStatusChange(ctx context.Context, sr *nvcav2beta1new.ICMSRequest, modify func(context.Context, *nvcav2beta1new.ICMSRequest), ) bool { @@ -2268,6 +2384,7 @@ func (c *BackendK8sCache) CreateICMSCreationMessageRequest(ctx context.Context, TaskID: mt.Details.TaskID, }) } + o.Labels = nvcatypes.AddControlPlaneLabel(o.Labels, c.controlPlaneID) obj, err := c.clients.BART.NvcaV2beta1().ICMSRequests(c.requestsNamespace).Create(ctx, &o, metav1.CreateOptions{}) if err != nil { return nil, fmt.Errorf("failed to persist the ICMS request on the backend, err: %v", err) @@ -2315,6 +2432,7 @@ func (c *BackendK8sCache) CreateICMSTerminationMessageRequest(ctx context.Contex Instances: map[string]nvcav2beta1new.InstanceStatus{}, }, } + o.Labels = nvcatypes.AddControlPlaneLabel(o.Labels, c.controlPlaneID) obj, err := c.clients.BART.NvcaV2beta1().ICMSRequests(c.requestsNamespace).Create(ctx, &o, metav1.CreateOptions{}) if err != nil { @@ -2473,6 +2591,9 @@ func (c *BackendK8sCache) CleanupCreationRequestResources(ctx context.Context, r -> "RequestCompletionACK / "RequestFailureACK" */ func (c *BackendK8sCache) SyncICMSRequest(ctx context.Context, nn apitypes.NamespacedName) error { + if c.controlPlaneID != "" && nn.Namespace != c.requestsNamespace { + return nvcaerrors.TerminalError(fmt.Errorf("ICMS request %s/%s is outside control plane %q", nn.Namespace, nn.Name, c.controlPlaneID)) + } req, err := c.icmsRequestLister.ICMSRequests(nn.Namespace).Get(nn.Name) if err != nil { // If the ICMS request no longer exists we need to consider it terminal @@ -2484,6 +2605,9 @@ func (c *BackendK8sCache) SyncICMSRequest(ctx context.Context, nn apitypes.Names // Deep copy the ICMS request to avoid data race condition // since the lister is pulling it from a cache req = req.DeepCopy() + if !ownsICMSRequest(req, c.requestsNamespace, c.controlPlaneID) { + return nvcaerrors.TerminalError(fmt.Errorf("ICMS request %s/%s is not owned by control plane %q", req.Namespace, req.Name, c.controlPlaneID)) + } ctx = logging.WithICMSRequestFieldLogger(ctx, req) return nvcaotel.InvokeWithSpan(ctx, c.tracer, "nvca.BackendK8sCache.SyncICMSRequest", func(ctx context.Context) error { @@ -2968,7 +3092,17 @@ func (c *BackendK8sCache) ensureImageCredentialUpdaterCronJob(ctx context.Contex return err } namespaceSelector := labels.NewSelector().Add(*namespaceSelectorReq) + if c.controlPlaneID != "" { + controlPlaneReq, reqErr := labels.NewRequirement(nvcatypes.ControlPlaneIDLabel, selection.Equals, []string{c.controlPlaneID}) + if reqErr != nil { + return reqErr + } + namespaceSelector = namespaceSelector.Add(*controlPlaneReq) + } cj := imagecredential.NewUpdaterCronJob(cjName, c.imageCredentialHelperImage, namespaceSelector.String()) + cj.Labels = nvcatypes.AddControlPlaneLabel(cj.Labels, c.controlPlaneID) + cj.Spec.JobTemplate.Labels = nvcatypes.AddControlPlaneLabel(cj.Spec.JobTemplate.Labels, c.controlPlaneID) + cj.Spec.JobTemplate.Spec.Template.Labels = nvcatypes.AddControlPlaneLabel(cj.Spec.JobTemplate.Spec.Template.Labels, c.controlPlaneID) // Use NVCA's service account to run the job for API access and image pull secrets. cj.Namespace = c.systemNamespace cj.Spec.JobTemplate.Spec.Template.Spec.ServiceAccountName = "nvca" diff --git a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go index c7681c4de..0c384c985 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go +++ b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go @@ -65,6 +65,7 @@ import ( fakek8sclient "k8s.io/client-go/kubernetes/fake" listersv1 "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/rest" + k8stesting "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" @@ -3752,6 +3753,246 @@ func TestSecretInformerSetup(t *testing.T) { assert.Equal(t, sourceNS.Name, mirroredSecret.Labels[SecretMirroredFromLabelKey]) } +func TestReconcileExistingMirroredSecretsColdStart(t *testing.T) { + ctx, cancel := context.WithTimeout(newTestContext(), 5*time.Second) + t.Cleanup(cancel) + + const ( + controlPlaneID = "plane-a" + sourceNamespace = "plane-a-nvca-operator" + targetNamespace = "plane-a-workload" + foreignNamespace = "plane-b-workload" + secretName = "preexisting-mirror-secret" + ) + + sourceNS := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: sourceNamespace}} + targetNS := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{ + Name: targetNamespace, + Labels: map[string]string{ + nvcatypes.WorkloadInstanceTypeLabel: "miniservice", + nvcatypes.ControlPlaneIDLabel: controlPlaneID, + }, + }} + foreignNS := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{ + Name: foreignNamespace, + Labels: map[string]string{ + nvcatypes.WorkloadInstanceTypeLabel: "miniservice", + nvcatypes.ControlPlaneIDLabel: "plane-b", + }, + }} + sourceSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: sourceNamespace, + Labels: map[string]string{"mirror": "true"}, + }, + Data: map[string][]byte{"marker": []byte("plane-a")}, + } + + client := fakek8sclient.NewSimpleClientset(sourceNS, targetNS, foreignNS, sourceSecret) + namespaceIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + for _, namespace := range []*corev1.Namespace{sourceNS, targetNS, foreignNS} { + require.NoError(t, namespaceIndexer.Add(namespace)) + } + secretIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{ + cache.NamespaceIndex: cache.MetaNamespaceIndexFunc, + }) + require.NoError(t, secretIndexer.Add(sourceSecret)) + + backendCache := &BackendK8sCache{ + clients: &kubeclients.KubeClients{K8s: client}, + controlPlaneID: controlPlaneID, + secretMirrorSourceNamespace: sourceNamespace, + secretMirrorLabelSelector: "mirror=true", + instanceNamespaceLister: listersv1.NewNamespaceLister(namespaceIndexer), + secretNamespaceLister: listersv1.NewSecretLister(secretIndexer).Secrets(sourceNamespace), + } + + // No source Secret event occurs after startup. Startup reconciliation must + // mirror the already-cached object without waiting for the 30-minute resync. + require.NoError(t, backendCache.reconcileExistingMirroredSecrets(ctx)) + require.NoError(t, backendCache.reconcileExistingMirroredSecrets(ctx), "startup replay must be idempotent") + + mirrored, err := client.CoreV1().Secrets(targetNamespace).Get(ctx, secretName, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, sourceSecret.Data, mirrored.Data) + assert.Equal(t, sourceNamespace, mirrored.Labels[SecretMirroredFromLabelKey]) + assert.Equal(t, controlPlaneID, mirrored.Labels[nvcatypes.ControlPlaneIDLabel]) + + _, err = client.CoreV1().Secrets(foreignNamespace).Get(ctx, secretName, metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(err), "startup reconciliation must not mirror into another control plane") +} + +func TestReconcileExistingMirroredSecretsDisabled(t *testing.T) { + backendCache := &BackendK8sCache{} + require.NoError(t, backendCache.reconcileExistingMirroredSecrets(newTestContext())) +} + +func TestReconcileExistingMirroredSecretsRequiresSyncedListers(t *testing.T) { + backendCache := &BackendK8sCache{secretMirrorLabelSelector: "mirror=true"} + err := backendCache.reconcileExistingMirroredSecrets(newTestContext()) + require.ErrorContains(t, err, "secret mirror lister is not initialized") + + secretIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{ + cache.NamespaceIndex: cache.MetaNamespaceIndexFunc, + }) + backendCache.secretNamespaceLister = listersv1.NewSecretLister(secretIndexer).Secrets("source") + err = backendCache.reconcileExistingMirroredSecrets(newTestContext()) + require.ErrorContains(t, err, "instance namespace lister is not initialized") +} + +func TestMirrorSecretReturnsTargetErrors(t *testing.T) { + ctx := newTestContext() + const ( + controlPlaneID = "plane-a" + sourceNamespace = "plane-a-nvca-operator" + targetNamespace = "plane-a-workload" + secretName = "mirror-error-probe" + ) + targetNS := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{ + Name: targetNamespace, + Labels: map[string]string{ + nvcatypes.WorkloadInstanceTypeLabel: "miniservice", + nvcatypes.ControlPlaneIDLabel: controlPlaneID, + }, + }} + sourceSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: sourceNamespace, + Labels: map[string]string{"mirror": "true"}, + }, + Data: map[string][]byte{"marker": []byte("source")}, + } + + newBackendCache := func(t *testing.T, client *fakek8sclient.Clientset) *BackendK8sCache { + t.Helper() + namespaceIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + require.NoError(t, namespaceIndexer.Add(targetNS)) + return &BackendK8sCache{ + clients: &kubeclients.KubeClients{K8s: client}, + controlPlaneID: controlPlaneID, + secretMirrorSourceNamespace: sourceNamespace, + secretMirrorLabelSelector: "mirror=true", + instanceNamespaceLister: listersv1.NewNamespaceLister(namespaceIndexer), + } + } + + t.Run("create failure", func(t *testing.T) { + client := fakek8sclient.NewSimpleClientset(targetNS) + client.PrependReactor("create", "secrets", func(action k8stesting.Action) (bool, runtime.Object, error) { + if action.GetNamespace() == targetNamespace { + return true, nil, errors.New("injected create failure") + } + return false, nil, nil + }) + + err := newBackendCache(t, client).mirrorSecret(ctx, sourceSecret) + require.ErrorContains(t, err, "injected create failure") + }) + + t.Run("get failure", func(t *testing.T) { + existing := sourceSecret.DeepCopy() + existing.Namespace = targetNamespace + existing.Labels = nvcatypes.AddControlPlaneLabel(existing.Labels, controlPlaneID) + client := fakek8sclient.NewSimpleClientset(targetNS, existing) + client.PrependReactor("get", "secrets", func(action k8stesting.Action) (bool, runtime.Object, error) { + if action.GetNamespace() == targetNamespace { + return true, nil, errors.New("injected get failure") + } + return false, nil, nil + }) + + err := newBackendCache(t, client).mirrorSecret(ctx, sourceSecret) + require.ErrorContains(t, err, "injected get failure") + }) + + t.Run("update failure", func(t *testing.T) { + existing := sourceSecret.DeepCopy() + existing.Namespace = targetNamespace + existing.Labels = nvcatypes.AddControlPlaneLabel(existing.Labels, controlPlaneID) + client := fakek8sclient.NewSimpleClientset(targetNS, existing) + client.PrependReactor("update", "secrets", func(action k8stesting.Action) (bool, runtime.Object, error) { + if action.GetNamespace() == targetNamespace { + return true, nil, errors.New("injected update failure") + } + return false, nil, nil + }) + + err := newBackendCache(t, client).mirrorSecret(ctx, sourceSecret) + require.ErrorContains(t, err, "injected update failure") + }) + + t.Run("ownership collision", func(t *testing.T) { + existing := sourceSecret.DeepCopy() + existing.Namespace = targetNamespace + existing.Labels = nvcatypes.AddControlPlaneLabel(existing.Labels, "plane-b") + existing.Data = map[string][]byte{"marker": []byte("foreign")} + client := fakek8sclient.NewSimpleClientset(targetNS, existing) + + err := newBackendCache(t, client).mirrorSecret(ctx, sourceSecret) + require.ErrorContains(t, err, "owned by another control plane") + unchanged, getErr := client.CoreV1().Secrets(targetNamespace).Get(ctx, secretName, metav1.GetOptions{}) + require.NoError(t, getErr) + assert.Equal(t, []byte("foreign"), unchanged.Data["marker"]) + }) +} + +func TestMirrorSecretRetriesUpdateConflict(t *testing.T) { + ctx := newTestContext() + const ( + controlPlaneID = "plane-a" + sourceNamespace = "plane-a-nvca-operator" + targetNamespace = "plane-a-workload" + secretName = "mirror-conflict-probe" + ) + targetNS := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{ + Name: targetNamespace, + Labels: map[string]string{ + nvcatypes.WorkloadInstanceTypeLabel: "miniservice", + nvcatypes.ControlPlaneIDLabel: controlPlaneID, + }, + }} + sourceSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: sourceNamespace, + Labels: map[string]string{"mirror": "true"}, + }, + Data: map[string][]byte{"marker": []byte("source")}, + } + existing := sourceSecret.DeepCopy() + existing.Namespace = targetNamespace + existing.Labels = nvcatypes.AddControlPlaneLabel(existing.Labels, controlPlaneID) + existing.Data = map[string][]byte{"marker": []byte("stale")} + + client := fakek8sclient.NewSimpleClientset(targetNS, existing) + var updateCalls atomic.Int32 + client.PrependReactor("update", "secrets", func(action k8stesting.Action) (bool, runtime.Object, error) { + if action.GetNamespace() == targetNamespace && updateCalls.Add(1) == 1 { + return true, nil, apierrors.NewConflict(corev1.Resource("secrets"), secretName, errors.New("injected conflict")) + } + return false, nil, nil + }) + + namespaceIndexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + require.NoError(t, namespaceIndexer.Add(targetNS)) + backendCache := &BackendK8sCache{ + clients: &kubeclients.KubeClients{K8s: client}, + controlPlaneID: controlPlaneID, + secretMirrorSourceNamespace: sourceNamespace, + secretMirrorLabelSelector: "mirror=true", + instanceNamespaceLister: listersv1.NewNamespaceLister(namespaceIndexer), + } + + require.NoError(t, backendCache.mirrorSecret(ctx, sourceSecret)) + assert.Equal(t, int32(2), updateCalls.Load()) + updated, err := client.CoreV1().Secrets(targetNamespace).Get(ctx, secretName, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, []byte("source"), updated.Data["marker"]) + assert.Equal(t, controlPlaneID, updated.Labels[nvcatypes.ControlPlaneIDLabel]) +} + func TestStartSecretMirroringInformer(t *testing.T) { ctx, cancel := context.WithTimeout(newTestContext(), 5*time.Second) defer cancel() diff --git a/src/compute-plane-services/nvca/pkg/nvca/cli.go b/src/compute-plane-services/nvca/pkg/nvca/cli.go index 53dc792ac..56389ae18 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/cli.go +++ b/src/compute-plane-services/nvca/pkg/nvca/cli.go @@ -69,6 +69,7 @@ type agentHostOverrides struct { ICMSHostHeaderOverride string `yaml:"icmsHostHeaderOverride"` HelmReValServiceHostHeaderOverride string `yaml:"helmReValServiceHostHeaderOverride"` NATSHostOverride string `yaml:"NATSHostOverride"` + ControlPlaneID string `yaml:"controlPlaneID"` } func readAgentHostOverrides(configFile string) (agentHostOverrides, error) { @@ -169,6 +170,7 @@ func newCobraCommand( ClusterGroupID: cfg.Cluster.GroupID, ClusterGroupName: cfg.Cluster.GroupName, ClusterAttributes: featureflag.GetEnabledAttributes(), + ControlPlaneID: hostOverrides.ControlPlaneID, CloudProvider: cfg.Cluster.CloudProvider, ICMSURL: cfg.Agent.ICMSURL, ICMSHostHeaderOverride: hostOverrides.ICMSHostHeaderOverride, diff --git a/src/compute-plane-services/nvca/pkg/nvca/cli_test.go b/src/compute-plane-services/nvca/pkg/nvca/cli_test.go index c18714520..1ae9ab98f 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/cli_test.go +++ b/src/compute-plane-services/nvca/pkg/nvca/cli_test.go @@ -60,6 +60,15 @@ func TestSetDefaultsPreservesConfiguredDefaultStargateAddress(t *testing.T) { assert.Equal(t, "llm-router.example.test:50071", cfg.Workload.DefaultStargateAddress) } +func TestReadAgentHostOverridesReadsControlPlaneID(t *testing.T) { + path := t.TempDir() + "/config.yaml" + require.NoError(t, os.WriteFile(path, []byte("agent:\n controlPlaneID: plane-a\n"), 0o600)) + + overrides, err := readAgentHostOverrides(path) + require.NoError(t, err) + assert.Equal(t, "plane-a", overrides.ControlPlaneID) +} + func TestNewCommand(t *testing.T) { newAgentFunc := func(a cliAgent) func(ctx context.Context, opts *AgentOptions) (cliAgent, error) { return func(ctx context.Context, opts *AgentOptions) (cliAgent, error) { diff --git a/src/compute-plane-services/nvca/pkg/nvca/control_plane_isolation_test.go b/src/compute-plane-services/nvca/pkg/nvca/control_plane_isolation_test.go new file mode 100644 index 000000000..62aed1400 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/control_plane_isolation_test.go @@ -0,0 +1,61 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package nvca + +import ( + "testing" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" +) + +func TestInstanceNamespaceSelectorScopesControlPlane(t *testing.T) { + selector, err := instanceNamespaceSelector("plane-a") + assert.NoError(t, err) + assert.True(t, selector.Matches(labels.Set{ + nvcatypes.WorkloadInstanceTypeLabel: "miniservice", + nvcatypes.ControlPlaneIDLabel: "plane-a", + })) + assert.False(t, selector.Matches(labels.Set{ + nvcatypes.WorkloadInstanceTypeLabel: "miniservice", + nvcatypes.ControlPlaneIDLabel: "plane-b", + })) + assert.True(t, legacyModelCacheResourcesEnabled("")) + assert.False(t, legacyModelCacheResourcesEnabled("plane-a")) +} + +func TestAgentBackendCacheControlPlaneIsolation(t *testing.T) { + b := NewBackendk8sCacheBuilder(). + WithSystemNamespace("plane-a-nvca-system"). + WithRequestsNamespace("plane-a-nvcf-backend"). + WithControlPlaneID("plane-a") + + assert.Equal(t, "plane-a", b.controlPlaneID) + assert.Equal(t, "plane-a-nvca-system", b.systemNamespace) + assert.Equal(t, "plane-a-nvcf-backend", b.requestsNamespace) +} + +func TestMiniServiceIdentityNames(t *testing.T) { + assert.Equal(t, "sr-request-miniservice", getMiniServiceInstanceID("sr-request")) + assert.Equal(t, "plane-a-sr-request-miniservice", getMiniServiceInstanceID("sr-request", "plane-a")) + assert.Equal(t, "sr-request", getMiniServiceNamespace("sr-request")) + assert.Equal(t, "plane-a-sr-request", getMiniServiceNamespace("sr-request", "plane-a")) +} + +func TestICMSRequestOwnership(t *testing.T) { + request := &nvcav2beta1.ICMSRequest{ObjectMeta: metav1.ObjectMeta{ + Namespace: "plane-a-nvcf-backend", + Labels: map[string]string{nvcatypes.ControlPlaneIDLabel: "plane-a"}, + }} + assert.True(t, ownsICMSRequest(request, "plane-a-nvcf-backend", "plane-a")) + assert.False(t, ownsICMSRequest(request, "plane-b-nvcf-backend", "plane-a")) + assert.False(t, ownsICMSRequest(request, "plane-a-nvcf-backend", "plane-b")) + assert.True(t, ownsICMSRequest(request, "", ""), "legacy mode remains unfiltered") +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend.go b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend.go index 48a8a36ed..f7e7467ac 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend.go +++ b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend.go @@ -927,6 +927,8 @@ func (c K8sComputeBackend) initializeImageCredentialHelper( tprUpdaterInitJob := imagecredential.NewInitJob(icmsReq.Name+"-cred-init", c.bk8s.imageCredentialHelperImage, targetNamespace, secretSel.String()) + tprUpdaterInitJob.Labels = nvcatypes.AddControlPlaneLabel(tprUpdaterInitJob.Labels, c.bk8s.controlPlaneID) + tprUpdaterInitJob.Spec.Template.Labels = nvcatypes.AddControlPlaneLabel(tprUpdaterInitJob.Spec.Template.Labels, c.bk8s.controlPlaneID) // Use NVCA's service account to run the job for API access and image pull secrets. tprUpdaterInitJob.Namespace = jobNamespace diff --git a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_miniservice.go b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_miniservice.go index bc0f8b4f5..290aad784 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_miniservice.go +++ b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_miniservice.go @@ -60,8 +60,19 @@ const ( helmChartInstanceRoleName = "mini-service-restrictions" ) -func getMiniServiceInstanceID(srName string) string { - return trimDNS1123Label(srName, len(miniserviceNameSuffix)) + miniserviceNameSuffix +func getMiniServiceInstanceID(srName string, controlPlaneIDs ...string) string { + legacyName := trimDNS1123Label(srName, len(miniserviceNameSuffix)) + miniserviceNameSuffix + if len(controlPlaneIDs) == 0 { + return legacyName + } + return nvcatypes.ControlPlaneResourceName(controlPlaneIDs[0], legacyName) +} + +func getMiniServiceNamespace(srName string, controlPlaneIDs ...string) string { + if len(controlPlaneIDs) == 0 { + return srName + } + return nvcatypes.ControlPlaneResourceName(controlPlaneIDs[0], srName) } func isMiniServiceInstance(name string) bool { @@ -129,9 +140,10 @@ func (c K8sComputeBackend) applyMiniServiceCreationMessage(ctx context.Context, "Creating %v requested instances", nil, instCount) labelsForReq := nvcatypes.GetLabelsForRequest(req, c.bk8s.featureFlagFetcher) + labelsForReq = nvcatypes.AddControlPlaneLabel(labelsForReq, c.bk8s.controlPlaneID) annosForReq := nvcatypes.GetAnnotationsForRequest(req) - instanceID := getMiniServiceInstanceID(req.Name) + instanceID := getMiniServiceInstanceID(req.Name, c.bk8s.controlPlaneID) hcCfg, err := common.ExtractHelmConfiguration(envB64, hcLaunchSpec) if err != nil { @@ -143,7 +155,7 @@ func (c K8sComputeBackend) applyMiniServiceCreationMessage(ctx context.Context, ms.Labels = labelsForReq ms.Annotations = annosForReq ms.Spec = v1alpha1.MiniServiceSpec{ - Namespace: req.Name, + Namespace: getMiniServiceNamespace(req.Name, c.bk8s.controlPlaneID), ICMSRequestName: req.Name, HelmChartConfig: hcCfg, } diff --git a/src/compute-plane-services/nvca/pkg/operator/cleanup/BUILD.bazel b/src/compute-plane-services/nvca/pkg/operator/cleanup/BUILD.bazel index 6034f166c..df504a8ed 100644 --- a/src/compute-plane-services/nvca/pkg/operator/cleanup/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/operator/cleanup/BUILD.bazel @@ -17,6 +17,7 @@ go_library( "//src/compute-plane-services/nvca/pkg/client/clientset/versioned", "//src/compute-plane-services/nvca/pkg/operator/types", "//src/compute-plane-services/nvca/pkg/storage", + "//src/compute-plane-services/nvca/pkg/types", "//src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/core", "//src/compute-plane-services/nvca/vendor/github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/types/nvca/config", "//src/compute-plane-services/nvca/vendor/k8s.io/api/core/v1:core", @@ -48,8 +49,10 @@ go_test( deps = [ "//src/compute-plane-services/nvca/pkg/apis/nvcf/v1:nvcf", "//src/compute-plane-services/nvca/pkg/client/clientset/versioned/fake", + "//src/compute-plane-services/nvca/pkg/types", "//src/compute-plane-services/nvca/vendor/github.com/stretchr/testify/assert", "//src/compute-plane-services/nvca/vendor/github.com/stretchr/testify/require", + "//src/compute-plane-services/nvca/vendor/k8s.io/api/admissionregistration/v1:admissionregistration", "//src/compute-plane-services/nvca/vendor/k8s.io/api/apps/v1:apps", "//src/compute-plane-services/nvca/vendor/k8s.io/api/core/v1:core", "//src/compute-plane-services/nvca/vendor/k8s.io/api/rbac/v1:rbac", diff --git a/src/compute-plane-services/nvca/pkg/operator/cleanup/cleanup.go b/src/compute-plane-services/nvca/pkg/operator/cleanup/cleanup.go index 1ad1189a9..279d0779b 100644 --- a/src/compute-plane-services/nvca/pkg/operator/cleanup/cleanup.go +++ b/src/compute-plane-services/nvca/pkg/operator/cleanup/cleanup.go @@ -38,6 +38,7 @@ import ( "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/client/clientset/versioned" nvcaoptypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/types" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" + nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" ) // Re-export constants from types package for backward compatibility @@ -64,12 +65,13 @@ type cleanupOptions struct { // BackendNamespaces returns the system and requests namespace names for an NVCFBackend func BackendNamespaces(nb *nvidiaiov1.NVCFBackend) (systemNS, requestsNS string) { - systemNS = DefaultNVCASystemNamespace + controlPlaneID := nb.Spec.ClusterConfig.ControlPlaneID + systemNS = nvcatypes.ControlPlaneResourceName(controlPlaneID, DefaultNVCASystemNamespace) if nb.Spec.ClusterConfig.SystemNamespace != "" { systemNS = nb.Spec.ClusterConfig.SystemNamespace } - requestsNS = DefaultNVCARequestsNamespace + requestsNS = nvcatypes.ControlPlaneResourceName(controlPlaneID, DefaultNVCARequestsNamespace) if nb.Spec.ClusterConfig.RequestsNamespace != "" { requestsNS = nb.Spec.ClusterConfig.RequestsNamespace } @@ -77,9 +79,9 @@ func BackendNamespaces(nb *nvidiaiov1.NVCFBackend) (systemNS, requestsNS string) return systemNS, requestsNS } -// CleanupBackendResources deletes all resources created by an NVCFBackend -// including namespaces, webhooks, and cluster roles. -// Note: Operator-managed CRDs are cleaned up via owner references. +// CleanupBackendResources deletes all per-backend resources created by an +// NVCFBackend, including namespaces, webhooks, and cluster roles. Shared CRD +// definitions are intentionally preserved for other control planes. func CleanupBackendResources( //nolint:revive // exported name is intentional ctx context.Context, k8sClient kubernetes.Interface, @@ -96,6 +98,19 @@ func CleanupBackendResources( //nolint:revive // exported name is intentional log.Infof("cleaning-up resources for nvcfbackend %v/%v", nb.Namespace, nb.Name) systemNS, requestsNS := BackendNamespaces(nb) + controlPlaneID := nb.Spec.ClusterConfig.ControlPlaneID + clusterResourceName := nvcatypes.ControlPlaneResourceName(controlPlaneID, NVCAModuleName) + modelCacheNamespace := nvcatypes.ControlPlaneResourceName(controlPlaneID, DefaultModelCacheInitNamespace) + if controlPlaneID != "" { + for _, namespace := range []string{systemNS, requestsNS, modelCacheNamespace} { + if err := validateNamespaceOwnership(ctx, k8sClient, namespace, controlPlaneID); err != nil { + return err + } + } + if err := validateClusterResourceOwnership(ctx, k8sClient, clusterResourceName, controlPlaneID); err != nil { + return err + } + } // Delete all ICMSRequest CRs (remove finalizers first, then delete) if err := deleteICMSRequests(ctx, dynamicClient, requestsNS); err != nil { @@ -106,59 +121,94 @@ func CleanupBackendResources( //nolint:revive // exported name is intentional // These are standalone top-level namespaces with no owner references, so they // won't be cleaned up by garbage collection. Normally NVCA's MiniService controller // handles this, but during forced cleanup NVCA is being torn down. - if err := deleteWorkloadNamespaces(ctx, k8sClient); err != nil { + if err := deleteWorkloadNamespaces(ctx, k8sClient, controlPlaneID); err != nil { log.WithError(err).Warn("failed to delete some workload namespaces") } // Cleanup the system namespace - err := k8sClient.CoreV1().Namespaces().Delete(ctx, systemNS, metav1.DeleteOptions{}) + err := deleteNamespaceForControlPlane(ctx, k8sClient, systemNS, controlPlaneID) if err != nil && !k8serrors.IsNotFound(err) { return fmt.Errorf("failed to cleanup namespace %v, err: %v", systemNS, err) } // Cleanup the requests namespace - err = k8sClient.CoreV1().Namespaces().Delete(ctx, requestsNS, metav1.DeleteOptions{}) + err = deleteNamespaceForControlPlane(ctx, k8sClient, requestsNS, controlPlaneID) if err != nil && !k8serrors.IsNotFound(err) { return fmt.Errorf("failed to cleanup namespace %v, err: %v", requestsNS, err) } // Cleanup the shared model-cache initialization namespace created by NVCA. - err = k8sClient.CoreV1().Namespaces().Delete(ctx, DefaultModelCacheInitNamespace, metav1.DeleteOptions{}) + err = deleteNamespaceForControlPlane(ctx, k8sClient, modelCacheNamespace, controlPlaneID) if err != nil && !k8serrors.IsNotFound(err) { - return fmt.Errorf("failed to cleanup namespace %v, err: %v", DefaultModelCacheInitNamespace, err) + return fmt.Errorf("failed to cleanup namespace %v, err: %v", modelCacheNamespace, err) } // Delete ValidatingWebhookConfiguration - err = k8sClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, NVCAModuleName, metav1.DeleteOptions{}) + err = k8sClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(ctx, clusterResourceName, metav1.DeleteOptions{}) if err != nil && !k8serrors.IsNotFound(err) { - return fmt.Errorf("failed to delete validatingwebhookconfiguration %v, err: %v", NVCAModuleName, err) + return fmt.Errorf("failed to delete validatingwebhookconfiguration %v, err: %v", clusterResourceName, err) } // Delete MutatingWebhookConfiguration - err = k8sClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(ctx, NVCAModuleName, metav1.DeleteOptions{}) + err = k8sClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(ctx, clusterResourceName, metav1.DeleteOptions{}) if err != nil && !k8serrors.IsNotFound(err) { - return fmt.Errorf("failed to delete mutatingwebhookconfiguration %v, err: %v", NVCAModuleName, err) + return fmt.Errorf("failed to delete mutatingwebhookconfiguration %v, err: %v", clusterResourceName, err) } // Delete ClusterRole - err = k8sClient.RbacV1().ClusterRoles().Delete(ctx, NVCAModuleName, metav1.DeleteOptions{}) + err = k8sClient.RbacV1().ClusterRoles().Delete(ctx, clusterResourceName, metav1.DeleteOptions{}) if err != nil && !k8serrors.IsNotFound(err) { - return fmt.Errorf("failed to delete cluster-role %v, err: %v", NVCAModuleName, err) + return fmt.Errorf("failed to delete cluster-role %v, err: %v", clusterResourceName, err) } // Delete ClusterRoleBinding - err = k8sClient.RbacV1().ClusterRoleBindings().Delete(ctx, NVCAModuleName, metav1.DeleteOptions{}) + err = k8sClient.RbacV1().ClusterRoleBindings().Delete(ctx, clusterResourceName, metav1.DeleteOptions{}) if err != nil && !k8serrors.IsNotFound(err) { - return fmt.Errorf("failed to delete cluster-role-bindings %v, err: %v", NVCAModuleName, err) + return fmt.Errorf("failed to delete cluster-role-bindings %v, err: %v", clusterResourceName, err) } - // Note: Operator-managed CRDs (ICMSRequest, StorageRequest, MiniServices) have owner references - // to the NVCFBackend CRD and will be garbage collected when Helm deletes that CRD. + // Namespaced CR instances are removed with their owning namespaces. Their + // shared CRD definitions remain installed for other control planes. log.Infof("Successfully cleaned up resources for nvcfbackend %v/%v", nb.Namespace, nb.Name) return nil } +func validateClusterResourceOwnership( + ctx context.Context, + k8sClient kubernetes.Interface, + name string, + controlPlaneID string, +) error { + check := func(kind string, object metav1.Object, err error) error { + if k8serrors.IsNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("failed to get %s %s before cleanup: %w", kind, name, err) + } + if !nvcatypes.IsOwnedByControlPlane(object, controlPlaneID) { + return fmt.Errorf("refusing to delete %s %s owned by another control plane", kind, name) + } + return nil + } + + validatingWebhook, err := k8sClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(ctx, name, metav1.GetOptions{}) + if err := check("validating webhook configuration", validatingWebhook, err); err != nil { + return err + } + mutatingWebhook, err := k8sClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(ctx, name, metav1.GetOptions{}) + if err := check("mutating webhook configuration", mutatingWebhook, err); err != nil { + return err + } + clusterRole, err := k8sClient.RbacV1().ClusterRoles().Get(ctx, name, metav1.GetOptions{}) + if err := check("cluster role", clusterRole, err); err != nil { + return err + } + clusterRoleBinding, err := k8sClient.RbacV1().ClusterRoleBindings().Get(ctx, name, metav1.GetOptions{}) + return check("cluster role binding", clusterRoleBinding, err) +} + // DeleteNVCFBackend deletes an NVCFBackend resource func DeleteNVCFBackend( ctx context.Context, @@ -639,11 +689,19 @@ func deleteICMSRequests(ctx context.Context, dynamicClient dynamic.Interface, na const workloadNamespaceLabelSelector = "nvca.nvcf.nvidia.io/workload-instance-type" // deleteWorkloadNamespaces lists and deletes all NVCA workload namespaces (sr-*). -func deleteWorkloadNamespaces(ctx context.Context, k8sClient kubernetes.Interface) error { +func deleteWorkloadNamespaces(ctx context.Context, k8sClient kubernetes.Interface, controlPlaneIDs ...string) error { log := core.GetLogger(ctx) + controlPlaneID := "" + if len(controlPlaneIDs) != 0 { + controlPlaneID = controlPlaneIDs[0] + } + selector := workloadNamespaceLabelSelector + if controlPlaneID != "" { + selector += "," + nvcatypes.ControlPlaneIDLabel + "=" + controlPlaneID + } nsList, err := k8sClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{ - LabelSelector: workloadNamespaceLabelSelector, + LabelSelector: selector, }) if err != nil { return fmt.Errorf("failed to list workload namespaces: %w", err) @@ -674,6 +732,34 @@ func deleteWorkloadNamespaces(ctx context.Context, k8sClient kubernetes.Interfac return nil } +func deleteNamespaceForControlPlane(ctx context.Context, k8sClient kubernetes.Interface, namespace, controlPlaneID string) error { + if controlPlaneID == "" { + return k8sClient.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}) + } + ns, err := k8sClient.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + if err != nil { + return err + } + if !nvcatypes.IsOwnedByControlPlane(ns, controlPlaneID) { + return fmt.Errorf("refusing to delete namespace %s owned by another control plane", namespace) + } + return k8sClient.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}) +} + +func validateNamespaceOwnership(ctx context.Context, k8sClient kubernetes.Interface, namespace, controlPlaneID string) error { + ns, err := k8sClient.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + if !nvcatypes.IsOwnedByControlPlane(ns, controlPlaneID) { + return fmt.Errorf("refusing cleanup: namespace %s is not owned by control plane %q", namespace, controlPlaneID) + } + return nil +} + // waitForDeploymentRollout waits for a deployment to complete its rollout func waitForDeploymentRollout(ctx context.Context, k8sClient kubernetes.Interface, namespace, name string, timeout time.Duration) error { log := core.GetLogger(ctx) diff --git a/src/compute-plane-services/nvca/pkg/operator/cleanup/cleanup_test.go b/src/compute-plane-services/nvca/pkg/operator/cleanup/cleanup_test.go index a2a0c3aa1..370493081 100644 --- a/src/compute-plane-services/nvca/pkg/operator/cleanup/cleanup_test.go +++ b/src/compute-plane-services/nvca/pkg/operator/cleanup/cleanup_test.go @@ -24,9 +24,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -36,6 +38,7 @@ import ( nvidiaiov1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1" fakenvcaop "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/client/clientset/versioned/fake" + nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" ) func TestBackendNamespaces(t *testing.T) { @@ -107,6 +110,130 @@ func TestBackendNamespaces(t *testing.T) { } } +func TestCleanupBackendResourcesPreservesOtherControlPlane(t *testing.T) { + ctx := context.Background() + icmsGVR := schema.GroupVersionResource{Group: "nvca.nvcf.nvidia.io", Version: "v2beta1", Resource: "icmsrequests"} + dynamicClient := fakedynamic.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ + icmsGVR: "ICMSRequestList", + }) + label := func(id string) map[string]string { return map[string]string{nvcatypes.ControlPlaneIDLabel: id} } + backend := &nvidiaiov1.NVCFBackend{Spec: nvidiaiov1.NVCFBackendSpec{NVCFBackendSpecT: nvidiaiov1.NVCFBackendSpecT{ + ClusterConfig: nvidiaiov1.ClusterConfig{ControlPlaneID: "plane-a"}, + }}} + k8sClient := fake.NewSimpleClientset( + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "plane-a-nvca-system", Labels: label("plane-a")}}, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "plane-a-nvcf-backend", Labels: label("plane-a")}}, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "plane-a-sr-one", Labels: map[string]string{ + workloadNamespaceLabelSelector: "miniservice", nvcatypes.ControlPlaneIDLabel: "plane-a", + }}}, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "plane-b-nvca-system", Labels: label("plane-b")}}, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "plane-b-nvcf-backend", Labels: label("plane-b")}}, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "plane-b-sr-one", Labels: map[string]string{ + workloadNamespaceLabelSelector: "miniservice", nvcatypes.ControlPlaneIDLabel: "plane-b", + }}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "plane-a-nvca", Labels: label("plane-a")}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "plane-b-nvca", Labels: label("plane-b")}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "plane-a-nvca", Labels: label("plane-a")}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "plane-b-nvca", Labels: label("plane-b")}}, + &admissionregistrationv1.MutatingWebhookConfiguration{ObjectMeta: metav1.ObjectMeta{Name: "plane-a-nvca", Labels: label("plane-a")}}, + &admissionregistrationv1.MutatingWebhookConfiguration{ObjectMeta: metav1.ObjectMeta{Name: "plane-b-nvca", Labels: label("plane-b")}}, + &admissionregistrationv1.ValidatingWebhookConfiguration{ObjectMeta: metav1.ObjectMeta{Name: "plane-a-nvca", Labels: label("plane-a")}}, + &admissionregistrationv1.ValidatingWebhookConfiguration{ObjectMeta: metav1.ObjectMeta{Name: "plane-b-nvca", Labels: label("plane-b")}}, + ) + + require.NoError(t, CleanupBackendResources(ctx, k8sClient, dynamicClient, backend)) + for _, namespace := range []string{"plane-a-nvca-system", "plane-a-nvcf-backend", "plane-a-sr-one"} { + _, err := k8sClient.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + require.True(t, k8serrors.IsNotFound(err), "plane A namespace %s must be deleted", namespace) + } + _, err := k8sClient.RbacV1().ClusterRoles().Get(ctx, "plane-a-nvca", metav1.GetOptions{}) + require.True(t, k8serrors.IsNotFound(err), "plane A cluster role must be deleted") + _, err = k8sClient.RbacV1().ClusterRoleBindings().Get(ctx, "plane-a-nvca", metav1.GetOptions{}) + require.True(t, k8serrors.IsNotFound(err), "plane A cluster role binding must be deleted") + _, err = k8sClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(ctx, "plane-a-nvca", metav1.GetOptions{}) + require.True(t, k8serrors.IsNotFound(err), "plane A mutating webhook must be deleted") + _, err = k8sClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(ctx, "plane-a-nvca", metav1.GetOptions{}) + require.True(t, k8serrors.IsNotFound(err), "plane A validating webhook must be deleted") + + for _, namespace := range []string{"plane-b-nvca-system", "plane-b-nvcf-backend", "plane-b-sr-one"} { + _, err := k8sClient.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + require.NoError(t, err, "plane B namespace %s must survive", namespace) + } + _, err = k8sClient.RbacV1().ClusterRoles().Get(ctx, "plane-b-nvca", metav1.GetOptions{}) + require.NoError(t, err, "plane B cluster role must survive") + _, err = k8sClient.RbacV1().ClusterRoleBindings().Get(ctx, "plane-b-nvca", metav1.GetOptions{}) + require.NoError(t, err, "plane B cluster role binding must survive") + _, err = k8sClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(ctx, "plane-b-nvca", metav1.GetOptions{}) + require.NoError(t, err, "plane B mutating webhook must survive") + _, err = k8sClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(ctx, "plane-b-nvca", metav1.GetOptions{}) + require.NoError(t, err, "plane B validating webhook must survive") +} + +func TestCleanupBackendResourcesRefusesForeignClusterResourceWithDesiredName(t *testing.T) { + ctx := context.Background() + label := func(id string) map[string]string { return map[string]string{nvcatypes.ControlPlaneIDLabel: id} } + tests := []struct { + name string + resource runtime.Object + get func(k8sClient *fake.Clientset) error + }{ + { + name: "cluster role", + resource: &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "plane-a-nvca", Labels: label("plane-b")}}, + get: func(k8sClient *fake.Clientset) error { + _, err := k8sClient.RbacV1().ClusterRoles().Get(ctx, "plane-a-nvca", metav1.GetOptions{}) + return err + }, + }, + { + name: "cluster role binding", + resource: &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "plane-a-nvca", Labels: label("plane-b")}}, + get: func(k8sClient *fake.Clientset) error { + _, err := k8sClient.RbacV1().ClusterRoleBindings().Get(ctx, "plane-a-nvca", metav1.GetOptions{}) + return err + }, + }, + { + name: "mutating webhook", + resource: &admissionregistrationv1.MutatingWebhookConfiguration{ObjectMeta: metav1.ObjectMeta{Name: "plane-a-nvca", Labels: label("plane-b")}}, + get: func(k8sClient *fake.Clientset) error { + _, err := k8sClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(ctx, "plane-a-nvca", metav1.GetOptions{}) + return err + }, + }, + { + name: "validating webhook", + resource: &admissionregistrationv1.ValidatingWebhookConfiguration{ObjectMeta: metav1.ObjectMeta{Name: "plane-a-nvca"}}, + get: func(k8sClient *fake.Clientset) error { + _, err := k8sClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(ctx, "plane-a-nvca", metav1.GetOptions{}) + return err + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + backend := &nvidiaiov1.NVCFBackend{Spec: nvidiaiov1.NVCFBackendSpec{NVCFBackendSpecT: nvidiaiov1.NVCFBackendSpecT{ + ClusterConfig: nvidiaiov1.ClusterConfig{ControlPlaneID: "plane-a"}, + }}} + k8sClient := fake.NewSimpleClientset( + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "plane-a-nvca-system", Labels: label("plane-a")}}, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "plane-a-nvcf-backend", Labels: label("plane-a")}}, + tt.resource, + ) + dynamicClient := fakedynamic.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ + {Group: "nvca.nvcf.nvidia.io", Version: "v2beta1", Resource: "icmsrequests"}: "ICMSRequestList", + }) + + err := CleanupBackendResources(ctx, k8sClient, dynamicClient, backend) + require.ErrorContains(t, err, "owned by another control plane") + require.NoError(t, tt.get(k8sClient), "foreign cluster-scoped resource must survive") + _, err = k8sClient.CoreV1().Namespaces().Get(ctx, "plane-a-nvca-system", metav1.GetOptions{}) + require.NoError(t, err, "ownership preflight must run before deleting plane A resources") + }) + } +} + func TestIsSentinelBeingDeleted(t *testing.T) { tests := []struct { name string diff --git a/src/compute-plane-services/nvca/pkg/operator/cleanup/shutdown.go b/src/compute-plane-services/nvca/pkg/operator/cleanup/shutdown.go index c2cb0fe31..eeae1891f 100644 --- a/src/compute-plane-services/nvca/pkg/operator/cleanup/shutdown.go +++ b/src/compute-plane-services/nvca/pkg/operator/cleanup/shutdown.go @@ -233,8 +233,9 @@ func RunShutdownCleanup(ctx context.Context, opts ShutdownHandlerOptions) Shutdo log.Infof("Cleaned up NVCFBackend %s/%s", nb.Namespace, nb.Name) } - // Note: Operator-managed CRDs (ICMSRequest, StorageRequest, MiniServices) have owner references - // to the NVCFBackend CRD, so they will be garbage collected when Helm deletes that CRD. + // Namespaced custom resources are removed with their per-control-plane namespaces. + // Their shared CRD definitions are cluster prerequisites and must remain installed so + // another control plane in this cluster can continue serving those resource kinds. // Remove finalizers from RBAC resources (ClusterRole, ClusterRoleBinding, ServiceAccount) // This must be done before removing the sentinel finalizer to ensure the operator retains permissions diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel b/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel index bb8762e2c..a4e8842df 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel @@ -128,6 +128,7 @@ go_test( "backendk8scache_builder_test.go", "backendk8scache_test.go", "cli_test.go", + "control_plane_isolation_test.go", "crd_reconcile_test.go", "gpu_profiling_configmap_test.go", "miniservice_restrictions_test.go", diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/agent.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/agent.go index 73e73dc9b..f20804554 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/agent.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/agent.go @@ -51,6 +51,7 @@ import ( nvcaoptel "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/otel" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt" nvcaoptypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/types" + nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" ) var ( @@ -89,6 +90,7 @@ type AgentOptions struct { K8sVersionOverride string PriorityClassName string ClusterName string + ControlPlaneID string ClusterSource nvcaoptypes.ClusterSource NodeSelectorKey string NodeSelectorValue string @@ -217,6 +219,7 @@ func (o *AgentOptions) sanitizedString() string { sanitized.SystemNamespace = o.SystemNamespace sanitized.K8sVersionOverride = o.K8sVersionOverride sanitized.ClusterName = o.ClusterName + sanitized.ControlPlaneID = o.ControlPlaneID sanitized.ClusterSource = o.ClusterSource sanitized.PriorityClassName = o.PriorityClassName sanitized.NodeSelectorKey = o.NodeSelectorKey @@ -412,6 +415,12 @@ func (a *Agent) getTickerEvents(ctx context.Context) <-chan *core.Event { func (a *Agent) Start(ctx context.Context) error { log := core.GetLogger(ctx) + if err := nvcatypes.ValidateControlPlaneID(a.ControlPlaneID); err != nil { + return fmt.Errorf("invalid control plane ID: %w", err) + } + if a.ControlPlaneID != "" && a.SystemNamespace == "" { + a.SystemNamespace = nvcatypes.ControlPlaneResourceName(a.ControlPlaneID, "nvca-operator") + } // agentStarted is used to ensure the agent handlers don't fire too early agentStarted := &atomic.Bool{} agentStarted.Store(false) @@ -471,6 +480,7 @@ func (a *Agent) Start(ctx context.Context) error { backendk8scache, _, err = NewBackendK8sCacheBuilder(). WithClients(backendK8sClients). WithSystemNamespace(a.SystemNamespace). + WithControlPlaneID(a.ControlPlaneID). WithK8sVersionOverride(a.K8sVersionOverride). WithNGCServiceKeyFetcher(a.TokenFetcher). WithNVCAImageRepo(a.NVCAImageRepo). diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache.go index 399095457..570f0607d 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache.go @@ -99,6 +99,7 @@ type BackendK8sCache struct { httpClient *http.Client operatorNamespace string systemNamespace string + controlPlaneID string k8sVersionOverride string ngcServiceKeyFetcher cmnsecret.TokenFetcher nvcaRunAsUserID int64 @@ -233,6 +234,14 @@ func (b *BackendK8sCacheBuilder) WithClients(clients *kubeclients.KubeClients) * func (b *BackendK8sCacheBuilder) WithSystemNamespace(systemNamespace string) *BackendK8sCacheBuilder { next := *b next.operatorNamespace = systemNamespace + next.systemNamespace = systemNamespace + return &next +} + +// WithControlPlaneID sets the stable identity used for resource isolation. +func (b *BackendK8sCacheBuilder) WithControlPlaneID(controlPlaneID string) *BackendK8sCacheBuilder { + next := *b + next.controlPlaneID = controlPlaneID return &next } @@ -397,7 +406,9 @@ func (b *BackendK8sCacheBuilder) Start(ctx context.Context) (*BackendK8sCache, < clients: b.clients, eventBroadcaster: eventBroadcaster, eventRecorder: eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "nvca-operator"}), + operatorNamespace: b.operatorNamespace, systemNamespace: b.systemNamespace, + controlPlaneID: b.controlPlaneID, ngcServiceKeyFetcher: b.ngcServiceKeyFetcher, k8sVersionOverride: b.k8sVersionOverride, tracer: b.tracer, @@ -773,6 +784,9 @@ func addConfigMapInformers(ctx context.Context, c *BackendK8sCache) error { func (bc *BackendK8sCache) CreateOrUpdateNVCFBackend(ctx context.Context, deltaNB *nvidiaiov1.NVCFBackend) error { log := core.GetLogger(ctx) deltaNB = deltaNB.DeepCopy() + if err := applyControlPlaneIdentity(deltaNB, bc.controlPlaneID); err != nil { + return err + } log.Debugf("create or update NVCFBackend %s/%s", bc.operatorNamespace, deltaNB.Name) deltaNB.Namespace = bc.operatorNamespace if bc.nvcaOTELConfig != nil { @@ -1060,10 +1074,14 @@ func (bc *BackendK8sCache) SyncNVCFBackend(ctx context.Context, nb *nvidiaiov1.N } func (bc *BackendK8sCache) syncNVCFBackend(ctx context.Context, nb *nvidiaiov1.NVCFBackend, forceRollout bool) error { + nb = nb.DeepCopy() log := core.GetLogger(ctx).WithFields(logrus.Fields{ "backend": nb.Name, "namespace": nb.Namespace, }) + if err := bc.validateAndApplyControlPlaneScope(nb); err != nil { + return fmt.Errorf("refusing to reconcile NVCFBackend %s/%s: %w", nb.Namespace, nb.Name, err) + } // If the backend is being deleted, we need to cleanup the resources if !nb.ObjectMeta.DeletionTimestamp.IsZero() { @@ -1138,6 +1156,10 @@ func (bc *BackendK8sCache) syncNVCFBackend(ctx context.Context, nb *nvidiaiov1.N if err := mergeOverrides(nbMerged); err != nil { return err } + if err := bc.validateAndApplyControlPlaneScope(nbMerged); err != nil { + return fmt.Errorf("refusing to reconcile NVCFBackend %s/%s after applying overrides: %w", + nbMerged.Namespace, nbMerged.Name, err) + } // version cannot be empty if nbMerged.Spec.Version == "" { diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/cli.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/cli.go index b6af5b5bb..0220abf4e 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/cli.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/cli.go @@ -37,6 +37,7 @@ import ( "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/mirror" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt" nvcaoptypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/types" + nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" "github.com/bombsimon/logrusr/v4" corev1 "k8s.io/api/core/v1" @@ -63,6 +64,11 @@ func NewOperatorCommand() *cli.Command { Value: NVCAOperatorNamespace, Usage: "Namespace where NVCA Operator will watch for NVCFBackend types", }, + &cli.StringFlag{ + Name: "control-plane-id", + Usage: "Stable identity used to isolate this control plane in a shared cluster", + EnvVars: []string{"NVCF_CONTROL_PLANE_ID"}, + }, &cli.StringFlag{ Name: "nca-id", Usage: "NVIDIA Cloud Account Id (NCAId)", @@ -367,6 +373,20 @@ func NewOperatorCommand() *cli.Command { } } +func resolveOperatorSystemNamespace(controlPlaneID, configuredNamespace string, explicitlySet bool) string { + if controlPlaneID != "" && !explicitlySet { + return nvcatypes.ControlPlaneResourceName(controlPlaneID, NVCAOperatorNamespace) + } + return configuredNamespace +} + +func resolveOperatorSecretMirrorSourceNamespace(controlPlaneID, configuredNamespace string, explicitlySet bool) string { + if controlPlaneID != "" && !explicitlySet { + return nvcatypes.ControlPlaneResourceName(controlPlaneID, NVCAOperatorNamespace) + } + return configuredNamespace +} + func doAction(c *cli.Context) error { ctx := c.Context log := core.GetLogger(ctx) @@ -460,41 +480,46 @@ func doAction(c *cli.Context) error { return fmt.Errorf("identity-source is not supported for cluster source %s", clusterSource) } + controlPlaneID := c.String("control-plane-id") opts := &AgentOptions{ - NCAID: c.String("nca-id"), - KubeConfigPath: c.String("kubeconfig"), - K8sVersionOverride: c.String("k8s-version-override"), - SvcAddress: c.String("listen"), - AdminAddr: c.String("listen-admin"), - ShutdownAddr: c.String("listen-shutdown"), - PodName: c.String("pod-name"), - PodNamespace: c.String("pod-namespace"), - DeploymentName: c.String("deployment-name"), - SystemNamespace: c.String("system-namespace"), - PriorityClassName: c.String("priority-class-name"), - ClusterName: c.String("cluster-name"), - ClusterSource: clusterSource, - NodeSelectorKey: c.String("node-selector-key"), - NodeSelectorValue: c.String("node-selector-value"), - NVCAClusterManagementAPIURL: c.String("ngc-api-url"), - NVCFClusterID: c.String("cluster-id"), - NVCAClusterAPIRefreshInterval: c.Duration("nvca-cluster-api-refresh-interval"), - NVCAImageRepo: c.String("nvca-image-repo"), - NVCARunAsUserID: c.Int64("nvca-run-as-userid"), - NVCARunAsGroupID: c.Int64("nvca-run-as-groupid"), - GXCacheNamespace: c.String("nvca-gxcache-namespace"), - HelmRepositoryPrefix: c.String("nvca-helm-repository-prefix"), - EnableGXCache: c.Bool("enable-gxcache"), - DDCSIPAllowList: c.StringSlice("ddcs-ip-allowlist"), - K8sClusterNetworkCIDRs: c.StringSlice("k8s-cluster-network-cidrs"), - AgentResources: agentRR, - WebhookResources: webhookRR, - NVCACacheMountOptionsEnabled: c.Bool("nvca-cache-mount-options-enabled"), - NVCACacheMountOptions: c.String("nvca-cache-mount-options"), - NVCAWorkerDegradationPeriod: c.Duration("nvca-worker-degradation-period"), - NVCAWorkloadTolerations: workloadTolerations, - NVCAAgentTolerations: agentTolerations, - NVCASecretMirrorSourceNamespace: c.String("nvca-secret-mirror-source-namespace"), + NCAID: c.String("nca-id"), + KubeConfigPath: c.String("kubeconfig"), + K8sVersionOverride: c.String("k8s-version-override"), + SvcAddress: c.String("listen"), + AdminAddr: c.String("listen-admin"), + ShutdownAddr: c.String("listen-shutdown"), + PodName: c.String("pod-name"), + PodNamespace: c.String("pod-namespace"), + DeploymentName: c.String("deployment-name"), + SystemNamespace: resolveOperatorSystemNamespace( + controlPlaneID, c.String("system-namespace"), c.IsSet("system-namespace")), + ControlPlaneID: controlPlaneID, + PriorityClassName: c.String("priority-class-name"), + ClusterName: c.String("cluster-name"), + ClusterSource: clusterSource, + NodeSelectorKey: c.String("node-selector-key"), + NodeSelectorValue: c.String("node-selector-value"), + NVCAClusterManagementAPIURL: c.String("ngc-api-url"), + NVCFClusterID: c.String("cluster-id"), + NVCAClusterAPIRefreshInterval: c.Duration("nvca-cluster-api-refresh-interval"), + NVCAImageRepo: c.String("nvca-image-repo"), + NVCARunAsUserID: c.Int64("nvca-run-as-userid"), + NVCARunAsGroupID: c.Int64("nvca-run-as-groupid"), + GXCacheNamespace: c.String("nvca-gxcache-namespace"), + HelmRepositoryPrefix: c.String("nvca-helm-repository-prefix"), + EnableGXCache: c.Bool("enable-gxcache"), + DDCSIPAllowList: c.StringSlice("ddcs-ip-allowlist"), + K8sClusterNetworkCIDRs: c.StringSlice("k8s-cluster-network-cidrs"), + AgentResources: agentRR, + WebhookResources: webhookRR, + NVCACacheMountOptionsEnabled: c.Bool("nvca-cache-mount-options-enabled"), + NVCACacheMountOptions: c.String("nvca-cache-mount-options"), + NVCAWorkerDegradationPeriod: c.Duration("nvca-worker-degradation-period"), + NVCAWorkloadTolerations: workloadTolerations, + NVCAAgentTolerations: agentTolerations, + NVCASecretMirrorSourceNamespace: resolveOperatorSecretMirrorSourceNamespace( + controlPlaneID, c.String("nvca-secret-mirror-source-namespace"), + c.IsSet("nvca-secret-mirror-source-namespace")), NVCASecretMirrorLabelSelector: c.String("nvca-secret-mirror-label-selector"), GenerateImagePullSecret: c.Bool("generate-image-pull-secret"), AdditionalImagePullSecrets: additionalSecrets, diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/cli_test.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/cli_test.go index 4cc187f0a..a165e78d4 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/cli_test.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/cli_test.go @@ -373,6 +373,7 @@ func TestOperatorCommand_NVCAFlags(t *testing.T) { assert.True(t, flagNames["enable-gxcache"], "enable-gxcache flag should still exist") assert.True(t, flagNames["nca-id"], "nca-id flag should still exist") assert.True(t, flagNames["cluster-name"], "cluster-name flag should still exist") + assert.True(t, flagNames["control-plane-id"], "control-plane-id flag should exist") } func TestOperatorCommand_NVCAFlagDefaults(t *testing.T) { diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt/configmapclient_test.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt/configmapclient_test.go index e034d95f5..d97d0b145 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt/configmapclient_test.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt/configmapclient_test.go @@ -43,7 +43,7 @@ func dummyFetcher(yaml string, err error) func(context.Context) (*corev1.ConfigM } func TestConfigMapClient_GetCluster_Success(t *testing.T) { - sampleYAML := "clusterId: cid\nclusterName: name" + sampleYAML := "controlPlaneID: plane-a\nclusterId: cid\nclusterName: name" called := false extra := func(ctx context.Context, _ nvidiaiov1.EnvType, _ *clusterDTO, dest *Cluster) error { @@ -58,6 +58,7 @@ func TestConfigMapClient_GetCluster_Success(t *testing.T) { require.NotNil(t, cluster) assert.True(t, called, "extra mapper should be called") assert.Equal(t, "patched", cluster.NVCFBackend.Name) + assert.Equal(t, "plane-a", cluster.NVCFBackend.Spec.ClusterConfig.ControlPlaneID) } func TestConfigMapClient_GetCluster_FetchError(t *testing.T) { diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt/ngcclient.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt/ngcclient.go index ce33dfdee..0d8cd238b 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt/ngcclient.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt/ngcclient.go @@ -253,6 +253,7 @@ func withRootNVCFBackendMapper() clusterMapper { } dest.NVCFBackend.Spec.ClusterConfig = nvidiaiov1.ClusterConfig{ + ControlPlaneID: src.controlPlaneID(), ClusterID: src.ID, ClusterName: src.Name, Description: src.Description, diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt/types.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt/types.go index f2ea1c8f0..834629cd9 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt/types.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt/types.go @@ -143,13 +143,15 @@ type agentDTO struct { // clusterDTO represents an NVCA clusterDTO definition stored in the NGC API type clusterDTO struct { - ID string `json:"clusterId"` - Name string `json:"clusterName"` - Description string `json:"clusterDescription"` - GroupName string `json:"clusterGroupName"` - GroupID string `json:"clusterGroupId"` - Status types.ClusterStatus `json:"status"` - LastConnected metav1.Time `json:"nvcaLastConnected"` + ControlPlaneID string `json:"controlPlaneId,omitempty"` + ControlPlaneIDYAML string `json:"controlPlaneID,omitempty"` + ID string `json:"clusterId"` + Name string `json:"clusterName"` + Description string `json:"clusterDescription"` + GroupName string `json:"clusterGroupName"` + GroupID string `json:"clusterGroupId"` + Status types.ClusterStatus `json:"status"` + LastConnected metav1.Time `json:"nvcaLastConnected"` NCAID string `json:"ncaID"` @@ -216,6 +218,13 @@ type clusterDTO struct { Agent *agentDTO `json:"agent,omitempty"` } +func (c *clusterDTO) controlPlaneID() string { + if c.ControlPlaneID != "" { + return c.ControlPlaneID + } + return c.ControlPlaneIDYAML +} + // getClientID returns the OAuth client ID. func (c *clusterDTO) getClientID() string { return c.OAuthClientID diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/control_plane_isolation_test.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/control_plane_isolation_test.go new file mode 100644 index 000000000..1b9e2ad66 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/control_plane_isolation_test.go @@ -0,0 +1,284 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package operator + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + fakek8sclient "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" + + nvidiaiov1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1" + fakenvcaopclient "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/client/clientset/versioned/fake" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/cleanup" + nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" + nvcaconfig "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/types/nvca/config" +) + +func assertNoMutatingIsolationActions(t *testing.T, actions []ktesting.Action) { + t.Helper() + for _, action := range actions { + switch action.GetVerb() { + case "create", "update", "patch", "delete", "deletecollection": + t.Errorf("unexpected mutating action after isolation rejection: %s %s", action.GetVerb(), action.GetResource().Resource) + } + } +} + +func newNamedIsolationTestCache() (*BackendK8sCache, *fakek8sclient.Clientset, *fakenvcaopclient.Clientset) { + clients := mockKubeClients() + k8sClient := clients.K8s.(*fakek8sclient.Clientset) + nvcaClient := clients.NVCAOP.(*fakenvcaopclient.Clientset) + return &BackendK8sCache{ + clients: clients, + operatorNamespace: "plane-a-nvca-operator", + controlPlaneID: "plane-a", + }, k8sClient, nvcaClient +} + +func TestControlPlaneNamespaces(t *testing.T) { + nb := &nvidiaiov1.NVCFBackend{} + nb.Spec.ClusterConfig.ControlPlaneID = "plane-a" + + assert.Equal(t, "plane-a-nvca-system", getSystemNamespace(nb)) + assert.Equal(t, "plane-a-nvcf-backend", getRequestsNamespace(nb)) + + nb.Spec.ClusterConfig.SystemNamespace = "explicit-system" + nb.Spec.ClusterConfig.RequestsNamespace = "explicit-requests" + assert.Equal(t, "explicit-system", getSystemNamespace(nb)) + assert.Equal(t, "explicit-requests", getRequestsNamespace(nb)) +} + +func TestApplyControlPlaneIdentity(t *testing.T) { + nb := &nvidiaiov1.NVCFBackend{} + require.NoError(t, applyControlPlaneIdentity(nb, "plane-a")) + assert.Equal(t, "plane-a", nb.Spec.ClusterConfig.ControlPlaneID) + + nb.Spec.ClusterConfig.ControlPlaneID = "plane-b" + assert.Error(t, applyControlPlaneIdentity(nb, "plane-a")) + assert.Error(t, applyControlPlaneIdentity(nb, ""), "legacy operator must not reconcile a named backend") + + nb.Spec.ClusterConfig.ControlPlaneID = "default" + assert.Error(t, applyControlPlaneIdentity(nb, "")) +} + +func TestSyncNVCFBackendRejectsForeignScopeBeforeMutation(t *testing.T) { + tests := map[string]func(*nvidiaiov1.NVCFBackend){ + "control plane ID": func(nb *nvidiaiov1.NVCFBackend) { + nb.Spec.ClusterConfig.ControlPlaneID = "plane-b" + }, + "operator namespace": func(nb *nvidiaiov1.NVCFBackend) { + nb.Namespace = "plane-b-nvca-operator" + }, + "system namespace": func(nb *nvidiaiov1.NVCFBackend) { + nb.Spec.ClusterConfig.SystemNamespace = "plane-b-nvca-system" + }, + "requests namespace": func(nb *nvidiaiov1.NVCFBackend) { + nb.Spec.ClusterConfig.RequestsNamespace = "plane-b-nvcf-backend" + }, + "deleting foreign backend": func(nb *nvidiaiov1.NVCFBackend) { + nb.Spec.ClusterConfig.ControlPlaneID = "plane-b" + nb.Finalizers = []string{cleanup.NVCAOperatorFinalizer} + now := metav1.NewTime(time.Now()) + nb.DeletionTimestamp = &now + }, + } + + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + bc, k8sClient, nvcaClient := newNamedIsolationTestCache() + nb := getTestNVCFBackendMinimal() + nb.Namespace = bc.operatorNamespace + nb.Spec.Overrides = nil + nb.Spec.ClusterConfig.ControlPlaneID = bc.controlPlaneID + mutate(nb) + original := nb.DeepCopy() + + err := bc.syncNVCFBackend(t.Context(), nb, false) + require.ErrorContains(t, err, "refusing to reconcile NVCFBackend") + assert.Equal(t, original, nb, "sync must not mutate the informer object") + assertNoMutatingIsolationActions(t, k8sClient.Actions()) + assertNoMutatingIsolationActions(t, nvcaClient.Actions()) + }) + } +} + +func TestSyncNVCFBackendRejectsForeignOverrideBeforeMutation(t *testing.T) { + tests := map[string]func(*nvidiaiov1.NVCFBackendSpecT){ + "control plane ID": func(overrides *nvidiaiov1.NVCFBackendSpecT) { + overrides.ClusterConfig.ControlPlaneID = "plane-b" + }, + "system namespace": func(overrides *nvidiaiov1.NVCFBackendSpecT) { + overrides.ClusterConfig.SystemNamespace = "plane-b-nvca-system" + }, + "requests namespace": func(overrides *nvidiaiov1.NVCFBackendSpecT) { + overrides.ClusterConfig.RequestsNamespace = "plane-b-nvcf-backend" + }, + } + + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + bc, k8sClient, nvcaClient := newNamedIsolationTestCache() + nb := getTestNVCFBackendMinimal() + nb.Namespace = bc.operatorNamespace + nb.Finalizers = []string{cleanup.NVCAOperatorFinalizer} + nb.Spec.ClusterConfig.ControlPlaneID = bc.controlPlaneID + nb.Spec.Overrides = &nvidiaiov1.NVCFBackendSpecT{Version: nb.Spec.Version} + mutate(nb.Spec.Overrides) + _, err := nvcaClient.NvcfV1().NVCFBackends(bc.operatorNamespace).Create( + t.Context(), nb, metav1.CreateOptions{}) + require.NoError(t, err) + k8sClient.ClearActions() + nvcaClient.ClearActions() + + err = bc.syncNVCFBackend(t.Context(), nb.DeepCopy(), false) + require.ErrorContains(t, err, "refusing to reconcile NVCFBackend") + assertNoMutatingIsolationActions(t, k8sClient.Actions()) + assertNoMutatingIsolationActions(t, nvcaClient.Actions()) + }) + } +} + +func TestValidateAndApplyControlPlaneScopeCompatibility(t *testing.T) { + tests := []struct { + name string + operatorNamespace string + operatorID string + backend *nvidiaiov1.NVCFBackend + expectedID string + }{ + { + name: "legacy remains unscoped", + operatorNamespace: NVCAOperatorNamespace, + backend: &nvidiaiov1.NVCFBackend{ObjectMeta: metav1.ObjectMeta{ + Namespace: NVCAOperatorNamespace, + }}, + }, + { + name: "named backend is normalized", + operatorNamespace: "plane-a-nvca-operator", + operatorID: "plane-a", + backend: &nvidiaiov1.NVCFBackend{ObjectMeta: metav1.ObjectMeta{ + Namespace: "plane-a-nvca-operator", + }}, + expectedID: "plane-a", + }, + { + name: "named backend accepts its derived namespaces", + operatorNamespace: "plane-a-nvca-operator", + operatorID: "plane-a", + backend: &nvidiaiov1.NVCFBackend{ + ObjectMeta: metav1.ObjectMeta{Namespace: "plane-a-nvca-operator"}, + Spec: nvidiaiov1.NVCFBackendSpec{NVCFBackendSpecT: nvidiaiov1.NVCFBackendSpecT{ + ClusterConfig: nvidiaiov1.ClusterConfig{ + ControlPlaneID: "plane-a", + SystemNamespace: "plane-a-nvca-system", + RequestsNamespace: "plane-a-nvcf-backend", + }, + }}, + }, + expectedID: "plane-a", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + bc := &BackendK8sCache{operatorNamespace: tt.operatorNamespace, controlPlaneID: tt.operatorID} + require.NoError(t, bc.validateAndApplyControlPlaneScope(tt.backend)) + assert.Equal(t, tt.expectedID, tt.backend.Spec.ClusterConfig.ControlPlaneID) + }) + } +} + +func TestResolveOperatorSystemNamespace(t *testing.T) { + assert.Equal(t, "plane-a-nvca-operator", + resolveOperatorSystemNamespace("plane-a", NVCAOperatorNamespace, false)) + assert.Equal(t, "custom-operator", + resolveOperatorSystemNamespace("plane-a", "custom-operator", true)) + assert.Equal(t, NVCAOperatorNamespace, + resolveOperatorSystemNamespace("", NVCAOperatorNamespace, false)) +} + +func TestResolveOperatorSecretMirrorSourceNamespace(t *testing.T) { + assert.Equal(t, "plane-a-nvca-operator", + resolveOperatorSecretMirrorSourceNamespace("plane-a", NVCAOperatorNamespace, false)) + assert.Equal(t, "shared-secrets", + resolveOperatorSecretMirrorSourceNamespace("plane-a", "shared-secrets", true)) + assert.Equal(t, NVCAOperatorNamespace, + resolveOperatorSecretMirrorSourceNamespace("", NVCAOperatorNamespace, false)) +} + +func TestScopeWebhookForControlPlane(t *testing.T) { + webhook := admissionregistrationv1.MutatingWebhook{ + Name: "mutate.nvca.nvcf.nvidia.io", + NamespaceSelector: &metav1.LabelSelector{}, + } + scopeMutatingWebhook(&webhook, "plane-a") + assert.Equal(t, "plane-a.mutate.nvca.nvcf.nvidia.io", webhook.Name) + assert.Equal(t, "plane-a", webhook.NamespaceSelector.MatchLabels[nvcatypes.ControlPlaneIDLabel]) +} + +func TestControlPlaneClusterResourceName(t *testing.T) { + nb := &nvidiaiov1.NVCFBackend{} + assert.Equal(t, "nvca", controlPlaneClusterResourceName(nb, "nvca")) + nb.Spec.ClusterConfig.ControlPlaneID = "plane-a" + assert.Equal(t, "plane-a-nvca", controlPlaneClusterResourceName(nb, "nvca")) +} + +func TestBackendK8sCacheBuilderPropagatesNamespaceAndControlPlane(t *testing.T) { + builder := NewBackendK8sCacheBuilder(). + WithSystemNamespace("plane-a-nvca-operator"). + WithControlPlaneID("plane-a") + + assert.Equal(t, "plane-a-nvca-operator", builder.operatorNamespace) + assert.Equal(t, "plane-a-nvca-operator", builder.systemNamespace) + assert.Equal(t, "plane-a", builder.controlPlaneID) +} + +func TestBackendK8sCacheStartPropagatesOperatorNamespace(t *testing.T) { + ctx, cancel := context.WithCancel(newTestContext()) + defer cancel() + + cache, _, err := NewBackendK8sCacheBuilder(). + WithClients(mockKubeClients()). + WithSystemNamespace("plane-a-nvca-operator"). + WithControlPlaneID("plane-a"). + Start(ctx) + require.NoError(t, err) + assert.Equal(t, "plane-a-nvca-operator", cache.operatorNamespace) + assert.Equal(t, "plane-a-nvca-operator", cache.systemNamespace) + assert.Equal(t, "plane-a", cache.controlPlaneID) +} + +func TestControlPlaneAppLabels(t *testing.T) { + legacy := getAppLabels() + assert.NotContains(t, legacy, nvcatypes.ControlPlaneIDLabel) + + named := getAppLabels("plane-a") + assert.Equal(t, "plane-a", named[nvcatypes.ControlPlaneIDLabel]) +} + +func TestAgentConfigCarriesControlPlaneID(t *testing.T) { + data, err := encodeAgentConfig(nvcaconfig.Config{}, nvcaconfig.Config{}, nil, agentHostOverrides{ + ControlPlaneID: "plane-a", + }) + assert.NoError(t, err) + assert.Contains(t, string(data), "controlPlaneID: plane-a") +} + +func TestAgentConfigLegacyOmitsControlPlaneID(t *testing.T) { + data, err := encodeAgentConfig(nvcaconfig.Config{}, nvcaconfig.Config{}, nil, agentHostOverrides{ + ICMSHostHeaderOverride: "legacy.example.test", + }) + assert.NoError(t, err) + assert.NotContains(t, string(data), "controlPlaneID") +} diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/gpu_profiling_configmap_test.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/gpu_profiling_configmap_test.go index 07e4aff1d..a001bbfdf 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/gpu_profiling_configmap_test.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/gpu_profiling_configmap_test.go @@ -70,3 +70,77 @@ func TestSetupGPUProfilingConfigMap(t *testing.T) { assert.Equal(t, "fn-1,fn-2", mirrored.Data["functionIds"]) }) } + +func TestNamedControlPlaneConfigMapMirrorsUseOperatorNamespace(t *testing.T) { + const ( + operatorNS = "plane-a-nvca-operator" + systemNS = "plane-a-nvca-system" + ) + nb := &nvidiaiov1.NVCFBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "plane-a", Namespace: operatorNS}, + } + nb.Spec.ClusterConfig.ControlPlaneID = "plane-a" + require.Equal(t, systemNS, getSystemNamespace(nb)) + + t.Run("required annotations ConfigMap", func(t *testing.T) { + ctx := newTestContext() + clients := mockKubeClientsForIntegrationTests() + bc := &BackendK8sCache{clients: clients, operatorNamespace: operatorNS} + src := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: nvcfCustomAnnotationsConfigMapName, Namespace: operatorNS}, + Data: map[string]string{"annotations": `{"owner":"plane-a"}`}, + } + _, err := clients.K8s.CoreV1().ConfigMaps(operatorNS).Create(ctx, src, metav1.CreateOptions{}) + require.NoError(t, err) + + require.NoError(t, bc.mirrorConfigMap(ctx, nb, nvcfCustomAnnotationsConfigMapName)) + mirrored, err := clients.K8s.CoreV1().ConfigMaps(systemNS).Get( + ctx, nvcfCustomAnnotationsConfigMapName, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, src.Data, mirrored.Data) + }) + + t.Run("optional GPU profiling ConfigMap", func(t *testing.T) { + ctx := newTestContext() + clients := mockKubeClientsForIntegrationTests() + bc := &BackendK8sCache{clients: clients, operatorNamespace: operatorNS} + src := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: nvcfGPUProfilingConfigMapName, Namespace: operatorNS}, + Data: map[string]string{"functionIds": "fn-plane-a"}, + } + _, err := clients.K8s.CoreV1().ConfigMaps(operatorNS).Create(ctx, src, metav1.CreateOptions{}) + require.NoError(t, err) + + require.NoError(t, bc.setupGPUProfilingConfigMap(ctx, nb)) + mirrored, err := clients.K8s.CoreV1().ConfigMaps(systemNS).Get( + ctx, nvcfGPUProfilingConfigMapName, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, src.Data, mirrored.Data) + }) + + t.Run("does not consume same-named ConfigMaps from foreign or legacy namespaces", func(t *testing.T) { + ctx := newTestContext() + clients := mockKubeClientsForIntegrationTests() + bc := &BackendK8sCache{clients: clients, operatorNamespace: operatorNS} + for _, namespace := range []string{"plane-b-nvca-operator", NVCAOperatorNamespace} { + for _, name := range []string{nvcfCustomAnnotationsConfigMapName, nvcfGPUProfilingConfigMapName} { + _, err := clients.K8s.CoreV1().ConfigMaps(namespace).Create(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Data: map[string]string{"source": namespace}, + }, metav1.CreateOptions{}) + require.NoError(t, err) + } + } + + err := bc.mirrorConfigMap(ctx, nb, nvcfCustomAnnotationsConfigMapName) + assert.True(t, k8serr.IsNotFound(err), "required mirror must not fall back to another plane") + _, err = clients.K8s.CoreV1().ConfigMaps(systemNS).Get( + ctx, nvcfCustomAnnotationsConfigMapName, metav1.GetOptions{}) + assert.True(t, k8serr.IsNotFound(err), "foreign annotations must not be mirrored") + + require.NoError(t, bc.setupGPUProfilingConfigMap(ctx, nb)) + _, err = clients.K8s.CoreV1().ConfigMaps(systemNS).Get( + ctx, nvcfGPUProfilingConfigMapName, metav1.GetOptions{}) + assert.True(t, k8serr.IsNotFound(err), "foreign profiling config must not be mirrored") + }) +} diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/miniservice_restrictions.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/miniservice_restrictions.go index a4bd6554f..53d30a12e 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/miniservice_restrictions.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/miniservice_restrictions.go @@ -63,7 +63,7 @@ func (bc *BackendK8sCache) setupMiniServiceRBACConfigmap(ctx context.Context, nb Name: MiniServiceRBACConfigmapName, Namespace: getSystemNamespace(nb), Annotations: getNBAnnotations(nb), - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Data: rbacData, } @@ -85,8 +85,8 @@ func (bc *BackendK8sCache) setupMiniServiceValidatingWebhook(ctx context.Context vw := &admissionregistrationv1.ValidatingWebhookConfiguration{ ObjectMeta: metav1.ObjectMeta{ - Name: nvcaoptypes.NVCAModuleName, - Labels: getAppLabels(), + Name: controlPlaneClusterResourceName(nb, nvcaoptypes.NVCAModuleName), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Webhooks: []admissionregistrationv1.ValidatingWebhook{ { @@ -158,6 +158,9 @@ func (bc *BackendK8sCache) setupMiniServiceValidatingWebhook(ctx context.Context }, }, } + for i := range vw.Webhooks { + scopeValidatingWebhook(&vw.Webhooks[i], nb.Spec.ClusterConfig.ControlPlaneID) + } return bc.createOrUpdateValidatingWebhookConfiguration(ctx, vw) } diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go index 501219933..26b1a058c 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go @@ -213,7 +213,7 @@ var ( ) func getSystemNamespace(nb *nvidiaiov1.NVCFBackend) string { - systemNamespace := DefaultNVCASystemNamespace + systemNamespace := nvcatypes.ControlPlaneResourceName(nb.Spec.ClusterConfig.ControlPlaneID, DefaultNVCASystemNamespace) if nb.Spec.ClusterConfig.SystemNamespace != "" { systemNamespace = nb.Spec.ClusterConfig.SystemNamespace } @@ -222,7 +222,7 @@ func getSystemNamespace(nb *nvidiaiov1.NVCFBackend) string { } func getRequestsNamespace(nb *nvidiaiov1.NVCFBackend) string { - requestsNamespace := DefaultNVCARequestsNamespace + requestsNamespace := nvcatypes.ControlPlaneResourceName(nb.Spec.ClusterConfig.ControlPlaneID, DefaultNVCARequestsNamespace) if nb.Spec.ClusterConfig.RequestsNamespace != "" { requestsNamespace = nb.Spec.ClusterConfig.RequestsNamespace } @@ -248,6 +248,7 @@ func (bc *BackendK8sCache) setupRequestsNamespace(ctx context.Context, nb *nvidi ManagedbyLabelKey: nvcaoptypes.NVCAModuleName, nvcatypes.WorkloadInstanceTypeLabel: WorkloadInstanceTypeValuePodSpec, } + labels = nvcatypes.AddControlPlaneLabel(labels, nb.Spec.ClusterConfig.ControlPlaneID) reqNSObj := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -296,7 +297,7 @@ func (bc *BackendK8sCache) setupSystemNamespace(ctx context.Context, nb *nvidiai ObjectMeta: metav1.ObjectMeta{ Name: systemNamespace, Annotations: getNBAnnotations(nb), - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, } @@ -310,7 +311,7 @@ func (bc *BackendK8sCache) setupSystemNamespace(ctx context.Context, nb *nvidiai Name: systemNamespace, Namespace: systemNamespace, Annotations: getNBAnnotations(nb), - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Spec: corev1.ResourceQuotaSpec{ ScopeSelector: &corev1.ScopeSelector{ @@ -683,7 +684,7 @@ func (bc *BackendK8sCache) setupNGCServiceAPIKeySecret(ctx context.Context, nb * Name: NGCServiceAPIKeySecretName, Namespace: getSystemNamespace(nb), Annotations: getNBAnnotations(nb), - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Data: map[string][]byte{ NGCServiceAPIKeySecretDataKey: []byte(ngcServiceAPIStr), @@ -711,7 +712,7 @@ func (bc *BackendK8sCache) setupOTELConfigSecret(ctx context.Context, nb *nvidia Name: OTELConfigSecretName, Namespace: getSystemNamespace(nb), Annotations: getNBAnnotations(nb), - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Data: map[string][]byte{ OTELExporterSecretKey: []byte(exporter), @@ -732,7 +733,7 @@ func (bc *BackendK8sCache) setupNVCARBAC(ctx context.Context, nb *nvidiaiov1.NVC Name: nvcaoptypes.NVCAModuleName, Namespace: getSystemNamespace(nb), Annotations: getNBAnnotations(nb), - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, AutomountServiceAccountToken: boolPtr(false), ImagePullSecrets: getImagePullSecretReferences(ctx, nb, bc.generateImagePullSecret, bc.additionalImagePullSecrets), @@ -748,11 +749,17 @@ func (bc *BackendK8sCache) setupNVCARBAC(ctx context.Context, nb *nvidiaiov1.NVC crudVerbs := []string{"get", "list", "watch", "create", "update", "delete", "patch"} crudWithCollectionVerbs := []string{"get", "list", "watch", "create", "update", "delete", "deletecollection", "patch"} + // A named control plane gets uniquely named RBAC and all controllers enforce + // its namespace/label identity. The agent still needs cluster-level bootstrap + // permissions to create and remove dynamic workload namespaces and their + // RoleBindings. Kubernetes RBAC cannot constrain those verbs by namespace + // prefix or label, so this is operational control-plane isolation, not an + // adversarial multi-tenant authorization boundary. cr := &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ - Name: nvcaoptypes.NVCAModuleName, + Name: controlPlaneClusterResourceName(nb, nvcaoptypes.NVCAModuleName), Annotations: getNBAnnotations(nb), - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Rules: []rbacv1.PolicyRule{ { @@ -799,7 +806,7 @@ func (bc *BackendK8sCache) setupNVCARBAC(ctx context.Context, nb *nvidiaiov1.NVC { APIGroups: []string{"admissionregistration.k8s.io"}, Resources: []string{"mutatingwebhookconfigurations", "validatingwebhookconfigurations"}, - ResourceNames: []string{nvcaoptypes.NVCAModuleName}, + ResourceNames: []string{controlPlaneClusterResourceName(nb, nvcaoptypes.NVCAModuleName)}, Verbs: []string{"get", "list", "watch"}, }, // NvSnap integration (PR-3 + PR-5): NVCA agent reads and @@ -958,9 +965,9 @@ func (bc *BackendK8sCache) setupNVCARBAC(ctx context.Context, nb *nvidiaiov1.NVC crb := &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: nvcaoptypes.NVCAModuleName, + Name: controlPlaneClusterResourceName(nb, nvcaoptypes.NVCAModuleName), Annotations: getNBAnnotations(nb), - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Subjects: []rbacv1.Subject{ { @@ -971,7 +978,7 @@ func (bc *BackendK8sCache) setupNVCARBAC(ctx context.Context, nb *nvidiaiov1.NVC }, RoleRef: rbacv1.RoleRef{ Kind: "ClusterRole", - Name: nvcaoptypes.NVCAModuleName, + Name: controlPlaneClusterResourceName(nb, nvcaoptypes.NVCAModuleName), APIGroup: "rbac.authorization.k8s.io", }, } @@ -987,7 +994,8 @@ func (bc *BackendK8sCache) setupNVCARBAC(ctx context.Context, nb *nvidiaiov1.NVC func (bc *BackendK8sCache) mirrorConfigMap(ctx context.Context, nb *nvidiaiov1.NVCFBackend, srcName string) error { log := core.GetLogger(ctx) - srcCM, err := bc.clients.K8s.CoreV1().ConfigMaps(NVCAOperatorNamespace).Get(ctx, srcName, metav1.GetOptions{}) + srcCM, err := bc.clients.K8s.CoreV1().ConfigMaps(bc.operatorConfigMapNamespace()).Get( + ctx, srcName, metav1.GetOptions{}) if err != nil { log.Errorf("failed to get source configmap %v", srcName) return err @@ -1003,13 +1011,23 @@ func (bc *BackendK8sCache) mirrorConfigMap(ctx context.Context, nb *nvidiaiov1.N return bc.createOrUpdateConfigMap(ctx, &cmTemplate) } +func (bc *BackendK8sCache) operatorConfigMapNamespace() string { + if bc.operatorNamespace != "" { + return bc.operatorNamespace + } + // Preserve the legacy namespace for callers that construct the cache + // directly without going through BackendK8sCacheBuilder. + return NVCAOperatorNamespace +} + // setupGPUProfilingConfigMap mirrors the chart-created nvca-gpu-profiling-config ConfigMap // into the agent's system namespace, where NVCA reads it live. Unlike mirrorConfigMap it is // optional: an absent source is skipped (profiling stays off) rather than failing reconcile. func (bc *BackendK8sCache) setupGPUProfilingConfigMap(ctx context.Context, nb *nvidiaiov1.NVCFBackend) error { log := core.GetLogger(ctx) - srcCM, err := bc.clients.K8s.CoreV1().ConfigMaps(NVCAOperatorNamespace).Get(ctx, nvcfGPUProfilingConfigMapName, metav1.GetOptions{}) + srcCM, err := bc.clients.K8s.CoreV1().ConfigMaps(bc.operatorConfigMapNamespace()).Get( + ctx, nvcfGPUProfilingConfigMapName, metav1.GetOptions{}) if err != nil { if k8serr.IsNotFound(err) { log.Debugf("%v configmap not found, skipping GPU profiling config mirror", nvcfGPUProfilingConfigMapName) @@ -1052,7 +1070,7 @@ func (bc *BackendK8sCache) setupNetworkPoliciesConfigMap(ctx context.Context, nb Name: NetworkPoliciesConfigmapName, Namespace: getSystemNamespace(nb), Annotations: getNBAnnotations(nb), - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Data: mergedNetPolData, } @@ -1178,7 +1196,7 @@ func (bc *BackendK8sCache) setupVaultConfigmap(ctx context.Context, nb *nvidiaio Name: NVCAVaultConfigmapName, Namespace: getSystemNamespace(nb), Annotations: getNBAnnotations(nb), - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Data: getVaultConfigData(nb), } @@ -1220,7 +1238,7 @@ func (bc *BackendK8sCache) setupStaticGPUConfigMap(ctx context.Context, nb *nvid Name: NVCAConfigmapName, Namespace: getSystemNamespace(nb), Annotations: getNBAnnotations(nb), - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Data: cmData, } @@ -1248,7 +1266,7 @@ func (bc *BackendK8sCache) setupOAuthClientSecrets(ctx context.Context, nb *nvid Name: clientKeySecretName, Namespace: getSystemNamespace(nb), Annotations: getNBAnnotations(nb), - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Data: map[string][]byte{ OAuthClientKeySecretDataKey: []byte(oauthConfig.ClientSecretKey), @@ -1266,7 +1284,7 @@ func (bc *BackendK8sCache) setupOAuthClientSecrets(ctx context.Context, nb *nvid Name: clientIDSecretName, Namespace: getSystemNamespace(nb), Annotations: getNBAnnotations(nb), - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Data: map[string][]byte{ OAuthClientIDSecretDataKey: []byte(oauthConfig.ClientID), @@ -1321,7 +1339,7 @@ func (bc *BackendK8sCache) newAgentConfigConfigMap( Name: agentConfigConfigMapName, Namespace: getSystemNamespace(nb), Annotations: getNBAnnotations(nb), - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Data: map[string]string{ agentConfigFile: string(cb), @@ -1333,6 +1351,7 @@ type agentHostOverrides struct { ICMSHostHeaderOverride string HelmReValServiceHostHeaderOverride string NATSHostOverride *string + ControlPlaneID string } func agentHostOverrideConfig(nb *nvidiaiov1.NVCFBackend, envType nvidiaiov1.EnvType) agentHostOverrides { @@ -1345,6 +1364,7 @@ func agentHostOverrideConfig(nb *nvidiaiov1.NVCFBackend, envType nvidiaiov1.EnvT ICMSHostHeaderOverride: nb.Spec.ICMSConfig.ICMSServiceHostHeaderOverride, HelmReValServiceHostHeaderOverride: reValHost, NATSHostOverride: nb.Spec.AgentConfig.NATSHostOverride, + ControlPlaneID: nb.Spec.ClusterConfig.ControlPlaneID, } } @@ -1364,7 +1384,8 @@ func encodeAgentConfig(cfg nvcaconfig.Config, mergeCfg nvcaconfig.Config, natsUR func applyAgentHostOverrides(data []byte, hostOverrides agentHostOverrides) ([]byte, error) { if hostOverrides.ICMSHostHeaderOverride == "" && hostOverrides.HelmReValServiceHostHeaderOverride == "" && - (hostOverrides.NATSHostOverride == nil || *hostOverrides.NATSHostOverride == "") { + (hostOverrides.NATSHostOverride == nil || *hostOverrides.NATSHostOverride == "") && + hostOverrides.ControlPlaneID == "" { return data, nil } @@ -1395,6 +1416,7 @@ func applyAgentHostOverrides(data []byte, hostOverrides agentHostOverrides) ([]b if hostOverrides.NATSHostOverride != nil { setYAMLString(agent, "NATSHostOverride", *hostOverrides.NATSHostOverride) } + setYAMLString(agent, "controlPlaneID", hostOverrides.ControlPlaneID) return yamlv3.Marshal(&doc) } @@ -1533,7 +1555,7 @@ func (bc *BackendK8sCache) setupImagePullSecrets(ctx context.Context, nb *nvidia Name: NVCAImagePullSecretName, Namespace: getSystemNamespace(nb), Annotations: getNBAnnotations(nb), - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Type: corev1.SecretTypeDockerConfigJson, Data: map[string][]byte{ @@ -1643,10 +1665,10 @@ func (bc *BackendK8sCache) setupNVCAService(ctx context.Context, nb *nvidiaiov1. Name: nvcaoptypes.NVCAModuleName, Namespace: getSystemNamespace(nb), Annotations: getNBAnnotations(nb), - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Spec: corev1.ServiceSpec{ - Selector: getAppLabels(), + Selector: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), Ports: []corev1.ServicePort{ { Name: nvcaoptypes.NVCAModuleName, @@ -2153,16 +2175,16 @@ func (bc *BackendK8sCache) setupNVCADeployment(ctx context.Context, original *nv Name: deployName, Namespace: getSystemNamespace(nb), Annotations: getNBAnnotations(nb), - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Spec: appsv1.DeploymentSpec{ Replicas: &replicas, Selector: &metav1.LabelSelector{ - MatchLabels: getAppLabels(), + MatchLabels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Spec: corev1.PodSpec{ AutomountServiceAccountToken: boolPtr(true), @@ -2392,7 +2414,7 @@ func (bc *BackendK8sCache) newAgentConfig(ctx context.Context, nb *nvidiaiov1.NV AdminAddr: fmt.Sprintf("127.0.0.1:%d", nvcaAdminPortHTTP), SystemNamespace: systemNamespace, RequestsNamespace: requestsNamespace, - NamespaceLabels: getAppLabels(), + NamespaceLabels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), ComputeBackend: backendType, HelmRepositoryPrefix: bc.helmRepositoryPrefix, HelmReValStageOAuthTokenURL: effectiveConfig.HelmReValStageOAuthTokenURL, @@ -2568,8 +2590,9 @@ func (bc *BackendK8sCache) setupNVCAMutatingWebhookConfiguration(ctx context.Con ) error { whc := &admissionregistrationv1.MutatingWebhookConfiguration{ ObjectMeta: metav1.ObjectMeta{ - Name: nvcaoptypes.NVCAModuleName, + Name: controlPlaneClusterResourceName(nb, nvcaoptypes.NVCAModuleName), Annotations: getNBAnnotations(nb), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, } @@ -2583,6 +2606,9 @@ func (bc *BackendK8sCache) setupNVCAMutatingWebhookConfiguration(ctx context.Con makeHelmStorageMutatingWebhook(nb, webhookCert), bc.makeHelmPersistentStorageWebhook(nb, webhookCert), makeNVCAMutatingWebhook(nb, webhookCert)) + for i := range whc.Webhooks { + scopeMutatingWebhook(&whc.Webhooks[i], nb.Spec.ClusterConfig.ControlPlaneID) + } return bc.createOrUpdateMutatingWebhookConfiguration(ctx, whc) } @@ -2709,7 +2735,7 @@ func (bc *BackendK8sCache) setupOTelCollectorConfigMap(ctx context.Context, nb * Name: NVCAOTelCollectorConfigMapName, Namespace: getSystemNamespace(nb), Annotations: getNBAnnotations(nb), - Labels: getAppLabels(), + Labels: getAppLabels(nb.Spec.ClusterConfig.ControlPlaneID), }, Data: configData, } diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/reconcile_helpers.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/reconcile_helpers.go index a4ff47137..dbb36b8e4 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/reconcile_helpers.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/reconcile_helpers.go @@ -35,6 +35,7 @@ import ( nvidiaiov1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1" nvcaoptypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/types" + nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" ) const ( @@ -94,12 +95,100 @@ func decodeEnvOverrides(b64 string) (map[string]string, error) { return envOverrides, nil } -func getAppLabels() map[string]string { - return map[string]string{ +func getAppLabels(controlPlaneIDs ...string) map[string]string { + labels := map[string]string{ InstanceLabelKey: nvcaoptypes.NVCAModuleName, ManagedbyLabelKey: NVCAOperatorName, NameLabelKey: nvcaoptypes.NVCAModuleName, } + if len(controlPlaneIDs) != 0 { + labels = nvcatypes.AddControlPlaneLabel(labels, controlPlaneIDs[0]) + } + return labels +} + +func controlPlaneClusterResourceName(nb *nvidiaiov1.NVCFBackend, legacyName string) string { + return nvcatypes.ControlPlaneResourceName(nb.Spec.ClusterConfig.ControlPlaneID, legacyName) +} + +func applyControlPlaneIdentity(nb *nvidiaiov1.NVCFBackend, operatorControlPlaneID string) error { + configured := nb.Spec.ClusterConfig.ControlPlaneID + if err := nvcatypes.ValidateControlPlaneID(configured); err != nil { + return fmt.Errorf("invalid NVCFBackend controlPlaneId: %w", err) + } + if err := nvcatypes.ValidateControlPlaneID(operatorControlPlaneID); err != nil { + return fmt.Errorf("invalid operator control plane ID: %w", err) + } + if operatorControlPlaneID == "" { + if configured != "" { + return fmt.Errorf("NVCFBackend controlPlaneId %q cannot be reconciled by a legacy operator", configured) + } + return nil + } + if configured != "" && configured != operatorControlPlaneID { + return fmt.Errorf("NVCFBackend controlPlaneId %q does not match operator control plane ID %q", configured, operatorControlPlaneID) + } + nb.Spec.ClusterConfig.ControlPlaneID = operatorControlPlaneID + return nil +} + +// validateAndApplyControlPlaneScope prevents one named operator from using an +// NVCFBackend to address another control plane's namespaces. It intentionally +// preserves the legacy operator's support for explicitly configured namespaces. +func (bc *BackendK8sCache) validateAndApplyControlPlaneScope(nb *nvidiaiov1.NVCFBackend) error { + if nb.Namespace != bc.operatorNamespace { + return fmt.Errorf("NVCFBackend namespace %q does not match operator namespace %q", + nb.Namespace, bc.operatorNamespace) + } + if err := applyControlPlaneIdentity(nb, bc.controlPlaneID); err != nil { + return err + } + if bc.controlPlaneID == "" { + return nil + } + + expectedSystemNamespace := nvcatypes.ControlPlaneResourceName( + bc.controlPlaneID, DefaultNVCASystemNamespace) + if configured := nb.Spec.ClusterConfig.SystemNamespace; configured != "" && configured != expectedSystemNamespace { + return fmt.Errorf("NVCFBackend system namespace %q does not match control plane %q namespace %q", + configured, bc.controlPlaneID, expectedSystemNamespace) + } + expectedRequestsNamespace := nvcatypes.ControlPlaneResourceName( + bc.controlPlaneID, DefaultNVCARequestsNamespace) + if configured := nb.Spec.ClusterConfig.RequestsNamespace; configured != "" && configured != expectedRequestsNamespace { + return fmt.Errorf("NVCFBackend requests namespace %q does not match control plane %q namespace %q", + configured, bc.controlPlaneID, expectedRequestsNamespace) + } + return nil +} + +func scopeNamespaceSelector(selector **metav1.LabelSelector, controlPlaneID string) { + if controlPlaneID == "" { + return + } + if *selector == nil { + *selector = &metav1.LabelSelector{} + } + if (*selector).MatchLabels == nil { + (*selector).MatchLabels = map[string]string{} + } + (*selector).MatchLabels[nvcatypes.ControlPlaneIDLabel] = controlPlaneID +} + +func scopeMutatingWebhook(webhook *admissionregistrationv1.MutatingWebhook, controlPlaneID string) { + if controlPlaneID == "" { + return + } + webhook.Name = controlPlaneID + "." + webhook.Name + scopeNamespaceSelector(&webhook.NamespaceSelector, controlPlaneID) +} + +func scopeValidatingWebhook(webhook *admissionregistrationv1.ValidatingWebhook, controlPlaneID string) { + if controlPlaneID == "" { + return + } + webhook.Name = controlPlaneID + "." + webhook.Name + scopeNamespaceSelector(&webhook.NamespaceSelector, controlPlaneID) } func getNBAnnotations(nb *nvidiaiov1.NVCFBackend) map[string]string { @@ -112,7 +201,7 @@ func getNBAnnotations(nb *nvidiaiov1.NVCFBackend) map[string]string { //nolint:dupl func (bc *BackendK8sCache) createOrUpdateNamespace(ctx context.Context, ns *v1.Namespace) error { // get and create if not exists - _, err := bc.clients.K8s.CoreV1().Namespaces().Get(ctx, ns.Name, metav1.GetOptions{}) + existing, err := bc.clients.K8s.CoreV1().Namespaces().Get(ctx, ns.Name, metav1.GetOptions{}) if err != nil { if k8serrors.IsNotFound(err) { _, err := bc.clients.K8s.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) @@ -123,7 +212,11 @@ func (bc *BackendK8sCache) createOrUpdateNamespace(ctx context.Context, ns *v1.N return fmt.Errorf("failed to get %v namespace, err: %v", ns.Name, err) } } else { + if id := ns.Labels[nvcatypes.ControlPlaneIDLabel]; id != "" && !nvcatypes.IsOwnedByControlPlane(existing, id) { + return fmt.Errorf("refusing to update namespace %s owned by another control plane", ns.Name) + } // update namespace with new labels + ns.ResourceVersion = existing.ResourceVersion _, err = bc.clients.K8s.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{}) if err != nil { return fmt.Errorf("failed to update %v namespace, err: %v", ns.Name, err) diff --git a/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel b/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel index eb31ee50c..09ecd8528 100644 --- a/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel @@ -100,6 +100,7 @@ go_test( "cachebackend_samba_test.go", "cachebackend_test.go", "controller_test.go", + "control_plane_isolation_test.go", "modelcache_cleanup_test.go", "modelcache_nvmesh_encrypt_test.go", "modelcache_test.go", diff --git a/src/compute-plane-services/nvca/pkg/storage/control_plane_isolation_test.go b/src/compute-plane-services/nvca/pkg/storage/control_plane_isolation_test.go new file mode 100644 index 000000000..96a61a653 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/control_plane_isolation_test.go @@ -0,0 +1,42 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package storage + +import ( + "testing" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + nvcav1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v1" + nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" +) + +func TestStorageRequestPredicateScopesNamespaceAndControlPlane(t *testing.T) { + filter := filterStorageRequest(nvcav1.SharedStorageRequest, "plane-a-nvcf-backend", "plane-a") + owned := &nvcav1.StorageRequest{ + ObjectMeta: metav1.ObjectMeta{Namespace: "plane-a-nvcf-backend", Labels: map[string]string{ + nvcatypes.ControlPlaneIDLabel: "plane-a", + }}, + Spec: nvcav1.StorageRequestSpec{Type: nvcav1.SharedStorageRequest}, + } + foreign := owned.DeepCopy() + foreign.Labels[nvcatypes.ControlPlaneIDLabel] = "plane-b" + wrongNamespace := owned.DeepCopy() + wrongNamespace.Namespace = "plane-b-nvcf-backend" + + assert.True(t, filter(owned)) + assert.False(t, filter(foreign)) + assert.False(t, filter(wrongNamespace)) +} + +func TestModelCacheDisabledForNamedControlPlane(t *testing.T) { + assert.Equal(t, []nvcav1.StorageRequestType{ + nvcav1.SharedStorageRequest, + nvcav1.InternalPersistentStorageRequest, + }, ControllerTypes(true, "plane-a")) + assert.Contains(t, ControllerTypes(true, ""), nvcav1.ModelCacheRequest) +} diff --git a/src/compute-plane-services/nvca/pkg/storage/controller.go b/src/compute-plane-services/nvca/pkg/storage/controller.go index df1a5752a..1cae4ab5f 100644 --- a/src/compute-plane-services/nvca/pkg/storage/controller.go +++ b/src/compute-plane-services/nvca/pkg/storage/controller.go @@ -19,6 +19,7 @@ package storage import ( "context" + "strings" "time" nvcaconfig "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/types/nvca/config" @@ -46,6 +47,7 @@ import ( "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v1alpha1" nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/client/clientset/versioned" + nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" ) var ( @@ -71,6 +73,7 @@ func init() { } type ControllerOptions struct { + ControlPlaneID string ICMSRequestNamespace string CSIVolumeMountOptions []string Metrics *metrics.Metrics @@ -94,6 +97,7 @@ func BuildController( reconcilerOpts := []ReconcilerOption{ WithNowFunc(opts.nowFunc), WithICMSRequestNamespace(opts.ICMSRequestNamespace), + WithControlPlaneID(opts.ControlPlaneID), WithCSIVolumeMountOptions(opts.CSIVolumeMountOptions), WithMetrics(opts.Metrics), } @@ -113,8 +117,8 @@ func BuildController( return buildControllerModelCache(r, mgr, opts) } - clusterwideEventHandler := handler.EnqueueRequestsFromMapFunc(getClusterWideEventHandlerMapFunc(storageReqType)) - storageReqPredicate := predicate.NewPredicateFuncs(filterOnStorageRequestType(storageReqType)) + clusterwideEventHandler := handler.EnqueueRequestsFromMapFunc(getClusterWideEventHandlerMapFunc(storageReqType, opts.ControlPlaneID)) + storageReqPredicate := predicate.NewPredicateFuncs(filterStorageRequest(storageReqType, opts.ICMSRequestNamespace, opts.ControlPlaneID)) b := builder. ControllerManagedBy(mgr). Named(string(storageReqType)). @@ -142,6 +146,38 @@ func BuildController( Complete(r) } +// ControllerTypes returns the safe controller set for one agent. Model cache +// is intentionally disabled in named mode because its legacy singleton init +// namespace and cluster-scoped persistent volumes are not yet identity-safe. +func ControllerTypes(cachingEnabled bool, controlPlaneID string) []nvcav1new.StorageRequestType { + sts := []nvcav1new.StorageRequestType{ + nvcav1new.SharedStorageRequest, + nvcav1new.InternalPersistentStorageRequest, + } + if cachingEnabled && controlPlaneID == "" { + sts = append(sts, nvcav1new.ModelCacheRequest) + } + return sts +} + +func filterStorageRequest(storageReqType nvcav1new.StorageRequestType, requestsNamespace, controlPlaneID string) func(client.Object) bool { + typeFilter := filterOnStorageRequestType(storageReqType) + return func(object client.Object) bool { + if !typeFilter(object) || !nvcatypes.IsOwnedByControlPlane(object, controlPlaneID) { + return false + } + if controlPlaneID == "" { + return true + } + switch object.(type) { + case *nvcav1new.StorageRequest, *nvcav2beta1.StorageRequest: + return strings.HasPrefix(object.GetNamespace(), controlPlaneID+"-") + default: + return true + } + } +} + func filterOnStorageRequestType(storageReqType nvcav1new.StorageRequestType) func(object client.Object) bool { return func(object client.Object) bool { if st, ok := object.(*nvcav1new.StorageRequest); ok { @@ -180,8 +216,15 @@ func getStorageRequestOwnerReference(storageReqType nvcav1new.StorageRequestType return metav1.OwnerReference{}, false } -func getClusterWideEventHandlerMapFunc(storageReqType nvcav1new.StorageRequestType) handler.MapFunc { +func getClusterWideEventHandlerMapFunc(storageReqType nvcav1new.StorageRequestType, controlPlaneIDs ...string) handler.MapFunc { return func(_ context.Context, o client.Object) []reconcile.Request { + controlPlaneID := "" + if len(controlPlaneIDs) != 0 { + controlPlaneID = controlPlaneIDs[0] + } + if !nvcatypes.IsOwnedByControlPlane(o, controlPlaneID) { + return nil + } labels := o.GetLabels() if len(labels) == 0 { return nil diff --git a/src/compute-plane-services/nvca/pkg/storage/reconcile.go b/src/compute-plane-services/nvca/pkg/storage/reconcile.go index 45309cae5..a3628287b 100644 --- a/src/compute-plane-services/nvca/pkg/storage/reconcile.go +++ b/src/compute-plane-services/nvca/pkg/storage/reconcile.go @@ -24,6 +24,7 @@ import ( "fmt" "io" "maps" + "strings" "sync/atomic" "time" @@ -99,6 +100,12 @@ func WithICMSRequestNamespace(icmsRequestNamespace string) ReconcilerOption { } } +func WithControlPlaneID(controlPlaneID string) ReconcilerOption { + return func(r *Reconciler) { + r.controlPlaneID = controlPlaneID + } +} + func WithNowFunc(nowFunc func() time.Time) ReconcilerOption { return func(r *Reconciler) { if nowFunc != nil { @@ -210,6 +217,7 @@ type Reconciler struct { cfg nvcaconfig.Config Client client.Client + controlPlaneID string ICMSRequestNamespace string Decoder runtime.Decoder clusterName string @@ -252,6 +260,10 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco if err != nil || st == nil { return reconcile.Result{}, err } + if r.controlPlaneID != "" && (!strings.HasPrefix(req.Namespace, r.controlPlaneID+"-") || !nvcatypes.IsOwnedByControlPlane(st, r.controlPlaneID)) { + log.V(1).Info("StorageRequest belongs to another control plane; ignoring", "controlPlaneID", r.controlPlaneID) + return reconcile.Result{}, nil + } if err := r.validateStorageRequest(st, ref); err != nil { return reconcile.Result{}, reconcile.TerminalError(err) } @@ -936,8 +948,12 @@ func getWorkloadLabels(obj client.Object) map[string]string { } func getClusterWideResourceLabels(st *nvcav1new.StorageRequest) map[string]string { - return map[string]string{ + result := map[string]string{ StorageRequestOwnerKey: st.Name, StorageRequestNamespaceKey: st.Namespace, } + if id := st.Labels[nvcatypes.ControlPlaneIDLabel]; id != "" { + result[nvcatypes.ControlPlaneIDLabel] = id + } + return result } diff --git a/src/compute-plane-services/nvca/pkg/types/BUILD.bazel b/src/compute-plane-services/nvca/pkg/types/BUILD.bazel index 91a591521..fbb954b4b 100644 --- a/src/compute-plane-services/nvca/pkg/types/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/types/BUILD.bazel @@ -7,6 +7,7 @@ go_library( name = "types", srcs = [ "common_labels.go", + "control_plane.go", "event_annotations.go", "gpu.go", "miniservice_types.go", @@ -28,7 +29,9 @@ go_library( "//src/compute-plane-services/nvca/vendor/gopkg.in/inf.v0:inf_v0", "//src/compute-plane-services/nvca/vendor/k8s.io/api/core/v1:core", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/api/resource", + "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/apis/meta/v1:meta", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/labels", + "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/util/validation", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/listers/core/v1:core", "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/controller-runtime/pkg/client", ], @@ -44,6 +47,7 @@ go_test( name = "types_test", srcs = [ "common_labels_test.go", + "control_plane_test.go", "event_annotations_test.go", "gpu_test.go", "miniservice_types_test.go", diff --git a/src/compute-plane-services/nvca/pkg/types/common_labels.go b/src/compute-plane-services/nvca/pkg/types/common_labels.go index 393783ac5..4799cee3c 100644 --- a/src/compute-plane-services/nvca/pkg/types/common_labels.go +++ b/src/compute-plane-services/nvca/pkg/types/common_labels.go @@ -120,6 +120,9 @@ func GetLabelsForRequest(req *nvcav2beta1.ICMSRequest, fff featureflag.Fetcher) MessageBatchIDKey: req.Spec.MessageBatchID, GPUNameKey: gpuName, } + if controlPlaneID := req.Labels[ControlPlaneIDLabel]; controlPlaneID != "" { + labelsForReq[ControlPlaneIDLabel] = controlPlaneID + } switch req.Spec.Action { case common.FunctionCreationAction: functionID := req.Spec.FunctionDetails.FunctionID diff --git a/src/compute-plane-services/nvca/pkg/types/control_plane.go b/src/compute-plane-services/nvca/pkg/types/control_plane.go new file mode 100644 index 000000000..6b1200f98 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/types/control_plane.go @@ -0,0 +1,78 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package types + +import ( + "crypto/sha256" + "fmt" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" +) + +const ( + // ControlPlaneIDLabel identifies resources owned by one control plane when + // multiple NVCF control planes share a Kubernetes cluster. + ControlPlaneIDLabel = "nvcf.nvidia.com/control-plane-id" + // MaxControlPlaneIDLength matches the supported self-managed stack prefix. + MaxControlPlaneIDLength = 20 +) + +// ValidateControlPlaneID validates the optional stable control-plane identity. +// Empty is valid and selects the legacy single-control-plane behavior. +func ValidateControlPlaneID(id string) error { + if id == "" { + return nil + } + if id == "default" { + return fmt.Errorf("control plane ID %q is reserved for legacy mode", id) + } + if len(id) > MaxControlPlaneIDLength { + return fmt.Errorf("control plane ID must be at most %d characters", MaxControlPlaneIDLength) + } + if errs := validation.IsDNS1123Label(id); len(errs) != 0 { + return fmt.Errorf("control plane ID must be a DNS-1123 label: %s", strings.Join(errs, "; ")) + } + return nil +} + +// ControlPlaneResourceName returns the legacy name for an empty ID and a +// deterministic, DNS-safe prefixed name in named mode. +func ControlPlaneResourceName(id, legacyName string) string { + if id == "" { + return legacyName + } + name := id + "-" + legacyName + if len(name) <= validation.DNS1123LabelMaxLength { + return name + } + hash := fmt.Sprintf("%x", sha256.Sum256([]byte(name)))[:8] + prefixLen := validation.DNS1123LabelMaxLength - len(hash) - 1 + return strings.TrimRight(name[:prefixLen], "-") + "-" + hash +} + +// IsOwnedByControlPlane returns whether an object belongs to the supplied +// identity. Legacy mode deliberately accepts unlabelled objects. +func IsOwnedByControlPlane(obj metav1.Object, id string) bool { + if id == "" { + return true + } + return obj != nil && obj.GetLabels()[ControlPlaneIDLabel] == id +} + +// AddControlPlaneLabel stamps the stable identity without changing legacy +// labels when the ID is empty. +func AddControlPlaneLabel(labels map[string]string, id string) map[string]string { + if id == "" { + return labels + } + if labels == nil { + labels = map[string]string{} + } + labels[ControlPlaneIDLabel] = id + return labels +} diff --git a/src/compute-plane-services/nvca/pkg/types/control_plane_test.go b/src/compute-plane-services/nvca/pkg/types/control_plane_test.go new file mode 100644 index 000000000..2acb99255 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/types/control_plane_test.go @@ -0,0 +1,71 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +package types + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" +) + +func TestValidateControlPlaneID(t *testing.T) { + tests := []struct { + name string + id string + wantErr bool + }{ + {name: "legacy empty", id: ""}, + {name: "named", id: "plane-a"}, + {name: "reserved default", id: "default", wantErr: true}, + {name: "uppercase", id: "Plane-A", wantErr: true}, + {name: "too long", id: "control-plane-id-over-twenty", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateControlPlaneID(tt.id) + if tt.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + }) + } +} + +func TestGetLabelsForRequestPreservesControlPlaneIdentity(t *testing.T) { + req := &nvcav2beta1.ICMSRequest{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{ + ControlPlaneIDLabel: "plane-a", + }}} + assert.Equal(t, "plane-a", GetLabelsForRequest(req, nil)[ControlPlaneIDLabel]) +} + +func TestControlPlaneResourceNames(t *testing.T) { + assert.Equal(t, "nvca-system", ControlPlaneResourceName("", "nvca-system")) + assert.Equal(t, "plane-a-nvca-system", ControlPlaneResourceName("plane-a", "nvca-system")) + assert.Equal(t, "plane-a-sr-request", ControlPlaneResourceName("plane-a", "sr-request")) + + long := ControlPlaneResourceName("plane-a", "sr-abcdefghijklmnopqrstuvwxyz-abcdefghijklmnopqrstuvwxyz-123") + require.LessOrEqual(t, len(long), 63) + assert.Equal(t, long, ControlPlaneResourceName("plane-a", "sr-abcdefghijklmnopqrstuvwxyz-abcdefghijklmnopqrstuvwxyz-123")) + assert.NotEqual(t, long, ControlPlaneResourceName("plane-b", "sr-abcdefghijklmnopqrstuvwxyz-abcdefghijklmnopqrstuvwxyz-123")) +} + +func TestIsOwnedByControlPlane(t *testing.T) { + owned := &metav1.PartialObjectMetadata{ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ControlPlaneIDLabel: "plane-a"}, + }} + unlabelled := &metav1.PartialObjectMetadata{} + + assert.True(t, IsOwnedByControlPlane(owned, "plane-a")) + assert.False(t, IsOwnedByControlPlane(owned, "plane-b")) + assert.False(t, IsOwnedByControlPlane(unlabelled, "plane-a")) + assert.True(t, IsOwnedByControlPlane(unlabelled, ""), "legacy mode preserves existing behavior") +} diff --git a/src/control-plane-services/admin-token-issuer-proxy/internal/platform/vault/client.go b/src/control-plane-services/admin-token-issuer-proxy/internal/platform/vault/client.go index 241cffb3b..b4de79d43 100644 --- a/src/control-plane-services/admin-token-issuer-proxy/internal/platform/vault/client.go +++ b/src/control-plane-services/admin-token-issuer-proxy/internal/platform/vault/client.go @@ -56,7 +56,19 @@ func NewVaultClient(addr string) (VaultSigner, error) { // SetToken sets the Vault token for authentication func (v *VaultClient) SetToken(token string) { - v.client.SetToken(token) + // The monorepo Bazel graph resolves vault/api to the ESS Agent fork. That + // fork sends the implicit client token as X-ESS-Token outside ESS local + // development. Keep the token out of the client's implicit credential slot + // and replace the standard Vault header explicitly so Go-module and + // Bazel/OCI builds send exactly one credential header. Replacing instead of + // appending also prevents stale credentials from surviving token rotation. + v.client.ClearToken() + headers := v.client.Headers() + if headers == nil { + headers = make(map[string][]string) + } + headers.Set("X-Vault-Token", token) + v.client.SetHeaders(headers) } // SignToken calls the Vault sign endpoint to mint a JWT diff --git a/src/control-plane-services/admin-token-issuer-proxy/internal/platform/vault/client_test.go b/src/control-plane-services/admin-token-issuer-proxy/internal/platform/vault/client_test.go index 02d72727a..4bd80d50f 100644 --- a/src/control-plane-services/admin-token-issuer-proxy/internal/platform/vault/client_test.go +++ b/src/control-plane-services/admin-token-issuer-proxy/internal/platform/vault/client_test.go @@ -97,8 +97,14 @@ func TestVaultClientHelpers(t *testing.T) { } }) - t.Run("SignToken wraps Vault errors with the resolved path", func(t *testing.T) { + t.Run("SignToken sends only the latest token in the standard Vault header", func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got, want := r.Header.Values("X-Vault-Token"), []string{"latest-token"}; !equalStrings(got, want) { + t.Errorf("X-Vault-Token headers = %q, want %q", got, want) + } + if got := r.Header.Values("X-ESS-Token"); len(got) != 0 { + t.Errorf("X-ESS-Token headers = %q, want none", got) + } http.Error(w, "signing failed", http.StatusInternalServerError) })) defer server.Close() @@ -107,7 +113,8 @@ func TestVaultClientHelpers(t *testing.T) { if err != nil { t.Fatalf("NewVaultClient() returned an unexpected error: %v", err) } - client.SetToken("test-token") + client.SetToken("old-token") + client.SetToken("latest-token") _, err = client.SignToken(t.Context(), "services/example/jwt/sign", "admin-issuer-proxy") if err == nil { @@ -163,3 +170,15 @@ func TestVaultClientHelpers(t *testing.T) { } }) } + +func equalStrings(got, want []string) bool { + if len(got) != len(want) { + return false + } + for i := range got { + if got[i] != want[i] { + return false + } + } + return true +} diff --git a/src/control-plane-services/admin-token-issuer-proxy/internal/servicecache/cache.go b/src/control-plane-services/admin-token-issuer-proxy/internal/servicecache/cache.go index 0f954cb93..f9b7d22f2 100644 --- a/src/control-plane-services/admin-token-issuer-proxy/internal/servicecache/cache.go +++ b/src/control-plane-services/admin-token-issuer-proxy/internal/servicecache/cache.go @@ -34,6 +34,7 @@ import ( ) const metadataErrorBodyLimit = 64 << 10 +const adminIssuerServiceName = "nvcf-api" // RetryPolicy controls the exponential backoff used while api-keys is starting. type RetryPolicy struct { @@ -110,13 +111,25 @@ func (c *Cache) FetchContext(ctx context.Context) error { if len(servicesResp.Services) == 0 { return fmt.Errorf("no services found in response") } - if servicesResp.Services[0].ServiceID == "" { + var serviceInfo *models.ServiceInfo + for i := range servicesResp.Services { + if servicesResp.Services[i].ServiceName == adminIssuerServiceName { + serviceInfo = &servicesResp.Services[i] + break + } + } + if serviceInfo == nil { + return fmt.Errorf("service metadata response does not contain %s", adminIssuerServiceName) + } + if serviceInfo.ServiceID == "" { return fmt.Errorf("service metadata is missing service_id") } - // Cache the first service (typically nvcf-api) + // Admin tokens are issued for the NVCF API. API Keys does not guarantee + // response order, so selecting the first service can mint a token for an + // unrelated service such as NVCT. c.mu.Lock() - c.serviceInfo = &servicesResp.Services[0] + c.serviceInfo = serviceInfo c.mu.Unlock() return nil diff --git a/src/control-plane-services/admin-token-issuer-proxy/internal/servicecache/cache_test.go b/src/control-plane-services/admin-token-issuer-proxy/internal/servicecache/cache_test.go index ff1ae637b..aa17bccc6 100644 --- a/src/control-plane-services/admin-token-issuer-proxy/internal/servicecache/cache_test.go +++ b/src/control-plane-services/admin-token-issuer-proxy/internal/servicecache/cache_test.go @@ -47,7 +47,7 @@ func TestCache_Fetch(t *testing.T) { Services: []models.ServiceInfo{ { ServiceID: "test-service-id", - ServiceName: "test-service", + ServiceName: "nvcf-api", AudienceServiceIDs: []string{"test-service-id"}, }, }, @@ -99,12 +99,23 @@ func TestCache_Fetch(t *testing.T) { serverHandler: func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) _ = json.NewEncoder(w).Encode(models.ServicesResponse{Services: []models.ServiceInfo{{ - ServiceName: "test-service", + ServiceName: "nvcf-api", }}}) }, expectError: true, errorContains: "service metadata is missing service_id", }, + { + name: "response missing nvcf api service", + serverHandler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(models.ServicesResponse{Services: []models.ServiceInfo{{ + ServiceName: "test-service", + }}}) + }, + expectError: true, + errorContains: "nvcf-api", + }, } for _, tc := range testCases { @@ -140,6 +151,31 @@ func TestCache_Fetch(t *testing.T) { } } +func TestCacheFetchSelectsNVCFAPIRegardlessOfResponseOrder(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(models.ServicesResponse{Services: []models.ServiceInfo{ + {ServiceName: "nvct-api", ServiceID: "tasks-service-id"}, + {ServiceName: "nvcf-api", ServiceID: "functions-service-id"}, + {ServiceName: "event-ledger", ServiceID: "ledger-service-id"}, + }}) + })) + defer server.Close() + + cache := New(server.URL) + if err := cache.Fetch(); err != nil { + t.Fatalf("Fetch() returned an unexpected error: %v", err) + } + if cache.Get() == nil { + t.Fatal("Fetch() did not cache service metadata") + } + if got := cache.Get().ServiceName; got != "nvcf-api" { + t.Fatalf("cached service name = %q, want nvcf-api", got) + } + if got := cache.Get().ServiceID; got != "functions-service-id" { + t.Fatalf("cached service ID = %q, want functions-service-id", got) + } +} + func TestCacheFetchWithRetryToleratesColdStart(t *testing.T) { var requests atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -150,7 +186,7 @@ func TestCacheFetchWithRetryToleratesColdStart(t *testing.T) { _ = json.NewEncoder(w).Encode(models.ServicesResponse{Services: []models.ServiceInfo{{ ServiceID: "test-service-id", - ServiceName: "test-service", + ServiceName: "nvcf-api", }}}) })) defer server.Close() @@ -184,7 +220,7 @@ func TestCacheFetchWithRetryToleratesConnectionRefused(t *testing.T) { Header: make(http.Header), Body: io.NopCloser(newJSONReader(t, models.ServicesResponse{Services: []models.ServiceInfo{{ ServiceID: "test-service-id", - ServiceName: "test-service", + ServiceName: "nvcf-api", }}})), }, nil }) @@ -218,7 +254,7 @@ func TestCacheFetchWithRetryToleratesDNSNotFound(t *testing.T) { Header: make(http.Header), Body: io.NopCloser(newJSONReader(t, models.ServicesResponse{Services: []models.ServiceInfo{{ ServiceID: "test-service-id", - ServiceName: "test-service", + ServiceName: "nvcf-api", }}})), }, nil }) diff --git a/tools/ncp-local-cluster/AGENTS.md b/tools/ncp-local-cluster/AGENTS.md index b3ff0577a..049ead398 100644 --- a/tools/ncp-local-cluster/AGENTS.md +++ b/tools/ncp-local-cluster/AGENTS.md @@ -19,6 +19,7 @@ make print-compute-clusters make test-cluster-lifecycle-make make test-multicluster-make make test-validate-gateway-route +make test-isolated-control-plane-gateways ``` Cluster lifecycle targets require local tools such as `k3d`, `kubectl`, `helm`, and Docker. diff --git a/tools/ncp-local-cluster/Makefile b/tools/ncp-local-cluster/Makefile index 50cb75132..c3f174cf9 100644 --- a/tools/ncp-local-cluster/Makefile +++ b/tools/ncp-local-cluster/Makefile @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -.PHONY: help clean build test test-coverage-html test-manual start stop destroy destroy-all-ncp-local status ensure-cluster ensure-context ensure-docker-config validate-compute-clusters print-compute-clusters start-control-plane deploy-control-plane-addons deploy-control-plane-endpoints build-and-deploy-control-plane-cluster destroy-control-plane start-compute-plane deploy-compute-plane-addons configure-compute-control-plane-dns deploy-compute-control-plane-endpoints build-and-deploy-compute-plane-cluster destroy-compute-plane build-and-deploy-multicluster destroy-multicluster test-cluster-lifecycle-make test-destroy-all-ncp-local test-multicluster-make test-validate-gateway-route setup-gateway-api setup-metallb check-gateway-api deploy-sample deploy-nginx wait-for-nginx wait-for-gateway validate-gateway cleanup-nginx wait-for-deployment validate-deployment cleanup-sample build-and-deploy-cluster build-csi-smb deploy-csi-smb wait-for-csi-smb build-fake-gpu-operator deploy-fake-gpu-operator wait-for-fake-gpu-operator deploy-prometheus-crds uninstall-prometheus-crds deploy-kube-state-metrics wait-for-kube-state-metrics uninstall-kube-state-metrics build-credential-provider-multiarch +.PHONY: help clean build test test-coverage-html test-manual start stop destroy destroy-all-ncp-local status ensure-cluster ensure-context ensure-docker-config validate-compute-clusters print-compute-clusters start-control-plane deploy-control-plane-addons deploy-control-plane-endpoints build-and-deploy-control-plane-cluster destroy-control-plane start-compute-plane deploy-compute-plane-addons configure-compute-control-plane-dns deploy-compute-control-plane-endpoints build-and-deploy-compute-plane-cluster destroy-compute-plane build-and-deploy-multicluster destroy-multicluster test-cluster-lifecycle-make test-destroy-all-ncp-local test-multicluster-make test-validate-gateway-route test-isolated-control-plane-gateways setup-gateway-api setup-metallb check-gateway-api deploy-isolated-control-plane-gateways destroy-isolated-control-plane-gateways deploy-sample deploy-nginx wait-for-nginx wait-for-gateway validate-gateway cleanup-nginx wait-for-deployment validate-deployment cleanup-sample build-and-deploy-cluster build-csi-smb deploy-csi-smb wait-for-csi-smb build-fake-gpu-operator deploy-fake-gpu-operator wait-for-fake-gpu-operator deploy-prometheus-crds uninstall-prometheus-crds deploy-kube-state-metrics wait-for-kube-state-metrics uninstall-kube-state-metrics build-credential-provider-multiarch # === Cluster Configuration === CLUSTER_NAME := ncp-local @@ -48,6 +48,13 @@ COMPUTE_CLUSTERS ?= GATEWAY_HTTP_PORT ?= $(CONTROL_PLANE_HTTP_PORT) GATEWAY_ROUTE_TIMEOUT_SECONDS ?= 60 GATEWAY_ROUTE_RETRY_INTERVAL_SECONDS ?= 2 +CONTROL_PLANE_ID ?= +ISOLATED_SHARED_HTTP_PORT ?= +ISOLATED_GRPC_API_PORT ?= +ISOLATED_GRPC_WORKER_PORT ?= +ISOLATED_NATS_PORT ?= +KUBECTL_CONTEXT ?= +KUBECTL_CONTEXT_FLAG = $(if $(KUBECTL_CONTEXT),--context $(KUBECTL_CONTEXT)) # === Go Credential Provider Configuration === GO_PROVIDER_PATH := ./credential-provider-go @@ -315,6 +322,9 @@ test-destroy-all-ncp-local: ## Run dry Makefile tests for host-wide local-cluste test-validate-gateway-route: ## Run Gateway route retry tests @tests/test-validate-gateway-route.sh +test-isolated-control-plane-gateways: ## Run isolated control-plane Gateway render tests + @tests/test-isolated-control-plane-gateways.sh + # --- Load Balancer & Gateway API Setup --- setup-gateway-api: $(call require,kubectl,See https://kubernetes.io/docs/tasks/tools/) @@ -330,14 +340,14 @@ setup-metallb: check-gateway-api: @echo ">>> Checking Gateway API infrastructure..." @echo -n " Gateway API CRDs: " - @if kubectl get crd gateways.gateway.networking.k8s.io >/dev/null 2>&1; then \ + @if kubectl $(KUBECTL_CONTEXT_FLAG) get crd gateways.gateway.networking.k8s.io >/dev/null 2>&1; then \ echo "OK Installed"; \ else \ echo "ERROR Not found. Run 'make setup-gateway-api'"; \ exit 1; \ fi @echo -n " GatewayClass: " - @if kubectl get gatewayclass eg >/dev/null 2>&1; then \ + @if kubectl $(KUBECTL_CONTEXT_FLAG) get gatewayclass eg >/dev/null 2>&1; then \ echo "OK Available"; \ else \ echo "ERROR Not found. Run 'make setup-gateway-api'"; \ @@ -345,6 +355,24 @@ check-gateway-api: fi @echo ">>> Gateway API infrastructure is ready!" +deploy-isolated-control-plane-gateways: check-gateway-api ## Create three plane-scoped Gateways; requires CONTROL_PLANE_ID and four ISOLATED_*_PORT values + @CONTROL_PLANE_ID="$(CONTROL_PLANE_ID)" \ + SHARED_HTTP_PORT="$(ISOLATED_SHARED_HTTP_PORT)" \ + GRPC_API_PORT="$(ISOLATED_GRPC_API_PORT)" \ + GRPC_WORKER_PORT="$(ISOLATED_GRPC_WORKER_PORT)" \ + NATS_PORT="$(ISOLATED_NATS_PORT)" \ + KUBECTL_CONTEXT="$(KUBECTL_CONTEXT)" \ + bash ./scripts/configure-isolated-control-plane-gateways.sh apply + +destroy-isolated-control-plane-gateways: ## Remove only the Gateways and EnvoyProxy policies owned by CONTROL_PLANE_ID + @CONTROL_PLANE_ID="$(CONTROL_PLANE_ID)" \ + SHARED_HTTP_PORT="$(ISOLATED_SHARED_HTTP_PORT)" \ + GRPC_API_PORT="$(ISOLATED_GRPC_API_PORT)" \ + GRPC_WORKER_PORT="$(ISOLATED_GRPC_WORKER_PORT)" \ + NATS_PORT="$(ISOLATED_NATS_PORT)" \ + KUBECTL_CONTEXT="$(KUBECTL_CONTEXT)" \ + bash ./scripts/configure-isolated-control-plane-gateways.sh delete + # --- Application & Provider Deployment --- deploy-sample: # Depends on the cluster being up @echo "========== SAMPLE WORKLOAD DEPLOYMENT ==========" diff --git a/tools/ncp-local-cluster/README.md b/tools/ncp-local-cluster/README.md index baf13fa56..7508872f3 100644 --- a/tools/ncp-local-cluster/README.md +++ b/tools/ncp-local-cluster/README.md @@ -295,6 +295,30 @@ The cluster uses Envoy Gateway with hostname-based routing. Routes use the `.loc Use the [self-hosted gateway route manifests](../../deploy/helm/gateway-routes) as the source of truth for route names and subdomains. +For multiple isolated control planes in one local cluster, create a distinct +Gateway set and non-overlapping listener ports for each plane after +`setup-gateway-api`: + +```bash +make deploy-isolated-control-plane-gateways \ + CONTROL_PLANE_ID=plane-a \ + ISOLATED_SHARED_HTTP_PORT=18080 \ + ISOLATED_GRPC_API_PORT=19081 \ + ISOLATED_GRPC_WORKER_PORT=19086 \ + ISOLATED_NATS_PORT=14222 +``` + +This creates `-shared-gw`, `-grpc-gw`, and `-nats-gw` plus owned +EnvoyProxy policies in `envoy-gateway-system`. Each policy gives the generated +Deployment and Service a bounded name so k3s ServiceLB labels remain within the +63-character Kubernetes limit. The lifecycle rejects foreign ownership and listener-port collisions, +allows routes only from namespaces labeled +`nvcf.nvidia.com/control-plane-id=`, and waits for all three Gateways to +become `Programmed`. The self-managed stack's named-plane install prepares +those labels. Remove only that plane's owned Gateways with the same variables and +`make destroy-isolated-control-plane-gateways`; the command refuses to delete +an unowned or differently owned Gateway. + ### Troubleshooting Hostname Resolution If `.localhost` domains don't resolve automatically, add entries to `/etc/hosts`: diff --git a/tools/ncp-local-cluster/scripts/configure-isolated-control-plane-gateways.sh b/tools/ncp-local-cluster/scripts/configure-isolated-control-plane-gateways.sh new file mode 100755 index 000000000..188b10d5a --- /dev/null +++ b/tools/ncp-local-cluster/scripts/configure-isolated-control-plane-gateways.sh @@ -0,0 +1,318 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +mode="${1:-apply}" +control_plane_id="${CONTROL_PLANE_ID:-}" +gateway_namespace="${GATEWAY_NAMESPACE:-envoy-gateway-system}" +gateway_class="${GATEWAY_CLASS:-eg}" + +if [[ -z "$control_plane_id" || ${#control_plane_id} -gt 20 || + ! "$control_plane_id" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ || + "$control_plane_id" == default ]]; then + echo "CONTROL_PLANE_ID must be a DNS-1123 label of at most 20 characters and must not be default" >&2 + exit 1 +fi + +declare -a port_names=(SHARED_HTTP_PORT GRPC_API_PORT GRPC_WORKER_PORT NATS_PORT) +declare -a ports=() +for port_name in "${port_names[@]}"; do + port="${!port_name:-}" + if [[ ! "$port" =~ ^[0-9]+$ || "$port" -lt 1 || "$port" -gt 65535 ]]; then + echo "$port_name must be an integer from 1 through 65535" >&2 + exit 1 + fi + ports+=("$port") +done + +if [[ "$(printf '%s\n' "${ports[@]}" | sort -u | wc -l | tr -d ' ')" -ne 4 ]]; then + echo "Gateway listener ports must be distinct for control plane $control_plane_id" >&2 + exit 1 +fi + +render() { + cat <}" >&2 + return 1 + fi + done < <(printf '%s\n' "$gateways_json" | jq -r \ + '.items[] | [.metadata.name, (.metadata.labels["nvcf.nvidia.com/control-plane-id"] // "")] | @tsv') + + while IFS=$'\t' read -r name reference_group reference_kind reference_name; do + if ! is_desired_gateway "$name" && + [[ "$reference_group" == "gateway.envoyproxy.io" ]] && + [[ "$reference_kind" == "EnvoyProxy" ]] && + is_desired_proxy "$reference_name"; then + echo "Refusing to modify EnvoyProxy $gateway_namespace/$reference_name: referenced by foreign Gateway $gateway_namespace/$name" >&2 + return 1 + fi + done < <(printf '%s\n' "$gateways_json" | jq -r \ + '.items[] | [.metadata.name, (.spec.infrastructure.parametersRef.group // ""), (.spec.infrastructure.parametersRef.kind // ""), (.spec.infrastructure.parametersRef.name // "")] | @tsv') + + while IFS=$'\t' read -r name listener_port; do + if is_desired_gateway "$name"; then + continue + fi + for desired_port in "${ports[@]}"; do + if [[ "$listener_port" == "$desired_port" ]]; then + echo "Gateway listener port $desired_port is already used by $gateway_namespace/$name" >&2 + return 1 + fi + done + done < <(printf '%s\n' "$gateways_json" | jq -r \ + '.items[] as $gateway | $gateway.spec.listeners[]? | [$gateway.metadata.name, (.port | tostring)] | @tsv') +} + +validate_existing_envoy_proxies() { + local proxies_json="$1" + local name owner + + while IFS=$'\t' read -r name owner; do + if is_desired_proxy "$name" && [[ "$owner" != "$control_plane_id" ]]; then + echo "Refusing to modify EnvoyProxy $gateway_namespace/$name owned by control plane ${owner:-}" >&2 + return 1 + fi + done < <(printf '%s\n' "$proxies_json" | jq -r \ + '.items[] | [.metadata.name, (.metadata.labels["nvcf.nvidia.com/control-plane-id"] // "")] | @tsv') +} + +case "$mode" in + render) + render + ;; + apply) + command -v jq >/dev/null 2>&1 || { + echo "jq is required to validate existing Gateways" >&2 + exit 1 + } + gateways_json="$(get_gateways)" + proxies_json="$(get_envoy_proxies)" + validate_existing_gateways "$gateways_json" + validate_existing_envoy_proxies "$proxies_json" + render | kubectl "${kubectl_args[@]}" apply -f - + for gateway_name in "${gateway_names[@]}"; do + kubectl "${kubectl_args[@]}" wait --namespace "$gateway_namespace" \ + --for=condition=Programmed --timeout="${GATEWAY_READY_TIMEOUT:-120s}" \ + "gateway/$gateway_name" + done + ;; + delete) + command -v jq >/dev/null 2>&1 || { + echo "jq is required to validate existing Gateways" >&2 + exit 1 + } + gateways_json="$(get_gateways)" + proxies_json="$(get_envoy_proxies)" + validate_existing_gateways "$gateways_json" + validate_existing_envoy_proxies "$proxies_json" + for gateway_name in "${gateway_names[@]}"; do + if printf '%s\n' "$gateways_json" | jq -e --arg name "$gateway_name" \ + '.items[] | select(.metadata.name == $name)' >/dev/null; then + kubectl "${kubectl_args[@]}" delete gateway "$gateway_name" \ + --namespace "$gateway_namespace" --wait=true + fi + done + for proxy_name in "${proxy_names[@]}"; do + if printf '%s\n' "$proxies_json" | jq -e --arg name "$proxy_name" \ + '.items[] | select(.metadata.name == $name)' >/dev/null; then + kubectl "${kubectl_args[@]}" delete envoyproxy "$proxy_name" \ + --namespace "$gateway_namespace" --wait=true + fi + done + ;; + *) + echo "usage: $0 [render|apply|delete]" >&2 + exit 1 + ;; +esac diff --git a/tools/ncp-local-cluster/tests/test-isolated-control-plane-gateways.sh b/tools/ncp-local-cluster/tests/test-isolated-control-plane-gateways.sh new file mode 100755 index 000000000..36aa4b823 --- /dev/null +++ b/tools/ncp-local-cluster/tests/test-isolated-control-plane-gateways.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +script="$root_dir/scripts/configure-isolated-control-plane-gateways.sh" +tmpdir="$(mktemp -d)" +trap 'rm -rf "$tmpdir"' EXIT + +fake_bin="$tmpdir/bin" +mkdir -p "$fake_bin" +cat >"$fake_bin/kubectl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +printf '%s\n' "$*" >>"${KUBECTL_CALL_LOG:?}" +case " $* " in + *" get gateways.gateway.networking.k8s.io "*) + if [[ -n "${FAKE_GATEWAYS_JSON:-}" ]]; then + printf '%s\n' "$FAKE_GATEWAYS_JSON" + else + printf '%s\n' '{"items":[]}' + fi + ;; + *" get envoyproxies.gateway.envoyproxy.io "*) + if [[ -n "${FAKE_ENVOY_PROXIES_JSON:-}" ]]; then + printf '%s\n' "$FAKE_ENVOY_PROXIES_JSON" + else + printf '%s\n' '{"items":[]}' + fi + ;; + *" apply -f - "*) + cat >>"${KUBECTL_APPLY_LOG:?}" + ;; + *" wait "*) + ;; + *" delete gateway "*) + ;; + *" delete envoyproxy "*) + ;; + *) + echo "unexpected kubectl arguments: $*" >&2 + exit 1 + ;; +esac +EOF +chmod +x "$fake_bin/kubectl" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +render_plane() { + local id="$1" + local port_base="$2" + CONTROL_PLANE_ID="$id" \ + SHARED_HTTP_PORT="$((port_base + 1))" \ + GRPC_API_PORT="$((port_base + 2))" \ + GRPC_WORKER_PORT="$((port_base + 3))" \ + NATS_PORT="$((port_base + 4))" \ + "$script" render +} + +render_plane plane-a 18000 >"$tmpdir/a.yaml" +render_plane plane-b 19000 >"$tmpdir/b.yaml" + +for plane in a b; do + id="plane-$plane" + manifest="$tmpdir/$plane.yaml" + for gateway in shared-gw grpc-gw nats-gw; do + grep -Fq "name: ${id}-${gateway}" "$manifest" || + fail "$id render missing ${id}-${gateway}" + done + grep -Fq "nvcf.nvidia.com/control-plane-id: ${id}" "$manifest" || + fail "$id render missing ownership label" + ruby -ryaml -e ' + id = ARGV[1] + docs = YAML.load_stream(File.read(ARGV[0])).compact + gateways = docs.select { |doc| doc["kind"] == "Gateway" } + proxies = docs.select { |doc| doc["kind"] == "EnvoyProxy" } + abort "expected three EnvoyProxy resources" unless proxies.length == 3 + proxies.each do |proxy| + service_name = proxy.dig("spec", "provider", "kubernetes", "envoyService", "name") + abort "Envoy Service name is not explicit" if service_name.to_s.empty? + abort "Envoy Service name exceeds k3s ServiceLB-safe length" if service_name.length > 47 + owner = proxy.dig("metadata", "labels", "nvcf.nvidia.com/control-plane-id") + abort "EnvoyProxy owner #{owner.inspect} does not match #{id}" unless owner == id + end + gateways.each do |gateway| + ref = gateway.dig("spec", "infrastructure", "parametersRef") || {} + expected = gateway.dig("metadata", "name").sub(/-gw$/, "-proxy") + abort "Gateway does not use EnvoyProxy #{expected}" unless + ref["group"] == "gateway.envoyproxy.io" && ref["kind"] == "EnvoyProxy" && ref["name"] == expected + end + listeners = gateways.flat_map { |gateway| gateway.dig("spec", "listeners") || [] } + abort "no Gateway listeners rendered" if listeners.empty? + listeners.each do |listener| + namespaces = listener.dig("allowedRoutes", "namespaces") || {} + abort "listener does not use namespace Selector" unless namespaces["from"] == "Selector" + actual = namespaces.dig("selector", "matchLabels", "nvcf.nvidia.com/control-plane-id") + abort "listener selector #{actual.inspect} does not select #{id}" unless actual == id + end + ' "$manifest" "$id" || fail "$id render does not isolate Envoy resources and routes" +done + +ruby -ryaml -e 'YAML.load_stream(File.read(ARGV[0])).compact.each { |doc| puts doc.dig("metadata", "name") if doc["kind"] == "Gateway" }' \ + "$tmpdir/a.yaml" | sort >"$tmpdir/a-names" +ruby -ryaml -e 'YAML.load_stream(File.read(ARGV[0])).compact.each { |doc| puts doc.dig("metadata", "name") if doc["kind"] == "Gateway" }' \ + "$tmpdir/b.yaml" | sort >"$tmpdir/b-names" +if comm -12 "$tmpdir/a-names" "$tmpdir/b-names" | grep -q .; then + fail "plane A and B Gateway names collide" +fi + +[[ "$(ruby -ryaml -e 'puts YAML.load_stream(File.read(ARGV[0])).compact.find { |doc| doc.dig("metadata", "name") == "plane-a-shared-gw" }.dig("spec", "listeners", 0, "port")' "$tmpdir/a.yaml")" == 18001 ]] || + fail "plane A shared HTTP port mismatch" +[[ "$(ruby -ryaml -e 'puts YAML.load_stream(File.read(ARGV[0])).compact.find { |doc| doc.dig("metadata", "name") == "plane-b-nats-gw" }.dig("spec", "listeners", 0, "port")' "$tmpdir/b.yaml")" == 19004 ]] || + fail "plane B NATS port mismatch" + +if CONTROL_PLANE_ID=INVALID_ID SHARED_HTTP_PORT=18001 GRPC_API_PORT=18002 \ + GRPC_WORKER_PORT=18003 NATS_PORT=18004 "$script" render >/dev/null 2>&1; then + fail "invalid control-plane ID was accepted" +fi +if CONTROL_PLANE_ID=plane-a SHARED_HTTP_PORT=18001 GRPC_API_PORT=18002 \ + GRPC_WORKER_PORT=18003 "$script" render >/dev/null 2>&1; then + fail "missing NATS port was accepted" +fi + +apply_plane() { + PATH="$fake_bin:$PATH" \ + KUBECTL_CALL_LOG="$tmpdir/kubectl-calls" \ + KUBECTL_APPLY_LOG="$tmpdir/kubectl-apply" \ + CONTROL_PLANE_ID=plane-a \ + SHARED_HTTP_PORT=18001 \ + GRPC_API_PORT=18002 \ + GRPC_WORKER_PORT=18003 \ + NATS_PORT=18004 \ + "$script" apply +} + +: >"$tmpdir/kubectl-calls" +: >"$tmpdir/kubectl-apply" +FAKE_GATEWAYS_JSON='{"items":[]}' apply_plane +[[ "$(grep -c '^wait ' "$tmpdir/kubectl-calls")" == 3 ]] || + fail "apply did not wait for all three Gateways to become Programmed" +grep -Fq 'get envoyproxies.gateway.envoyproxy.io' "$tmpdir/kubectl-calls" || + fail "apply did not preflight existing EnvoyProxy ownership" + +foreign_owner='{"items":[{"metadata":{"name":"plane-a-shared-gw","labels":{"nvcf.nvidia.com/control-plane-id":"plane-b"}},"spec":{"listeners":[{"port":19001}]}}]}' +if FAKE_GATEWAYS_JSON="$foreign_owner" apply_plane >/dev/null 2>&1; then + fail "apply accepted a desired Gateway owned by another control plane" +fi + +foreign_proxy_owner='{"items":[{"metadata":{"name":"plane-a-shared-proxy","labels":{"nvcf.nvidia.com/control-plane-id":"plane-b"}}}]}' +if FAKE_GATEWAYS_JSON='{"items":[]}' FAKE_ENVOY_PROXIES_JSON="$foreign_proxy_owner" \ + apply_plane >/dev/null 2>&1; then + fail "apply accepted a desired EnvoyProxy owned by another control plane" +fi + +colliding_port='{"items":[{"metadata":{"name":"plane-b-shared-gw","labels":{"nvcf.nvidia.com/control-plane-id":"plane-b"}},"spec":{"listeners":[{"port":18001}]}}]}' +if FAKE_GATEWAYS_JSON="$colliding_port" apply_plane >/dev/null 2>&1; then + fail "apply accepted a listener port already used by another control plane" +fi + +foreign_proxy_reference='{"items":[{"metadata":{"name":"plane-b-shared-gw","labels":{"nvcf.nvidia.com/control-plane-id":"plane-b"}},"spec":{"infrastructure":{"parametersRef":{"group":"gateway.envoyproxy.io","kind":"EnvoyProxy","name":"plane-a-shared-proxy"}},"listeners":[{"port":19001}]}}]}' +: >"$tmpdir/kubectl-calls" +: >"$tmpdir/kubectl-apply" +if FAKE_GATEWAYS_JSON="$foreign_proxy_reference" apply_plane >/dev/null 2>&1; then + fail "apply accepted a foreign Gateway coupled to this plane's EnvoyProxy" +fi +if grep -Eq '(^| )apply -f -($| )|(^| )delete (gateway|envoyproxy)($| )' "$tmpdir/kubectl-calls"; then + fail "apply mutated resources after finding a foreign EnvoyProxy reference" +fi + +: >"$tmpdir/kubectl-calls" +if FAKE_GATEWAYS_JSON="$foreign_proxy_reference" \ + PATH="$fake_bin:$PATH" \ + KUBECTL_CALL_LOG="$tmpdir/kubectl-calls" \ + KUBECTL_APPLY_LOG="$tmpdir/kubectl-apply" \ + CONTROL_PLANE_ID=plane-a \ + SHARED_HTTP_PORT=18001 \ + GRPC_API_PORT=18002 \ + GRPC_WORKER_PORT=18003 \ + NATS_PORT=18004 \ + "$script" delete >/dev/null 2>&1; then + fail "delete accepted a foreign Gateway coupled to this plane's EnvoyProxy" +fi +if grep -Eq '(^| )apply -f -($| )|(^| )delete (gateway|envoyproxy)($| )' "$tmpdir/kubectl-calls"; then + fail "delete mutated resources after finding a foreign EnvoyProxy reference" +fi + +: >"$tmpdir/kubectl-calls" +FAKE_GATEWAYS_JSON='{"items":[{"metadata":{"name":"plane-a-shared-gw","labels":{"nvcf.nvidia.com/control-plane-id":"plane-a"}},"spec":{"listeners":[{"port":18001}]}}]}' \ + FAKE_ENVOY_PROXIES_JSON='{"items":[{"metadata":{"name":"plane-a-shared-proxy","labels":{"nvcf.nvidia.com/control-plane-id":"plane-a"}}}]}' \ + PATH="$fake_bin:$PATH" \ + KUBECTL_CALL_LOG="$tmpdir/kubectl-calls" \ + KUBECTL_APPLY_LOG="$tmpdir/kubectl-apply" \ + CONTROL_PLANE_ID=plane-a \ + SHARED_HTTP_PORT=18001 \ + GRPC_API_PORT=18002 \ + GRPC_WORKER_PORT=18003 \ + NATS_PORT=18004 \ + "$script" delete +grep -Fq 'delete gateway plane-a-shared-gw' "$tmpdir/kubectl-calls" || + fail "delete did not remove the owned Gateway" +grep -Fq 'delete envoyproxy plane-a-shared-proxy' "$tmpdir/kubectl-calls" || + fail "delete did not remove the owned EnvoyProxy" + +if FAKE_GATEWAYS_JSON="$foreign_owner" \ + PATH="$fake_bin:$PATH" \ + KUBECTL_CALL_LOG="$tmpdir/kubectl-calls" \ + KUBECTL_APPLY_LOG="$tmpdir/kubectl-apply" \ + CONTROL_PLANE_ID=plane-a \ + SHARED_HTTP_PORT=18001 \ + GRPC_API_PORT=18002 \ + GRPC_WORKER_PORT=18003 \ + NATS_PORT=18004 \ + "$script" delete >/dev/null 2>&1; then + fail "delete accepted a desired Gateway owned by another control plane" +fi + +echo "Isolated control-plane Gateway lifecycle checks passed."