fix(self-hosted): resolve local installation failures

.github/workflows/build-test.yml

@@ -130,6 +130,11 @@ jobs:
         with:
           go-version-file: tools/go-toolchain/go.mod
 
+      - name: Run ncp-local-cluster Makefile tests
+        run: |
+          make -C tools/ncp-local-cluster test-cluster-lifecycle-make
+          make -C tools/ncp-local-cluster test-multicluster-make
+
       # The repo tooling modules carry tests that no workflow ran, so a pull
       # request could break them and still go green. tools/docs-version-sync
       # reached main with three failing tests that way.

docs/dev/local-development.md

@@ -108,8 +108,9 @@ the rationale.
 - [k3d](https://k3d.io/#installation) v5.x or later
 - `kubectl`
 - `helm` >= 3.12
-- `helmfile` >= 1.1.0, < 1.2.0 (Helmfile flows only)
-- `helm-diff` plugin (Helmfile flows only):
+- [Go](https://go.dev/doc/install) >= 1.24.0 (required to build `nvcf-cli`)
+- `helmfile` >= 1.1.0, < 1.2.0
+- `helm-diff` plugin:
   `helm plugin install https://github.com/databus23/helm-diff`
 - An NGC API key with access to the NVCF chart and image registry.
 - `nvcf-cli` built from this repo:

docs/user/local-development/multi-cluster-cli.md

@@ -45,6 +45,9 @@ Install the following tools:
 - [k3d](https://k3d.io/#installation) v5.x or later
 - `kubectl`
 - `helm` >= 3.12
+- [Go](https://go.dev/doc/install) >= 1.24.0 (required to build `nvcf-cli`)
+- `helmfile` >= 1.1.0, < 1.2.0
+- `helm-diff` plugin: `helm plugin install https://github.com/databus23/helm-diff`
 - An NGC API key from [ngc.nvidia.com](https://ngc.nvidia.com) with
   access to the NVCF chart and image registry.
 - The NGC organization and team slugs that hold the chart and image

docs/user/local-development/multi-cluster-helmfile.md

@@ -45,6 +45,7 @@ Install the following tools:
 - [k3d](https://k3d.io/#installation) v5.x or later
 - `kubectl`
 - `helm` >= 3.12
+- [Go](https://go.dev/doc/install) >= 1.24.0 (required to build `nvcf-cli`)
 - `helmfile` >= 1.1.0, < 1.2.0
 - `helm-diff` plugin: `helm plugin install https://github.com/databus23/helm-diff`
 - An NGC API key from [ngc.nvidia.com](https://ngc.nvidia.com) with

docs/user/local-development/single-cluster-cli.md

@@ -25,6 +25,9 @@ Install the following tools:
 - [k3d](https://k3d.io/#installation) v5.x or later
 - `kubectl`
 - `helm` >= 3.12
+- [Go](https://go.dev/doc/install) >= 1.24.0 (required to build `nvcf-cli`)
+- `helmfile` >= 1.1.0, < 1.2.0
+- `helm-diff` plugin: `helm plugin install https://github.com/databus23/helm-diff`
 - An NGC API key from [ngc.nvidia.com](https://ngc.nvidia.com) with
   access to the NVCF chart and image registry.
 - The NGC organization and team slugs that hold the chart and image

docs/user/local-development/single-cluster-helmfile.md

@@ -25,6 +25,7 @@ Install the following tools:
 - [k3d](https://k3d.io/#installation) v5.x or later
 - `kubectl`
 - `helm` >= 3.12
+- [Go](https://go.dev/doc/install) >= 1.24.0 (required to build `nvcf-cli`)
 - `helmfile` >= 1.1.0, < 1.2.0
 - `helm-diff` plugin: `helm plugin install https://github.com/databus23/helm-diff`
 - An NGC API key from [ngc.nvidia.com](https://ngc.nvidia.com) with

src/clis/nvcf-cli/cmd/self_hosted_compute_plane_test.go

@@ -923,12 +923,14 @@ func installFakeComputePlaneHelmfile(t *testing.T) string {
 	t.Helper()
 	fakeBin := filepath.Join(t.TempDir(), "helmfile")
 	body := `#!/bin/sh
-last=
+verb=
 for arg in "$@"; do
   printf 'arg=%s\n' "$arg"
-  last="$arg"
+  case "$arg" in
+    apply|template) verb="$arg" ;;
+  esac
 done
-printf 'verb=%s\n' "$last"
+printf 'verb=%s\n' "$verb"
 printf 'env:CLUSTER_NAME=%s\n' "$CLUSTER_NAME"
 printf 'env:NCA_ID=%s\n' "$NCA_ID"
 printf 'env:OUTPUT_DIR=%s\n' "$OUTPUT_DIR"

src/clis/nvcf-cli/cmd/self_hosted_install.go

@@ -136,15 +136,21 @@ func runSelfHostedInstall(c *cobra.Command, _ []string) error {
 	}
 	fmt.Fprintf(c.ErrOrStderr(), ">>> Resolving stack: %s\n", stackDescriptor(resolved))
 
+	helmRuntimeMode, err := resolveSelfHostedHelmRuntimeMode(c.Context())
+	if err != nil {
+		return fmt.Errorf("resolving helm runtime: %w", err)
+	}
+
 	if installControlPlane {
 		if err := selfhosted.Render(selfhosted.RenderOptions{
-			StackPath:   resolved.Path,
-			Env:         selfHostedEnv,
-			Apply:       !selfHostedNoApply,
-			KubeContext: selfHostedControlPlaneContext, // M+9: empty in single-cluster mode
-			Stdout:      c.OutOrStdout(),
-			Stderr:      c.ErrOrStderr(),
-			Ctx:         c.Context(),
+			StackPath:       resolved.Path,
+			Env:             selfHostedEnv,
+			Apply:           !selfHostedNoApply,
+			HelmRuntimeMode: helmRuntimeMode,
+			KubeContext:     selfHostedControlPlaneContext, // M+9: empty in single-cluster mode
+			Stdout:          c.OutOrStdout(),
+			Stderr:          c.ErrOrStderr(),
+			Ctx:             c.Context(),
 		}); err != nil {
 			return err
 		}
@@ -238,13 +244,14 @@ func runSelfHostedInstall(c *cobra.Command, _ []string) error {
 	}
 
 	return selfhosted.Render(selfhosted.RenderOptions{
-		StackPath:   resolved.Path,
-		Env:         selfHostedEnv,
-		Apply:       !selfHostedNoApply,
-		KubeContext: selfHostedComputePlaneContext, // M+9: empty in single-cluster mode
-		Stdout:      c.OutOrStdout(),
-		Stderr:      c.ErrOrStderr(),
-		Ctx:         c.Context(),
+		StackPath:       resolved.Path,
+		Env:             selfHostedEnv,
+		Apply:           !selfHostedNoApply,
+		HelmRuntimeMode: helmRuntimeMode,
+		KubeContext:     selfHostedComputePlaneContext, // M+9: empty in single-cluster mode
+		Stdout:          c.OutOrStdout(),
+		Stderr:          c.ErrOrStderr(),
+		Ctx:             c.Context(),
 		ExtraEnv: []string{
 			"CLUSTER_NAME=" + installClusterName,
 			"CLUSTER_ID=" + resp.ClusterID,

src/clis/nvcf-cli/cmd/self_hosted_install_test.go

@@ -45,6 +45,10 @@ func resetInstallFlags(t *testing.T) {
 	selfHostedToken = ""
 	selfHostedControlPlaneContext = ""
 	selfHostedComputePlaneContext = ""
+	prevRuntimeResolver := resolveSelfHostedHelmRuntimeMode
+	resolveSelfHostedHelmRuntimeMode = func(context.Context) (selfhosted.HelmRuntimeMode, error) {
+		return selfhosted.HelmRuntimeHelm3Legacy, nil
+	}
 	prevFetchRootCA := fetchControlPlaneRootCAPEM
 	fetchControlPlaneRootCAPEM = func(context.Context, string) (string, error) {
 		return "", nil
@@ -60,6 +64,7 @@ func resetInstallFlags(t *testing.T) {
 		selfHostedToken = ""
 		selfHostedControlPlaneContext = ""
 		selfHostedComputePlaneContext = ""
+		resolveSelfHostedHelmRuntimeMode = prevRuntimeResolver
 		fetchControlPlaneRootCAPEM = prevFetchRootCA
 	})
 }
@@ -89,15 +94,15 @@ func TestSelfHostedInstall_ControlPlane_NoApply(t *testing.T) {
 	assert.Contains(t, stdout.String(), "kind: ConfigMap")
 }
 
-func TestSelfHostedInstall_ControlPlane_AppliesByDefault(t *testing.T) {
+func TestSelfHostedInstall_ControlPlane_Helm4AppliesStateFilesSequentially(t *testing.T) {
 	resetInstallFlags(t)
 	stackDir := t.TempDir()
 	require.NoError(t, os.MkdirAll(filepath.Join(stackDir, "helmfile.d"), 0o755))
 	require.NoError(t, os.WriteFile(filepath.Join(stackDir, "global.yaml.gotmpl"), []byte("# stub\n"), 0o644))
 
 	fakeBin := filepath.Join(t.TempDir(), "helmfile")
 	require.NoError(t, os.WriteFile(fakeBin,
-		[]byte("#!/bin/sh\nlast=\nfor arg in \"$@\"; do last=\"$arg\"; done\nprintf 'verb=%s\\n' \"$last\"\n"),
+		[]byte("#!/bin/sh\nprintf '%s\\n' \"$@\"\n"),
 		0o755))
 	t.Setenv("PATH", filepath.Dir(fakeBin)+":"+os.Getenv("PATH"))
 	t.Setenv("HOME", t.TempDir())
@@ -119,12 +124,17 @@ func TestSelfHostedInstall_ControlPlane_AppliesByDefault(t *testing.T) {
 	}
 	require.NoError(t, sm.Save())
 
+	prevRuntimeResolver := resolveSelfHostedHelmRuntimeMode
 	prevAuthProbe := authProbe
 	prevInit := runSelfHostedInit
 	t.Cleanup(func() {
+		resolveSelfHostedHelmRuntimeMode = prevRuntimeResolver
 		authProbe = prevAuthProbe
 		runSelfHostedInit = prevInit
 	})
+	resolveSelfHostedHelmRuntimeMode = func(context.Context) (selfhosted.HelmRuntimeMode, error) {
+		return selfhosted.HelmRuntimeHelm4Compat, nil
+	}
 	authProbe = func(context.Context, string) (*auth.Fingerprint, error) {
 		return &auth.Fingerprint{IssuerURL: "http://api.localhost:8080", JWKSKid: "kid", APIKeysEndpoint: "http://api-keys.localhost:8080"}, nil
 	}
@@ -141,7 +151,9 @@ func TestSelfHostedInstall_ControlPlane_AppliesByDefault(t *testing.T) {
 	})
 	require.NoError(t, rootCmd.Execute())
 
-	assert.Contains(t, stdout.String(), "verb=apply")
+	assert.Contains(t, stdout.String(), "--sequential-helmfiles")
+	assert.Contains(t, stdout.String(), "apply")
+	assert.Contains(t, stdout.String(), "--skip-diff-on-install")
 	assert.Equal(t, 1, initCalls)
 }
 
@@ -295,11 +307,11 @@ func TestSelfHostedInstall_ComputePlane_AppliesByDefault(t *testing.T) {
 
 	stackDir := t.TempDir()
 	require.NoError(t, os.MkdirAll(filepath.Join(stackDir, "helmfile.d"), 0o755))
-	// Fake helmfile echoes the verb passed as its last arg so the test can assert
-	// that 'apply' (not 'template') ran when --no-apply is omitted.
+	// Fake helmfile echoes every argument so the test can assert that apply runs
+	// with the first-install diff guard when --no-apply is omitted.
 	fakeBin := filepath.Join(t.TempDir(), "helmfile")
 	require.NoError(t, os.WriteFile(fakeBin,
-		[]byte("#!/bin/sh\nlast=\nfor arg in \"$@\"; do last=\"$arg\"; done\nprintf 'verb=%s\\n' \"$last\"\n"),
+		[]byte("#!/bin/sh\nprintf '%s\\n' \"$@\"\n"),
 		0o755))
 	t.Setenv("PATH", filepath.Dir(fakeBin)+":"+os.Getenv("PATH"))
 
@@ -314,7 +326,8 @@ func TestSelfHostedInstall_ComputePlane_AppliesByDefault(t *testing.T) {
 	require.NoError(t, rootCmd.Execute())
 
 	assert.Equal(t, 1, fakeCC.registerCalls)
-	assert.Contains(t, stdout.String(), "verb=apply")
+	assert.Contains(t, stdout.String(), "apply")
+	assert.Contains(t, stdout.String(), "--skip-diff-on-install")
 }
 
 func TestSelfHostedInstall_ComputePlane_LocalSplitWritesExternalControlPlaneEndpoints(t *testing.T) {

src/clis/nvcf-cli/internal/selfhosted/render.go

@@ -26,7 +26,8 @@ import (
 )
 
 // RenderOptions controls a single invocation of 'helmfile template' or
-// 'helmfile apply'. Stdout/Stderr writers are caller-supplied so the install
+// 'helmfile apply --skip-diff-on-install'. Stdout/Stderr writers are
+// caller-supplied so the install
 // command can split YAML output (stdout) from helmfile progress (stderr) per
 // spec §6.1.
 type RenderOptions struct {
@@ -48,8 +49,8 @@ type RenderOptions struct {
 	// Selector (optional) is the -l flag value for narrowing to a single
 	// release group (e.g. "component=control-plane").
 	Selector string
-	// Apply runs 'helmfile apply' when true; otherwise runs 'helmfile template'.
-	// Apply=true is reserved for the up orchestrator (M5).
+	// Apply runs 'helmfile apply --skip-diff-on-install' when true; otherwise
+	// runs 'helmfile template'.
 	Apply bool
 	// ExtraEnv holds additional environment variables in "KEY=VALUE" form that
 	// are appended to the helmfile subprocess environment after HELMFILE_ENV.
@@ -69,7 +70,7 @@ type RenderOptions struct {
 	Ctx    context.Context
 }
 
-// Render invokes 'helmfile {template,apply}' against the resolved stack tree,
+// Render invokes helmfile template or apply against the resolved stack tree,
 // plumbing HELMFILE_ENV through the subprocess environment. Stdout and Stderr
 // are written to the caller-supplied writers. A non-zero exit code from
 // helmfile is returned as a wrapped error containing the exit status.
@@ -94,6 +95,9 @@ func Render(opts RenderOptions) error {
 		args = append(args, "--kube-context="+opts.KubeContext)
 	}
 	args = append(args, verb)
+	if opts.Apply {
+		args = append(args, "--skip-diff-on-install")
+	}
 
 	ctx := opts.Ctx
 	if ctx == nil {

src/clis/nvcf-cli/internal/selfhosted/render_test.go

@@ -191,6 +191,30 @@ func TestRender_Helm4CompatApplyUsesLegacyTrackMode(t *testing.T) {
 	assert.Contains(t, out, "helm-legacy")
 }
 
+func TestRender_ApplySkipsDiffOnInstall(t *testing.T) {
+	dir := t.TempDir()
+	fake := filepath.Join(dir, "helmfile")
+	require.NoError(t, os.WriteFile(fake,
+		[]byte("#!/bin/sh\nprintf '%s\\n' \"$@\"\n"), 0o755))
+
+	stack := t.TempDir()
+	require.NoError(t, os.MkdirAll(filepath.Join(stack, "helmfile.d"), 0o755))
+
+	var stdout, stderr bytes.Buffer
+	err := Render(RenderOptions{
+		StackPath:   stack,
+		Env:         "local",
+		HelmfileBin: fake,
+		Apply:       true,
+		Stdout:      &stdout,
+		Stderr:      &stderr,
+	})
+	require.NoError(t, err)
+	args := strings.Split(strings.TrimSpace(stdout.String()), "\n")
+	require.GreaterOrEqual(t, len(args), 2)
+	assert.Equal(t, []string{"apply", "--skip-diff-on-install"}, args[len(args)-2:])
+}
+
 func TestRender_Helm4CompatSingleFileOmitsSequentialFlag(t *testing.T) {
 	dir := t.TempDir()
 	fake := filepath.Join(dir, "helmfile")

tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml

@@ -44,6 +44,10 @@ global:
       revalServiceURL: http://reval.nvcf.svc.cluster.local:8080
       natsURL: nats://nats.nats-system.svc.cluster.local:4222
 
+# Local BDD does not install the optional observability stack.
+observability:
+  profile: disabled
+
 agentConfig:
   mergeConfig: |
     cluster:

tests/bdd/fixtures/nvcf-compute-plane-local-bdd.yaml

@@ -38,6 +38,10 @@ global:
       revalServiceURL: http://reval.nvcf.svc.cluster.local:8080
       natsURL: nats://nats.nats-system.svc.cluster.local:4222
 
+# Local BDD does not install the optional observability stack.
+observability:
+  profile: disabled
+
 agentConfig:
   mergeConfig: |
     cluster:

tests/bdd/fixtures/self-managed-local-bdd-multi.yaml

@@ -38,6 +38,10 @@ global:
     # NVCF API advertises this value as LLM_REQUEST_ROUTER_ADDRESS.
     llmRequestRouterAddress: https://llm-request-router.nvcf.svc.cluster.local:50071
 
+# Local BDD does not install the optional observability stack.
+observability:
+  profile: disabled
+
 cassandra:
   replicaCount: 1
   resourcesPreset: "large"

tests/bdd/fixtures/self-managed-local-bdd.yaml

@@ -31,6 +31,10 @@ global:
     essServiceURL: http://ess-api.ess.svc.cluster.local:8080
     invocationServiceURL: http://invocation.nvcf.svc.cluster.local:8080
 
+# Local BDD does not install the optional observability stack.
+observability:
+  profile: disabled
+
 cassandra:
   replicaCount: 1
   resourcesPreset: "large"

tools/ncp-local-cluster/AGENTS.md

@@ -16,6 +16,7 @@ Run Makefile-only validation from `tools/ncp-local-cluster`:
 ```bash
 make validate-compute-clusters
 make print-compute-clusters
+make test-cluster-lifecycle-make
 make test-multicluster-make
 make test-validate-gateway-route
 ```

tools/ncp-local-cluster/Makefile

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-.PHONY: help clean build test test-coverage-html test-manual start stop destroy destroy-all-ncp-local status ensure-cluster ensure-context ensure-docker-config validate-compute-clusters print-compute-clusters start-control-plane deploy-control-plane-addons deploy-control-plane-endpoints build-and-deploy-control-plane-cluster destroy-control-plane start-compute-plane deploy-compute-plane-addons configure-compute-control-plane-dns deploy-compute-control-plane-endpoints build-and-deploy-compute-plane-cluster destroy-compute-plane build-and-deploy-multicluster destroy-multicluster test-destroy-all-ncp-local test-multicluster-make test-validate-gateway-route setup-gateway-api setup-metallb check-gateway-api deploy-sample deploy-nginx wait-for-nginx wait-for-gateway validate-gateway cleanup-nginx wait-for-deployment validate-deployment cleanup-sample build-and-deploy-cluster build-csi-smb deploy-csi-smb wait-for-csi-smb build-fake-gpu-operator deploy-fake-gpu-operator wait-for-fake-gpu-operator deploy-prometheus-crds uninstall-prometheus-crds deploy-kube-state-metrics wait-for-kube-state-metrics uninstall-kube-state-metrics build-credential-provider-multiarch
+.PHONY: help clean build test test-coverage-html test-manual start stop destroy destroy-all-ncp-local status ensure-cluster ensure-context ensure-docker-config validate-compute-clusters print-compute-clusters start-control-plane deploy-control-plane-addons deploy-control-plane-endpoints build-and-deploy-control-plane-cluster destroy-control-plane start-compute-plane deploy-compute-plane-addons configure-compute-control-plane-dns deploy-compute-control-plane-endpoints build-and-deploy-compute-plane-cluster destroy-compute-plane build-and-deploy-multicluster destroy-multicluster test-cluster-lifecycle-make test-destroy-all-ncp-local test-multicluster-make test-validate-gateway-route setup-gateway-api setup-metallb check-gateway-api deploy-sample deploy-nginx wait-for-nginx wait-for-gateway validate-gateway cleanup-nginx wait-for-deployment validate-deployment cleanup-sample build-and-deploy-cluster build-csi-smb deploy-csi-smb wait-for-csi-smb build-fake-gpu-operator deploy-fake-gpu-operator wait-for-fake-gpu-operator deploy-prometheus-crds uninstall-prometheus-crds deploy-kube-state-metrics wait-for-kube-state-metrics uninstall-kube-state-metrics build-credential-provider-multiarch
 
 # === Cluster Configuration ===
 CLUSTER_NAME := ncp-local
@@ -98,7 +98,7 @@ ensure-cluster: ## Create cluster if it doesn't exist, otherwise skip
 	$(call require,kubectl,See https://kubernetes.io/docs/tasks/tools/)
 	@if ! k3d cluster get $(CLUSTER_NAME) >/dev/null 2>&1; then \
 		echo "Creating k3d cluster $(CLUSTER_NAME) using config $(K3D_CONFIG_FILE)..."; \
-		K3D_CLUSTER_NAME="$(CLUSTER_NAME)" \
+		if ! K3D_CLUSTER_NAME="$(CLUSTER_NAME)" \
 			CONTROL_PLANE_HTTP_PORT="$(CONTROL_PLANE_HTTP_PORT)" \
 			CONTROL_PLANE_HTTPS_PORT="$(CONTROL_PLANE_HTTPS_PORT)" \
 			CONTROL_PLANE_GRPC_PORT="$(CONTROL_PLANE_GRPC_PORT)" \
@@ -107,7 +107,10 @@ ensure-cluster: ## Create cluster if it doesn't exist, otherwise skip
 			CONTROL_PLANE_LLM_GRPC_PORT="$(CONTROL_PLANE_LLM_GRPC_PORT)" \
 			CONTROL_PLANE_LLM_QUIC_PORT="$(CONTROL_PLANE_LLM_QUIC_PORT)" \
 			CONTROL_PLANE_NATS_PORT="$(CONTROL_PLANE_NATS_PORT)" \
-			k3d cluster create $(CLUSTER_NAME) --config $(K3D_CONFIG_FILE); \
+			k3d cluster create $(CLUSTER_NAME) --config $(K3D_CONFIG_FILE); then \
+			echo "ERROR Failed to create k3d cluster $(CLUSTER_NAME)." >&2; \
+			exit 1; \
+		fi; \
 		echo "OK Cluster $(CLUSTER_NAME) created."; \
 	else \
 		echo "OK Cluster $(CLUSTER_NAME) already exists. Skipping creation."; \
@@ -135,7 +138,10 @@ start: ensure-docker-config
 	@echo "========== START CLUSTER =========="
 	@$(MAKE) ensure-cluster
 	@echo "Ensuring cluster $(CLUSTER_NAME) is ready..."
-	@k3d cluster start $(CLUSTER_NAME) 2>/dev/null || echo "Cluster already running."
+	@if ! k3d cluster start $(CLUSTER_NAME); then \
+		echo "ERROR Failed to start k3d cluster $(CLUSTER_NAME)." >&2; \
+		exit 1; \
+	fi
 	@echo "OK Cluster $(CLUSTER_NAME) is running."
 	@$(MAKE) ensure-context
 
@@ -297,6 +303,9 @@ destroy-multicluster: validate-compute-clusters ## Destroy the default control-p
 		$(MAKE) CLUSTER_NAME="$$cluster" destroy || true; \
 	done
 
+test-cluster-lifecycle-make: ## Run dry Makefile tests for cluster lifecycle behavior
+	@tests/test-cluster-lifecycle-make.sh
+
 test-multicluster-make: ## Run dry Makefile tests for multi-cluster name derivation
 	@tests/test-multicluster-make.sh