From 2ef166f596a58a39fe3d81d7be60d40a4243b7e4 Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Sat, 29 Aug 2026 23:53:15 -0700 Subject: [PATCH 1/6] test(bdd): add shared Pylon observability DSL Keep TLS WatchStargates targets and Pylon metric expectations visible while sharing repeated command, polling, and output parsing mechanics. Refs #1358 Signed-off-by: Stephanie Baum --- tests/bdd/PLAN.md | 30 +++ tests/bdd/dsl/registration.go | 106 +++++++++++ tests/bdd/dsl/registration_test.go | 189 +++++++++++++++++++ tests/bdd/scripts/observe-watch-stargates.sh | 75 ++++++++ tests/bdd/scripts/wait-pylon-metrics.sh | 132 +++++++++++++ tests/bdd/steps/assertion_steps.go | 37 ++++ tests/bdd/steps/context.go | 1 + tests/bdd/steps/registration_steps.go | 105 +++++++++++ tests/bdd/steps/steps_test.go | 103 ++++++++++ 9 files changed, 778 insertions(+) create mode 100644 tests/bdd/dsl/registration.go create mode 100644 tests/bdd/dsl/registration_test.go create mode 100644 tests/bdd/scripts/observe-watch-stargates.sh create mode 100644 tests/bdd/scripts/wait-pylon-metrics.sh create mode 100644 tests/bdd/steps/registration_steps.go diff --git a/tests/bdd/PLAN.md b/tests/bdd/PLAN.md index e53c0aecf..8b940a83b 100644 --- a/tests/bdd/PLAN.md +++ b/tests/bdd/PLAN.md @@ -122,6 +122,16 @@ refactor in every consumer; that is a feature. | `When I run command with a terminal:` (docstring) | Same as the docstring form, but stdin is attached to a pseudo-terminal so the child sees a TTY on fd 0. For commands that gate interactive-only behavior on a TTY, such as `nvcf-cli self-hosted up` (its auth-gate mints the admin token only when stdin is a terminal). No input is written; stdout and stderr are captured separately as usual. | | `When I export command output to environment variable {string}` | Exports the previous command's trimmed stdout under the named env var. Fails the step unless the prior command exited 0 and produced non-empty stdout. Snapshotted by the env Ledger; restored at suite teardown. | +#### Registration observability command adapters + +These steps wrap repeated client, trust-material, polling, and output-format +mechanics while keeping every operator-selected target and expectation visible. +They preserve the real command output for subsequent assertions. + +| Step | Command | +|------|---------| +| `When I successfully observe WatchStargates at {string} with TLS authority {string} using CA secret {string} in namespace {string} and context {string} for {string} seconds` | Reads the named CA certificate from the explicit Kubernetes secret and context, runs the public `WatchStargates` gRPC method against the visible endpoint and TLS authority, and requires a streamed response before accepting the expected client deadline. | + #### Function lifecycle command adapters These steps hide the repeated executable, config prefix, fixed subcommand, shell @@ -156,8 +166,11 @@ original order. Repeated options and empty values are preserved. | Step | Notes | |------|-------| | `Then the command exit code should be {int}` | Last-run exit code. | +| `Then the command should fail` | Requires a non-zero last-run exit code. It does not accept a runner error that prevented command execution and never records the failed command in the successful-command cache. | | `Then the command output should contain {string}` | Substring match on combined stdout + stderr. | | `Then the command output should not contain {string}` | Negative substring match. | +| `Then the command output should contain all:` (table) | Requires a `text` header and one or more strings. Every interpolated string must appear in combined stdout + stderr. | +| `Then the command output should contain one of:` (table) | Requires a `text` header and one or more strings. At least one interpolated string must appear in combined stdout + stderr. | | `Then file {string} should exist` | | | `Then yaml file {string} key {string} should equal {string}` | Reads the YAML file, walks the dotted key path, compares to the value (with `${VAR}` expansion). | | `Then yaml file {string} key {string} should not be empty` | Same key resolution; passes if the resolved value is non-empty. Use for non-deterministic outputs (cluster IDs, identity sources) where exact-value assertions are wrong. | @@ -179,6 +192,7 @@ original order. Repeated options and empty values are preserved. | `Then deployment {string} in namespace {string} using context {string} should complete rollout within {string}` | Runs `kubectl rollout status` for the named deployment with the explicit namespace, context, and timeout. Failure messages name the deployment without printing command output. | | `Then NVCFBackend {string} in namespace {string} using context {string} should report agent status {string} within {string}` | Waits for the named backend's `status.agentStatus` to equal the visible value using the explicit namespace, context, and timeout. Failure messages name the backend without printing resource output. | | `Then these Gateway API routes should be accepted and resolved using context {string} within {string}:` (table) | Requires `kind`, `name`, `namespace`, and `parent` headers. Waits for every named route to report both `Accepted=True` and `ResolvedRefs=True` for the named Gateway parent using the explicit context and timeout. The route kind is passed through without an allowlist. Failures name the table row, route, namespace, parent, and unmet condition without printing resource output. | +| `Then a pod containing container {string} using context {string} should report Pylon metrics within {string}:` (table) | Requires `metric`, `comparison`, and `count` headers. Polls the Pylon metrics endpoint of a running pod containing the visible container name. Each metric row counts connected series with value `1`; `comparison` is `exactly` or `at least`, and the expected non-negative count remains visible. | #### YAML comparison semantics @@ -514,6 +528,22 @@ func SubstituteFile(path, placeholder, replacement string) error // exists in the array. Extra objects in the array are fine. func JSONContainsRows(raw string, rows []map[string]string) error +// WatchStargatesCommand builds a TLS WatchStargates observation with an +// explicit endpoint, authority, CA source, Kubernetes context, and duration. +func WatchStargatesCommand(endpoint, authority, caSecret, namespace, kubeContext, durationSeconds string) (string, error) + +// PylonMetricExpectation describes an expected count of connected metric +// series exposed by one Pylon sidecar. +type PylonMetricExpectation struct { + Metric string + Comparison string + Count int +} + +// PylonMetricsCommand builds a bounded Pylon metrics observation for a pod +// selected by container name in an explicit Kubernetes context. +func PylonMetricsCommand(containerName, kubeContext, timeout string, expectations []PylonMetricExpectation) (string, error) + // FilesDoNotContain recursively inspects regular files under root and // fails if any interpolated fixed string appears. func FilesDoNotContain(root string, needles []string) error diff --git a/tests/bdd/dsl/registration.go b/tests/bdd/dsl/registration.go new file mode 100644 index 000000000..4b546bab9 --- /dev/null +++ b/tests/bdd/dsl/registration.go @@ -0,0 +1,106 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dsl + +import ( + "fmt" + "regexp" + "strconv" + "strings" +) + +const ( + observeWatchStargatesScript = "tests/bdd/scripts/observe-watch-stargates.sh" + waitPylonMetricsScript = "tests/bdd/scripts/wait-pylon-metrics.sh" +) + +var prometheusMetricNameRE = regexp.MustCompile(`^[a-zA-Z_:][a-zA-Z0-9_:]*$`) + +// PylonMetricExpectation describes an expected count of connected metric +// series exposed by one Pylon sidecar. +type PylonMetricExpectation struct { + Metric string + Comparison string + Count int +} + +// WatchStargatesCommand builds a TLS WatchStargates observation with an +// explicit endpoint, authority, CA source, Kubernetes context, and duration. +func WatchStargatesCommand(endpoint, authority, caSecret, namespace, kubeContext, durationSeconds string) (string, error) { + values := []*string{&endpoint, &authority, &caSecret, &namespace, &kubeContext, &durationSeconds} + labels := []string{"endpoint", "TLS authority", "CA secret", "namespace", "kube context", "duration seconds"} + for index := range values { + *values[index] = strings.TrimSpace(Interpolate(*values[index])) + if *values[index] == "" { + return "", fmt.Errorf("%s is empty", labels[index]) + } + } + duration, err := strconv.Atoi(durationSeconds) + if err != nil || duration <= 0 { + return "", fmt.Errorf("duration seconds must be a positive integer") + } + + return BuildCommand( + "bash", + observeWatchStargatesScript, + endpoint, + authority, + caSecret, + namespace, + kubeContext, + durationSeconds, + ), nil +} + +// PylonMetricsCommand builds a bounded Pylon metrics observation for a pod +// selected by container name in an explicit Kubernetes context. +func PylonMetricsCommand(containerName, kubeContext, timeout string, expectations []PylonMetricExpectation) (string, error) { + containerName = strings.TrimSpace(Interpolate(containerName)) + kubeContext = strings.TrimSpace(Interpolate(kubeContext)) + timeout = strings.TrimSpace(Interpolate(timeout)) + if containerName == "" { + return "", fmt.Errorf("container name is empty") + } + if kubeContext == "" { + return "", fmt.Errorf("kube context is empty") + } + if timeout == "" { + return "", fmt.Errorf("timeout is empty") + } + if len(expectations) == 0 { + return "", fmt.Errorf("Pylon metric expectations are empty") + } + + args := []string{"bash", waitPylonMetricsScript, containerName, kubeContext, timeout} + for index, expectation := range expectations { + expectation.Metric = strings.TrimSpace(Interpolate(expectation.Metric)) + expectation.Comparison = strings.TrimSpace(Interpolate(expectation.Comparison)) + if !prometheusMetricNameRE.MatchString(expectation.Metric) { + return "", fmt.Errorf("metric row %d has invalid metric name %q", index+1, expectation.Metric) + } + if expectation.Comparison != "exactly" && expectation.Comparison != "at least" { + return "", fmt.Errorf("metric row %d comparison must be exactly or at least", index+1) + } + if expectation.Count < 0 { + return "", fmt.Errorf("metric row %d count must be non-negative", index+1) + } + args = append(args, expectation.Metric, expectation.Comparison, strconv.Itoa(expectation.Count)) + } + + return BuildCommand(args...), nil +} diff --git a/tests/bdd/dsl/registration_test.go b/tests/bdd/dsl/registration_test.go new file mode 100644 index 000000000..606765864 --- /dev/null +++ b/tests/bdd/dsl/registration_test.go @@ -0,0 +1,189 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dsl + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestWatchStargatesCommandKeepsObservationInputsExplicit(t *testing.T) { + t.Setenv("BDD_WATCH_CONTEXT", "k3d-ncp-local-cp") + got, err := WatchStargatesCommand( + "127.0.0.1:50071", + "llm-request-router.nvcf.svc.cluster.local", + "stargate-quic-tls", + "nvcf", + "${BDD_WATCH_CONTEXT}", + "3", + ) + if err != nil { + t.Fatalf("build WatchStargates command: %v", err) + } + want := "bash tests/bdd/scripts/observe-watch-stargates.sh 127.0.0.1:50071 llm-request-router.nvcf.svc.cluster.local stargate-quic-tls nvcf k3d-ncp-local-cp 3" + if got != want { + t.Fatalf("command = %q, want %q", got, want) + } +} + +func TestWatchStargatesCommandRejectsMissingOrInvalidInputs(t *testing.T) { + tests := []struct { + name string + endpoint string + authority string + duration string + }{ + {name: "empty endpoint", endpoint: "", authority: "router.nvcf.svc", duration: "3"}, + {name: "empty authority", endpoint: "127.0.0.1:50071", authority: "", duration: "3"}, + {name: "invalid duration", endpoint: "127.0.0.1:50071", authority: "router.nvcf.svc", duration: "3s"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := WatchStargatesCommand(test.endpoint, test.authority, "tls", "nvcf", "context", test.duration); err == nil { + t.Fatal("expected validation error") + } + }) + } +} + +func TestPylonMetricsCommandKeepsExpectationsExplicit(t *testing.T) { + got, err := PylonMetricsCommand("llm-worker", "k3d-ncp-local-compute-1", "10m", []PylonMetricExpectation{ + {Metric: "pylon_registration_stream_connected", Comparison: "exactly", Count: 5}, + {Metric: "pylon_reverse_tunnel_connected", Comparison: "at least", Count: 3}, + }) + if err != nil { + t.Fatalf("build Pylon metrics command: %v", err) + } + want := "bash tests/bdd/scripts/wait-pylon-metrics.sh llm-worker k3d-ncp-local-compute-1 10m pylon_registration_stream_connected exactly 5 pylon_reverse_tunnel_connected 'at least' 3" + if got != want { + t.Fatalf("command = %q, want %q", got, want) + } +} + +func TestPylonMetricsCommandRejectsInvalidExpectations(t *testing.T) { + tests := []struct { + name string + expectations []PylonMetricExpectation + }{ + {name: "empty", expectations: nil}, + {name: "invalid metric", expectations: []PylonMetricExpectation{{Metric: "metric name", Comparison: "exactly", Count: 1}}}, + {name: "invalid comparison", expectations: []PylonMetricExpectation{{Metric: "metric_name", Comparison: "more than", Count: 1}}}, + {name: "negative count", expectations: []PylonMetricExpectation{{Metric: "metric_name", Comparison: "exactly", Count: -1}}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := PylonMetricsCommand("llm-worker", "context", "10m", test.expectations); err == nil { + t.Fatal("expected validation error") + } + }) + } +} + +func TestObserveWatchStargatesScriptAcceptsSnapshotThenDeadline(t *testing.T) { + fakeBin := t.TempDir() + writeExecutable(t, filepath.Join(fakeBin, "kubectl"), "#!/bin/sh\nprintf 'dGVzdC1jYQ=='\n") + writeExecutable(t, filepath.Join(fakeBin, "grpcurl"), `#!/bin/sh +printf '{\n "stargates": []\n}\n' +printf 'ERROR:\n Code: DeadlineExceeded\n Message: context deadline exceeded\n' >&2 +exit 1 +`) + + output, err := runRegistrationScript(t, fakeBin, "observe-watch-stargates.sh", "127.0.0.1:50071", "router.nvcf.svc", "tls", "nvcf", "context", "3") + if err != nil { + t.Fatalf("observe WatchStargates: %v\n%s", err, output) + } + for _, want := range []string{`"stargates"`, "DeadlineExceeded"} { + if !strings.Contains(output, want) { + t.Fatalf("output = %q, want %q", output, want) + } + } +} + +func TestObserveWatchStargatesScriptRejectsDeadlineWithoutSnapshot(t *testing.T) { + fakeBin := t.TempDir() + writeExecutable(t, filepath.Join(fakeBin, "kubectl"), "#!/bin/sh\nprintf 'dGVzdC1jYQ=='\n") + writeExecutable(t, filepath.Join(fakeBin, "grpcurl"), "#!/bin/sh\nprintf 'context deadline exceeded\\n' >&2\nexit 1\n") + + output, err := runRegistrationScript(t, fakeBin, "observe-watch-stargates.sh", "127.0.0.1:50071", "router.nvcf.svc", "tls", "nvcf", "context", "3") + if err == nil { + t.Fatal("expected missing snapshot failure") + } + if !strings.Contains(output, "did not return a streamed snapshot") { + t.Fatalf("output = %q, want missing snapshot diagnostic", output) + } +} + +func TestWaitPylonMetricsScriptCountsConnectedSeries(t *testing.T) { + fakeBin := t.TempDir() + writeExecutable(t, filepath.Join(fakeBin, "kubectl"), `#!/bin/sh +case "$*" in + *"get pods -A -o json"*) printf '{"items":[]}' ;; + *) + printf 'pylon_registration_stream_connected{router="a"} 1\n' + printf 'pylon_registration_stream_connected{router="b"} 1\n' + printf 'pylon_registration_stream_connected{router="c"} 1\n' + printf 'pylon_reverse_tunnel_connected{router="a"} 1\n' + printf 'pylon_reverse_tunnel_connected{router="b"} 1\n' + printf 'pylon_reverse_tunnel_connected{router="c"} 1\n' + ;; +esac +`) + writeExecutable(t, filepath.Join(fakeBin, "jq"), "#!/bin/sh\nprintf 'functions\\tworker-0\\n'\n") + + output, err := runRegistrationScript( + t, + fakeBin, + "wait-pylon-metrics.sh", + "llm-worker", + "k3d-ncp-local-compute-1", + "1s", + "pylon_registration_stream_connected", + "exactly", + "3", + "pylon_reverse_tunnel_connected", + "at least", + "3", + ) + if err != nil { + t.Fatalf("wait for Pylon metrics: %v\n%s", err, output) + } + for _, want := range []string{"pylon_registration_stream_connected=3", "pylon_reverse_tunnel_connected=3"} { + if !strings.Contains(output, want) { + t.Fatalf("output = %q, want %q", output, want) + } + } +} + +func runRegistrationScript(t *testing.T, fakeBin, scriptName string, args ...string) (string, error) { + t.Helper() + script := filepath.Join("..", "scripts", scriptName) + command := exec.Command("bash", append([]string{script}, args...)...) + command.Env = append(os.Environ(), "PATH="+fakeBin+string(os.PathListSeparator)+os.Getenv("PATH")) + output, err := command.CombinedOutput() + return string(output), err +} + +func writeExecutable(t *testing.T, path, body string) { + t.Helper() + if err := os.WriteFile(path, []byte(body), 0o755); err != nil { + t.Fatalf("write executable %s: %v", path, err) + } +} diff --git a/tests/bdd/scripts/observe-watch-stargates.sh b/tests/bdd/scripts/observe-watch-stargates.sh new file mode 100644 index 000000000..139ed8706 --- /dev/null +++ b/tests/bdd/scripts/observe-watch-stargates.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [[ $# -ne 6 ]]; then + echo "usage: $0 " >&2 + exit 64 +fi + +endpoint="$1" +tls_authority="$2" +ca_secret="$3" +namespace="$4" +kube_context="$5" +duration_seconds="$6" + +for value_name in endpoint tls_authority ca_secret namespace kube_context; do + if [[ -z "${!value_name}" ]]; then + echo "$value_name must be non-empty" >&2 + exit 64 + fi +done +if ! [[ "$duration_seconds" =~ ^[1-9][0-9]*$ ]]; then + echo "duration-seconds must be a positive integer, got: $duration_seconds" >&2 + exit 64 +fi +for tool in kubectl base64 grpcurl; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "required tool not found: $tool" >&2 + exit 127 + fi +done + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +repo_root="$(cd "$script_dir/../../.." && pwd -P)" +proto_path="$repo_root/src/libraries/rust/stargate/crates/proto/proto" +ca_file="$(mktemp "${TMPDIR:-/tmp}/nvcf-bdd-watch-ca.XXXXXX")" +trap 'rm -f "$ca_file"' EXIT + +kubectl --context "$kube_context" get secret "$ca_secret" -n "$namespace" \ + -o 'jsonpath={.data.ca\.crt}' | base64 -d >"$ca_file" +if [[ ! -s "$ca_file" ]]; then + echo "CA secret $namespace/$ca_secret did not contain ca.crt" >&2 + exit 1 +fi + +set +e +output="$(grpcurl \ + -max-time "$duration_seconds" \ + -cacert "$ca_file" \ + -authority "$tls_authority" \ + -import-path "$proto_path" \ + -proto stargate.proto \ + "$endpoint" \ + stargate.StargateControlPlane/WatchStargates 2>&1)" +grpcurl_status=$? +set -e + +printf '%s\n' "$output" + +if [[ "$grpcurl_status" -eq 0 ]]; then + echo "WatchStargates ended before the observation deadline" >&2 + exit 1 +fi +if ! grep -Eq '^[[:space:]]*\{' <<<"$output"; then + echo "WatchStargates did not return a streamed snapshot" >&2 + exit 1 +fi +if ! grep -Eiq 'DeadlineExceeded|context deadline exceeded' <<<"$output"; then + echo "WatchStargates failed before the expected observation deadline" >&2 + exit 1 +fi + diff --git a/tests/bdd/scripts/wait-pylon-metrics.sh b/tests/bdd/scripts/wait-pylon-metrics.sh new file mode 100644 index 000000000..2eb697598 --- /dev/null +++ b/tests/bdd/scripts/wait-pylon-metrics.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [[ $# -lt 6 || $(( ( $# - 3 ) % 3 )) -ne 0 ]]; then + echo "usage: $0 [...]" >&2 + exit 64 +fi + +container_name="$1" +kube_context="$2" +timeout="$3" +shift 3 + +if [[ -z "$container_name" || -z "$kube_context" ]]; then + echo "container and kube-context must be non-empty" >&2 + exit 64 +fi +if ! [[ "$timeout" =~ ^([1-9][0-9]*)(s|m|h)$ ]]; then + echo "timeout must be a positive duration ending in s, m, or h, got: $timeout" >&2 + exit 64 +fi +case "${BASH_REMATCH[2]}" in + s) timeout_seconds="${BASH_REMATCH[1]}" ;; + m) timeout_seconds=$(( BASH_REMATCH[1] * 60 )) ;; + h) timeout_seconds=$(( BASH_REMATCH[1] * 3600 )) ;; +esac +for tool in kubectl jq awk; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "required tool not found: $tool" >&2 + exit 127 + fi +done + +metric_names=() +comparisons=() +expected_counts=() +while [[ $# -gt 0 ]]; do + metric="$1" + comparison="$2" + expected_count="$3" + shift 3 + + if ! [[ "$metric" =~ ^[a-zA-Z_:][a-zA-Z0-9_:]*$ ]]; then + echo "invalid Prometheus metric name: $metric" >&2 + exit 64 + fi + if [[ "$comparison" != "exactly" && "$comparison" != "at least" ]]; then + echo "comparison must be exactly or at least, got: $comparison" >&2 + exit 64 + fi + if ! [[ "$expected_count" =~ ^[0-9]+$ ]]; then + echo "count must be a non-negative integer, got: $expected_count" >&2 + exit 64 + fi + + metric_names+=("$metric") + comparisons+=("$comparison") + expected_counts+=("$expected_count") +done + +connected_series_count() { + local metric="$1" + awk -v metric="$metric" ' + { + series = $1 + if ((series == metric || index(series, metric "{") == 1) && $NF == "1") { + count++ + } + } + END { print count + 0 } + ' +} + +deadline=$(( $(date +%s) + timeout_seconds )) +last_summary="no running pod containing container $container_name" + +while true; do + pod_row="$(kubectl --context "$kube_context" get pods -A -o json | jq -r --arg container "$container_name" ' + [.items[] + | select(.metadata.deletionTimestamp == null) + | select(.status.phase == "Running") + | select(any(.spec.containers[]?; .name == $container)) + | [.metadata.namespace, .metadata.name] + | @tsv] + | first // empty + ' || true)" + + if [[ -n "$pod_row" ]]; then + read -r namespace pod_name <<<"$pod_row" + metrics="$(kubectl --context "$kube_context" get --raw "/api/v1/namespaces/$namespace/pods/$pod_name:9089/proxy/metrics" 2>/dev/null || true)" + all_match=true + summaries=() + + for index in "${!metric_names[@]}"; do + metric="${metric_names[$index]}" + comparison="${comparisons[$index]}" + expected_count="${expected_counts[$index]}" + observed_count="$(printf '%s\n' "$metrics" | connected_series_count "$metric")" + summaries+=("$metric=$observed_count") + + case "$comparison" in + exactly) + [[ "$observed_count" -eq "$expected_count" ]] || all_match=false + ;; + "at least") + [[ "$observed_count" -ge "$expected_count" ]] || all_match=false + ;; + esac + done + + last_summary="${summaries[*]}" + if [[ "$all_match" == true ]]; then + printf '%s\n' "${summaries[@]}" + exit 0 + fi + fi + + now="$(date +%s)" + if [[ "$now" -ge "$deadline" ]]; then + echo "timed out after $timeout waiting for Pylon metrics in context $kube_context ($last_summary)" >&2 + exit 1 + fi + remaining=$(( deadline - now )) + sleep_seconds=5 + if [[ "$remaining" -lt "$sleep_seconds" ]]; then + sleep_seconds="$remaining" + fi + sleep "$sleep_seconds" +done diff --git a/tests/bdd/steps/assertion_steps.go b/tests/bdd/steps/assertion_steps.go index dcd5bac6a..0b036b912 100644 --- a/tests/bdd/steps/assertion_steps.go +++ b/tests/bdd/steps/assertion_steps.go @@ -32,8 +32,11 @@ import ( // values from the feature file. func registerAssertionSteps(ctx *godog.ScenarioContext, sc *ScenarioContext) { ctx.Step(`^the command exit code should be (\d+)$`, sc.commandExitCodeShouldBe) + ctx.Step(`^the command should fail$`, sc.commandShouldFail) ctx.Step(`^the command output should contain "([^"]*)"$`, sc.commandOutputShouldContain) ctx.Step(`^the command output should not contain "([^"]*)"$`, sc.commandOutputShouldNotContain) + ctx.Step(`^the command output should contain all:$`, sc.commandOutputShouldContainAll) + ctx.Step(`^the command output should contain one of:$`, sc.commandOutputShouldContainOneOf) ctx.Step(`^file "([^"]*)" should exist$`, sc.fileShouldExist) ctx.Step(`^yaml file "([^"]*)" key "([^"]*)" should equal "([^"]*)"$`, sc.yamlFileKeyShouldEqual) ctx.Step(`^yaml file "([^"]*)" key "([^"]*)" should not be empty$`, sc.yamlFileKeyShouldNotBeEmpty) @@ -87,6 +90,13 @@ func (sc *ScenarioContext) commandExitCodeShouldBe(expected int) error { return nil } +func (sc *ScenarioContext) commandShouldFail() error { + if sc.LastResult.ExitCode == 0 { + return fmt.Errorf("exit code = 0, want non-zero (see %s for stdout/stderr)", sc.Suite.Config.CommandLogDir) + } + return nil +} + func (sc *ScenarioContext) commandOutputShouldContain(needle string) error { combined := combinedOutput(sc.LastResult) resolved := dsl.Interpolate(needle) @@ -105,6 +115,33 @@ func (sc *ScenarioContext) commandOutputShouldNotContain(needle string) error { return nil } +func (sc *ScenarioContext) commandOutputShouldContainAll(table *godog.Table) error { + needles, err := tableToSingleColumn(table, "text") + if err != nil { + return err + } + for _, needle := range needles { + if err := sc.commandOutputShouldContain(needle); err != nil { + return err + } + } + return nil +} + +func (sc *ScenarioContext) commandOutputShouldContainOneOf(table *godog.Table) error { + needles, err := tableToSingleColumn(table, "text") + if err != nil { + return err + } + combined := combinedOutput(sc.LastResult) + for _, needle := range needles { + if strings.Contains(combined, dsl.Interpolate(needle)) { + return nil + } + } + return fmt.Errorf("output does not contain any of the %d expected values", len(needles)) +} + func (sc *ScenarioContext) yamlFileKeyShouldEqual(path, key, expected string) error { resolvedPath := sc.resolvePath(dsl.Interpolate(path)) got, found, err := dsl.ReadYAMLKey(resolvedPath, key) diff --git a/tests/bdd/steps/context.go b/tests/bdd/steps/context.go index 55acfc397..a88bed910 100644 --- a/tests/bdd/steps/context.go +++ b/tests/bdd/steps/context.go @@ -80,6 +80,7 @@ func RegisterAll(ctx *godog.ScenarioContext, sc *ScenarioContext) { registerFileSteps(ctx, sc) registerCommandSteps(ctx, sc) registerNVCFCLISteps(ctx, sc) + registerRegistrationSteps(ctx, sc) registerAssertionSteps(ctx, sc) registerInfraSteps(ctx, sc) } diff --git a/tests/bdd/steps/registration_steps.go b/tests/bdd/steps/registration_steps.go new file mode 100644 index 000000000..74f28dae4 --- /dev/null +++ b/tests/bdd/steps/registration_steps.go @@ -0,0 +1,105 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package steps + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/cucumber/godog" + + "nvcf-bdd/dsl" +) + +func registerRegistrationSteps(ctx *godog.ScenarioContext, sc *ScenarioContext) { + ctx.Step(`^I successfully observe WatchStargates at "([^"]*)" with TLS authority "([^"]*)" using CA secret "([^"]*)" in namespace "([^"]*)" and context "([^"]*)" for "([^"]*)" seconds$`, sc.iSuccessfullyObserveWatchStargates) + ctx.Step(`^a pod containing container "([^"]*)" using context "([^"]*)" should report Pylon metrics within "([^"]*)":$`, sc.podShouldReportPylonMetrics) +} + +func (sc *ScenarioContext) iSuccessfullyObserveWatchStargates( + ctx context.Context, + endpoint, + authority, + caSecret, + namespace, + kubeContext, + durationSeconds string, +) error { + command, err := dsl.WatchStargatesCommand(endpoint, authority, caSecret, namespace, kubeContext, durationSeconds) + if err != nil { + return err + } + if err := sc.runResolvedSuccessfully(ctx, command); err != nil { + return fmt.Errorf("WatchStargates at %q with TLS authority %q failed: %w", dsl.Interpolate(endpoint), dsl.Interpolate(authority), err) + } + return nil +} + +func (sc *ScenarioContext) podShouldReportPylonMetrics( + ctx context.Context, + containerName, + kubeContext, + timeout string, + table *godog.Table, +) error { + expectations, err := tableToPylonMetricExpectations(table) + if err != nil { + return err + } + command, err := dsl.PylonMetricsCommand(containerName, kubeContext, timeout, expectations) + if err != nil { + return err + } + if err := sc.runResolvedSuccessfully(ctx, command); err != nil { + return fmt.Errorf("pod containing container %q did not report expected Pylon metrics: %w", dsl.Interpolate(containerName), err) + } + return nil +} + +func tableToPylonMetricExpectations(table *godog.Table) ([]dsl.PylonMetricExpectation, error) { + if table == nil || len(table.Rows) < 2 { + return nil, fmt.Errorf("table must have metric, comparison, and count headers and at least one data row") + } + headers := table.Rows[0].Cells + if len(headers) != 3 || + strings.TrimSpace(headers[0].Value) != "metric" || + strings.TrimSpace(headers[1].Value) != "comparison" || + strings.TrimSpace(headers[2].Value) != "count" { + return nil, fmt.Errorf("table headers must be metric, comparison, and count") + } + + expectations := make([]dsl.PylonMetricExpectation, 0, len(table.Rows)-1) + for index, row := range table.Rows[1:] { + if len(row.Cells) != len(headers) { + return nil, fmt.Errorf("row %d has %d cells, expected %d", index+1, len(row.Cells), len(headers)) + } + countText := strings.TrimSpace(dsl.Interpolate(row.Cells[2].Value)) + count, err := strconv.Atoi(countText) + if err != nil || count < 0 { + return nil, fmt.Errorf("row %d count must be a non-negative integer", index+1) + } + expectations = append(expectations, dsl.PylonMetricExpectation{ + Metric: row.Cells[0].Value, + Comparison: row.Cells[1].Value, + Count: count, + }) + } + return expectations, nil +} diff --git a/tests/bdd/steps/steps_test.go b/tests/bdd/steps/steps_test.go index 1189730d8..af21d385f 100644 --- a/tests/bdd/steps/steps_test.go +++ b/tests/bdd/steps/steps_test.go @@ -1418,6 +1418,109 @@ func TestCommandOutputContainsAssertion(t *testing.T) { } } +func TestCommandShouldFailAcceptsNonZeroWithoutCaching(t *testing.T) { + sc, _ := newScenarioContext(t) + sc.LastCommand = "grpcurl rejected-call" + sc.LastResult = harness.Result{ExitCode: 1, Stderr: "certificate is not trusted"} + + if err := sc.commandShouldFail(); err != nil { + t.Fatalf("assert command failure: %v", err) + } + if sc.Suite.Cache.Has(sc.LastCommand) { + t.Fatal("failed command should not enter the successful-command cache") + } + + sc.LastResult = harness.Result{ExitCode: 0} + if err := sc.commandShouldFail(); err == nil { + t.Fatal("expected successful command to fail the negative assertion") + } +} + +func TestCommandOutputTableAssertionsInterpolateExpectedText(t *testing.T) { + sc, _ := newScenarioContext(t) + t.Setenv("BDD_EXPECTED_DIAGNOSTIC", "certificate is not trusted") + sc.LastResult = harness.Result{ + Stdout: "request rejected\n", + Stderr: "certificate is not trusted\ncontext deadline exceeded\n", + } + + all := docTable(t, [][]string{ + {"text"}, + {"${BDD_EXPECTED_DIAGNOSTIC}"}, + {"context deadline exceeded"}, + }) + if err := sc.commandOutputShouldContainAll(all); err != nil { + t.Fatalf("contain all: %v", err) + } + + oneOf := docTable(t, [][]string{ + {"text"}, + {"certificate signed by unknown authority"}, + {"${BDD_EXPECTED_DIAGNOSTIC}"}, + }) + if err := sc.commandOutputShouldContainOneOf(oneOf); err != nil { + t.Fatalf("contain one of: %v", err) + } +} + +func TestISuccessfullyObserveWatchStargatesRunsExplicitCommand(t *testing.T) { + sc, fake := newScenarioContext(t) + fake.result = harness.Result{ExitCode: 0, Stdout: "{\n \"stargates\": []\n}\n"} + + err := sc.iSuccessfullyObserveWatchStargates( + context.Background(), + "127.0.0.1:50071", + "llm-request-router.nvcf.svc.cluster.local", + "stargate-quic-tls", + "nvcf", + "k3d-ncp-local-cp", + "3", + ) + if err != nil { + t.Fatalf("observe WatchStargates: %v", err) + } + want := "bash tests/bdd/scripts/observe-watch-stargates.sh 127.0.0.1:50071 llm-request-router.nvcf.svc.cluster.local stargate-quic-tls nvcf k3d-ncp-local-cp 3" + if len(fake.runs) != 1 || fake.runs[0].command != want { + t.Fatalf("runs = %#v, want %q", fake.runs, want) + } + if !strings.Contains(sc.LastResult.Stdout, "stargates") { + t.Fatalf("last result = %#v, want preserved WatchStargates output", sc.LastResult) + } +} + +func TestPodShouldReportPylonMetricsRunsVisibleExpectations(t *testing.T) { + sc, fake := newScenarioContext(t) + fake.result = harness.Result{ExitCode: 0} + table := docTable(t, [][]string{ + {"metric", "comparison", "count"}, + {"pylon_registration_stream_connected", "exactly", "5"}, + {"pylon_reverse_tunnel_connected", "at least", "3"}, + }) + + if err := sc.podShouldReportPylonMetrics(context.Background(), "llm-worker", "k3d-ncp-local-compute-1", "10m", table); err != nil { + t.Fatalf("observe Pylon metrics: %v", err) + } + want := "bash tests/bdd/scripts/wait-pylon-metrics.sh llm-worker k3d-ncp-local-compute-1 10m pylon_registration_stream_connected exactly 5 pylon_reverse_tunnel_connected 'at least' 3" + if len(fake.runs) != 1 || fake.runs[0].command != want { + t.Fatalf("runs = %#v, want %q", fake.runs, want) + } +} + +func TestPylonMetricTableRejectsInvalidStructureBeforeRunning(t *testing.T) { + sc, fake := newScenarioContext(t) + table := docTable(t, [][]string{ + {"metric", "comparison", "count"}, + {"pylon_registration_stream_connected", "exactly", "not-a-count"}, + }) + + if err := sc.podShouldReportPylonMetrics(context.Background(), "llm-worker", "context", "10m", table); err == nil { + t.Fatal("expected invalid count error") + } + if len(fake.runs) != 0 { + t.Fatalf("runs = %d, want 0 before table validation", len(fake.runs)) + } +} + func TestJSONOutputContainsRowsAssertion(t *testing.T) { sc, _ := newScenarioContext(t) sc.LastResult = harness.Result{Stdout: `[{"name":"api","namespace":"nvcf"}]`} From e9c456092e474bfa21ce9f339bba6e6f5d57e8a6 Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Sun, 30 Aug 2026 00:39:24 -0700 Subject: [PATCH 2/6] fix(bdd): harden registration observability assertions Reject transport failures, premature watch termination, ambiguous Pylon targets, empty resolved output expectations, and timestamped sample miscounts. Refs #1358 Signed-off-by: Stephanie Baum --- tests/bdd/PLAN.md | 17 +-- tests/bdd/dsl/registration.go | 15 ++- tests/bdd/dsl/registration_test.go | 106 +++++++++++++++++-- tests/bdd/scripts/observe-watch-stargates.sh | 26 +++-- tests/bdd/scripts/wait-pylon-metrics.sh | 84 +++++++++------ tests/bdd/steps/assertion_steps.go | 41 +++++-- tests/bdd/steps/registration_steps.go | 9 +- tests/bdd/steps/steps_test.go | 37 ++++++- 8 files changed, 256 insertions(+), 79 deletions(-) diff --git a/tests/bdd/PLAN.md b/tests/bdd/PLAN.md index 8b940a83b..e1b7ce77b 100644 --- a/tests/bdd/PLAN.md +++ b/tests/bdd/PLAN.md @@ -167,10 +167,10 @@ original order. Repeated options and empty values are preserved. |------|-------| | `Then the command exit code should be {int}` | Last-run exit code. | | `Then the command should fail` | Requires a non-zero last-run exit code. It does not accept a runner error that prevented command execution and never records the failed command in the successful-command cache. | -| `Then the command output should contain {string}` | Substring match on combined stdout + stderr. | -| `Then the command output should not contain {string}` | Negative substring match. | -| `Then the command output should contain all:` (table) | Requires a `text` header and one or more strings. Every interpolated string must appear in combined stdout + stderr. | -| `Then the command output should contain one of:` (table) | Requires a `text` header and one or more strings. At least one interpolated string must appear in combined stdout + stderr. | +| `Then the command output should contain {string}` | Substring match on combined stdout + stderr. The interpolated value must not be empty or whitespace-only. | +| `Then the command output should not contain {string}` | Negative substring match. The interpolated value must not be empty or whitespace-only. | +| `Then the command output should contain all:` (table) | Requires a `text` header and one or more strings. Every interpolated string must be non-empty and appear in combined stdout + stderr. | +| `Then the command output should contain one of:` (table) | Requires a `text` header and one or more strings. Every interpolated candidate must be non-empty, and at least one must appear in combined stdout + stderr. | | `Then file {string} should exist` | | | `Then yaml file {string} key {string} should equal {string}` | Reads the YAML file, walks the dotted key path, compares to the value (with `${VAR}` expansion). | | `Then yaml file {string} key {string} should not be empty` | Same key resolution; passes if the resolved value is non-empty. Use for non-deterministic outputs (cluster IDs, identity sources) where exact-value assertions are wrong. | @@ -192,7 +192,7 @@ original order. Repeated options and empty values are preserved. | `Then deployment {string} in namespace {string} using context {string} should complete rollout within {string}` | Runs `kubectl rollout status` for the named deployment with the explicit namespace, context, and timeout. Failure messages name the deployment without printing command output. | | `Then NVCFBackend {string} in namespace {string} using context {string} should report agent status {string} within {string}` | Waits for the named backend's `status.agentStatus` to equal the visible value using the explicit namespace, context, and timeout. Failure messages name the backend without printing resource output. | | `Then these Gateway API routes should be accepted and resolved using context {string} within {string}:` (table) | Requires `kind`, `name`, `namespace`, and `parent` headers. Waits for every named route to report both `Accepted=True` and `ResolvedRefs=True` for the named Gateway parent using the explicit context and timeout. The route kind is passed through without an allowlist. Failures name the table row, route, namespace, parent, and unmet condition without printing resource output. | -| `Then a pod containing container {string} using context {string} should report Pylon metrics within {string}:` (table) | Requires `metric`, `comparison`, and `count` headers. Polls the Pylon metrics endpoint of a running pod containing the visible container name. Each metric row counts connected series with value `1`; `comparison` is `exactly` or `at least`, and the expected non-negative count remains visible. | +| `Then every Pylon for function {string} using container {string} and context {string} should report metrics within {string}:` (table) | Requires `metric`, `comparison`, and `count` headers. Polls every running pod selected by the visible `function-name` annotation and container name. Each pod must expose non-empty metrics, and each metric row counts connected series whose sample value is `1`; `comparison` is `exactly` or `at least`, and the expected non-negative count remains visible. Discovery, parsing, and scrape failures remain failures rather than zero metric counts. | #### YAML comparison semantics @@ -540,9 +540,10 @@ type PylonMetricExpectation struct { Count int } -// PylonMetricsCommand builds a bounded Pylon metrics observation for a pod -// selected by container name in an explicit Kubernetes context. -func PylonMetricsCommand(containerName, kubeContext, timeout string, expectations []PylonMetricExpectation) (string, error) +// PylonMetricsCommand builds a Pylon metrics observation for every running +// Pylon pod selected by function name and container name in an explicit +// Kubernetes context and polling window. +func PylonMetricsCommand(functionName, containerName, kubeContext, timeout string, expectations []PylonMetricExpectation) (string, error) // FilesDoNotContain recursively inspects regular files under root and // fails if any interpolated fixed string appears. diff --git a/tests/bdd/dsl/registration.go b/tests/bdd/dsl/registration.go index 4b546bab9..1a2102b6f 100644 --- a/tests/bdd/dsl/registration.go +++ b/tests/bdd/dsl/registration.go @@ -67,12 +67,17 @@ func WatchStargatesCommand(endpoint, authority, caSecret, namespace, kubeContext ), nil } -// PylonMetricsCommand builds a bounded Pylon metrics observation for a pod -// selected by container name in an explicit Kubernetes context. -func PylonMetricsCommand(containerName, kubeContext, timeout string, expectations []PylonMetricExpectation) (string, error) { +// PylonMetricsCommand builds a Pylon metrics observation for every running +// Pylon pod selected by function name and container name in an explicit +// Kubernetes context and polling window. +func PylonMetricsCommand(functionName, containerName, kubeContext, timeout string, expectations []PylonMetricExpectation) (string, error) { + functionName = strings.TrimSpace(Interpolate(functionName)) containerName = strings.TrimSpace(Interpolate(containerName)) kubeContext = strings.TrimSpace(Interpolate(kubeContext)) timeout = strings.TrimSpace(Interpolate(timeout)) + if functionName == "" { + return "", fmt.Errorf("function name is empty") + } if containerName == "" { return "", fmt.Errorf("container name is empty") } @@ -83,10 +88,10 @@ func PylonMetricsCommand(containerName, kubeContext, timeout string, expectation return "", fmt.Errorf("timeout is empty") } if len(expectations) == 0 { - return "", fmt.Errorf("Pylon metric expectations are empty") + return "", fmt.Errorf("pylon metric expectations are empty") } - args := []string{"bash", waitPylonMetricsScript, containerName, kubeContext, timeout} + args := []string{"bash", waitPylonMetricsScript, functionName, containerName, kubeContext, timeout} for index, expectation := range expectations { expectation.Metric = strings.TrimSpace(Interpolate(expectation.Metric)) expectation.Comparison = strings.TrimSpace(Interpolate(expectation.Comparison)) diff --git a/tests/bdd/dsl/registration_test.go b/tests/bdd/dsl/registration_test.go index 606765864..e52a33bdd 100644 --- a/tests/bdd/dsl/registration_test.go +++ b/tests/bdd/dsl/registration_test.go @@ -65,14 +65,14 @@ func TestWatchStargatesCommandRejectsMissingOrInvalidInputs(t *testing.T) { } func TestPylonMetricsCommandKeepsExpectationsExplicit(t *testing.T) { - got, err := PylonMetricsCommand("llm-worker", "k3d-ncp-local-compute-1", "10m", []PylonMetricExpectation{ + got, err := PylonMetricsCommand("bdd-registration-tls", "llm-worker", "k3d-ncp-local-compute-1", "10m", []PylonMetricExpectation{ {Metric: "pylon_registration_stream_connected", Comparison: "exactly", Count: 5}, {Metric: "pylon_reverse_tunnel_connected", Comparison: "at least", Count: 3}, }) if err != nil { t.Fatalf("build Pylon metrics command: %v", err) } - want := "bash tests/bdd/scripts/wait-pylon-metrics.sh llm-worker k3d-ncp-local-compute-1 10m pylon_registration_stream_connected exactly 5 pylon_reverse_tunnel_connected 'at least' 3" + want := "bash tests/bdd/scripts/wait-pylon-metrics.sh bdd-registration-tls llm-worker k3d-ncp-local-compute-1 10m pylon_registration_stream_connected exactly 5 pylon_reverse_tunnel_connected 'at least' 3" if got != want { t.Fatalf("command = %q, want %q", got, want) } @@ -90,7 +90,7 @@ func TestPylonMetricsCommandRejectsInvalidExpectations(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - if _, err := PylonMetricsCommand("llm-worker", "context", "10m", test.expectations); err == nil { + if _, err := PylonMetricsCommand("function", "llm-worker", "context", "10m", test.expectations); err == nil { t.Fatal("expected validation error") } }) @@ -102,11 +102,12 @@ func TestObserveWatchStargatesScriptAcceptsSnapshotThenDeadline(t *testing.T) { writeExecutable(t, filepath.Join(fakeBin, "kubectl"), "#!/bin/sh\nprintf 'dGVzdC1jYQ=='\n") writeExecutable(t, filepath.Join(fakeBin, "grpcurl"), `#!/bin/sh printf '{\n "stargates": []\n}\n' +sleep 1 printf 'ERROR:\n Code: DeadlineExceeded\n Message: context deadline exceeded\n' >&2 exit 1 `) - output, err := runRegistrationScript(t, fakeBin, "observe-watch-stargates.sh", "127.0.0.1:50071", "router.nvcf.svc", "tls", "nvcf", "context", "3") + output, err := runRegistrationScript(t, fakeBin, "observe-watch-stargates.sh", "127.0.0.1:50071", "router.nvcf.svc", "tls", "nvcf", "context", "1") if err != nil { t.Fatalf("observe WatchStargates: %v\n%s", err, output) } @@ -117,6 +118,24 @@ exit 1 } } +func TestObserveWatchStargatesScriptRejectsImmediateProductDeadline(t *testing.T) { + fakeBin := t.TempDir() + writeExecutable(t, filepath.Join(fakeBin, "kubectl"), "#!/bin/sh\nprintf 'dGVzdC1jYQ=='\n") + writeExecutable(t, filepath.Join(fakeBin, "grpcurl"), `#!/bin/sh +printf '{\n "stargates": []\n}\n' +printf 'ERROR:\n Code: DeadlineExceeded\n Message: product returned deadline exceeded\n' >&2 +exit 1 +`) + + output, err := runRegistrationScript(t, fakeBin, "observe-watch-stargates.sh", "127.0.0.1:50071", "router.nvcf.svc", "tls", "nvcf", "context", "3") + if err == nil { + t.Fatal("expected immediate product deadline failure") + } + if !strings.Contains(output, "before the 3s observation deadline") { + t.Fatalf("output = %q, want early deadline diagnostic", output) + } +} + func TestObserveWatchStargatesScriptRejectsDeadlineWithoutSnapshot(t *testing.T) { fakeBin := t.TempDir() writeExecutable(t, filepath.Join(fakeBin, "kubectl"), "#!/bin/sh\nprintf 'dGVzdC1jYQ=='\n") @@ -131,13 +150,19 @@ func TestObserveWatchStargatesScriptRejectsDeadlineWithoutSnapshot(t *testing.T) } } -func TestWaitPylonMetricsScriptCountsConnectedSeries(t *testing.T) { +func TestWaitPylonMetricsScriptChecksEverySelectedPodAndCountsTimestampedSeries(t *testing.T) { fakeBin := t.TempDir() writeExecutable(t, filepath.Join(fakeBin, "kubectl"), `#!/bin/sh case "$*" in - *"get pods -A -o json"*) printf '{"items":[]}' ;; + *"get pods -A -o json"*) + printf '%s\n' '{"items":[' + printf '%s\n' '{"metadata":{"namespace":"functions","name":"worker-0","annotations":{"function-name":"bdd-registration-tls"}},"status":{"phase":"Running"},"spec":{"containers":[{"name":"llm-worker"}]}},' + printf '%s\n' '{"metadata":{"namespace":"functions","name":"worker-1","annotations":{"function-name":"bdd-registration-tls"}},"status":{"phase":"Running"},"spec":{"containers":[{"name":"llm-worker"}]}},' + printf '%s\n' '{"metadata":{"namespace":"other","name":"worker-other","annotations":{"function-name":"another-function"}},"status":{"phase":"Running"},"spec":{"containers":[{"name":"llm-worker"}]}}' + printf '%s\n' ']}' + ;; *) - printf 'pylon_registration_stream_connected{router="a"} 1\n' + printf 'pylon_registration_stream_connected{router="a"} 1 1712345678\n' printf 'pylon_registration_stream_connected{router="b"} 1\n' printf 'pylon_registration_stream_connected{router="c"} 1\n' printf 'pylon_reverse_tunnel_connected{router="a"} 1\n' @@ -146,12 +171,12 @@ case "$*" in ;; esac `) - writeExecutable(t, filepath.Join(fakeBin, "jq"), "#!/bin/sh\nprintf 'functions\\tworker-0\\n'\n") output, err := runRegistrationScript( t, fakeBin, "wait-pylon-metrics.sh", + "bdd-registration-tls", "llm-worker", "k3d-ncp-local-compute-1", "1s", @@ -165,11 +190,74 @@ esac if err != nil { t.Fatalf("wait for Pylon metrics: %v\n%s", err, output) } - for _, want := range []string{"pylon_registration_stream_connected=3", "pylon_reverse_tunnel_connected=3"} { + for _, want := range []string{ + "functions/worker-0 pylon_registration_stream_connected=3", + "functions/worker-0 pylon_reverse_tunnel_connected=3", + "functions/worker-1 pylon_registration_stream_connected=3", + "functions/worker-1 pylon_reverse_tunnel_connected=3", + } { if !strings.Contains(output, want) { t.Fatalf("output = %q, want %q", output, want) } } + if strings.Contains(output, "worker-other") { + t.Fatalf("output = %q, did not want metrics from another function", output) + } +} + +func TestWaitPylonMetricsScriptDoesNotTreatScrapeFailureAsZero(t *testing.T) { + fakeBin := t.TempDir() + writeExecutable(t, filepath.Join(fakeBin, "kubectl"), `#!/bin/sh +case "$*" in + *"get pods -A -o json"*) + printf '%s\n' '{"items":[{"metadata":{"namespace":"functions","name":"worker-0","annotations":{"function-name":"bdd-registration-tls"}},"status":{"phase":"Running"},"spec":{"containers":[{"name":"llm-worker"}]}}]}' + ;; + *) printf 'metrics endpoint unavailable\n' >&2; exit 1 ;; +esac +`) + + output, err := runRegistrationScript( + t, + fakeBin, + "wait-pylon-metrics.sh", + "bdd-registration-tls", + "llm-worker", + "k3d-ncp-local-compute-1", + "1s", + "pylon_registration_stream_connected", + "exactly", + "0", + ) + if err == nil { + t.Fatal("expected metrics scrape failure") + } + if !strings.Contains(output, "metrics scrape failed: metrics endpoint unavailable") { + t.Fatalf("output = %q, want preserved scrape failure", output) + } +} + +func TestWaitPylonMetricsScriptPreservesPodDiscoveryFailure(t *testing.T) { + fakeBin := t.TempDir() + writeExecutable(t, filepath.Join(fakeBin, "kubectl"), "#!/bin/sh\nprintf 'API server unavailable\\n' >&2\nexit 1\n") + + output, err := runRegistrationScript( + t, + fakeBin, + "wait-pylon-metrics.sh", + "bdd-registration-tls", + "llm-worker", + "k3d-ncp-local-compute-1", + "1s", + "pylon_registration_stream_connected", + "exactly", + "0", + ) + if err == nil { + t.Fatal("expected pod discovery failure") + } + if !strings.Contains(output, "pod discovery failed: API server unavailable") { + t.Fatalf("output = %q, want preserved discovery failure", output) + } } func runRegistrationScript(t *testing.T, fakeBin, scriptName string, args ...string) (string, error) { diff --git a/tests/bdd/scripts/observe-watch-stargates.sh b/tests/bdd/scripts/observe-watch-stargates.sh index 139ed8706..9813b3fcb 100644 --- a/tests/bdd/scripts/observe-watch-stargates.sh +++ b/tests/bdd/scripts/observe-watch-stargates.sh @@ -26,7 +26,7 @@ if ! [[ "$duration_seconds" =~ ^[1-9][0-9]*$ ]]; then echo "duration-seconds must be a positive integer, got: $duration_seconds" >&2 exit 64 fi -for tool in kubectl base64 grpcurl; do +for tool in kubectl base64 grpcurl jq; do if ! command -v "$tool" >/dev/null 2>&1; then echo "required tool not found: $tool" >&2 exit 127 @@ -37,7 +37,9 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" repo_root="$(cd "$script_dir/../../.." && pwd -P)" proto_path="$repo_root/src/libraries/rust/stargate/crates/proto/proto" ca_file="$(mktemp "${TMPDIR:-/tmp}/nvcf-bdd-watch-ca.XXXXXX")" -trap 'rm -f "$ca_file"' EXIT +stdout_file="$(mktemp "${TMPDIR:-/tmp}/nvcf-bdd-watch-stdout.XXXXXX")" +stderr_file="$(mktemp "${TMPDIR:-/tmp}/nvcf-bdd-watch-stderr.XXXXXX")" +trap 'rm -f "$ca_file" "$stdout_file" "$stderr_file"' EXIT kubectl --context "$kube_context" get secret "$ca_secret" -n "$namespace" \ -o 'jsonpath={.data.ca\.crt}' | base64 -d >"$ca_file" @@ -47,29 +49,37 @@ if [[ ! -s "$ca_file" ]]; then fi set +e -output="$(grpcurl \ +started_at="$(date +%s)" +grpcurl \ -max-time "$duration_seconds" \ + -emit-defaults \ -cacert "$ca_file" \ -authority "$tls_authority" \ -import-path "$proto_path" \ -proto stargate.proto \ "$endpoint" \ - stargate.StargateControlPlane/WatchStargates 2>&1)" + stargate.StargateControlPlane/WatchStargates >"$stdout_file" 2>"$stderr_file" grpcurl_status=$? +finished_at="$(date +%s)" set -e -printf '%s\n' "$output" +cat "$stdout_file" +cat "$stderr_file" >&2 if [[ "$grpcurl_status" -eq 0 ]]; then echo "WatchStargates ended before the observation deadline" >&2 exit 1 fi -if ! grep -Eq '^[[:space:]]*\{' <<<"$output"; then +if ! jq -se 'length > 0 and all(.[]; type == "object" and (has("stargates") or has("watchStargateUrls")))' "$stdout_file" >/dev/null; then echo "WatchStargates did not return a streamed snapshot" >&2 exit 1 fi -if ! grep -Eiq 'DeadlineExceeded|context deadline exceeded' <<<"$output"; then +if ! grep -Eiq 'DeadlineExceeded|context deadline exceeded' "$stderr_file"; then echo "WatchStargates failed before the expected observation deadline" >&2 exit 1 fi - +elapsed_seconds=$(( finished_at - started_at )) +if [[ "$elapsed_seconds" -lt "$duration_seconds" ]]; then + echo "WatchStargates ended after ${elapsed_seconds}s before the ${duration_seconds}s observation deadline" >&2 + exit 1 +fi diff --git a/tests/bdd/scripts/wait-pylon-metrics.sh b/tests/bdd/scripts/wait-pylon-metrics.sh index 2eb697598..56cf1e1ea 100644 --- a/tests/bdd/scripts/wait-pylon-metrics.sh +++ b/tests/bdd/scripts/wait-pylon-metrics.sh @@ -4,18 +4,19 @@ set -euo pipefail -if [[ $# -lt 6 || $(( ( $# - 3 ) % 3 )) -ne 0 ]]; then - echo "usage: $0 [...]" >&2 +if [[ $# -lt 7 || $(( ( $# - 4 ) % 3 )) -ne 0 ]]; then + echo "usage: $0 [...]" >&2 exit 64 fi -container_name="$1" -kube_context="$2" -timeout="$3" -shift 3 +function_name="$1" +container_name="$2" +kube_context="$3" +timeout="$4" +shift 4 -if [[ -z "$container_name" || -z "$kube_context" ]]; then - echo "container and kube-context must be non-empty" >&2 +if [[ -z "$function_name" || -z "$container_name" || -z "$kube_context" ]]; then + echo "function-name, container, and kube-context must be non-empty" >&2 exit 64 fi if ! [[ "$timeout" =~ ^([1-9][0-9]*)(s|m|h)$ ]]; then @@ -66,7 +67,7 @@ connected_series_count() { awk -v metric="$metric" ' { series = $1 - if ((series == metric || index(series, metric "{") == 1) && $NF == "1") { + if ((series == metric || index(series, metric "{") == 1) && $2 == "1") { count++ } } @@ -75,41 +76,56 @@ connected_series_count() { } deadline=$(( $(date +%s) + timeout_seconds )) -last_summary="no running pod containing container $container_name" +last_summary="no running pod for function $function_name containing container $container_name" while true; do - pod_row="$(kubectl --context "$kube_context" get pods -A -o json | jq -r --arg container "$container_name" ' - [.items[] + if ! pods_json="$(kubectl --context "$kube_context" get pods -A -o json 2>&1)"; then + last_summary="pod discovery failed: $pods_json" + elif ! pod_rows="$(printf '%s\n' "$pods_json" | jq -r --arg function "$function_name" --arg container "$container_name" ' + .items[]? | select(.metadata.deletionTimestamp == null) | select(.status.phase == "Running") + | select(.metadata.annotations["function-name"] == $function) | select(any(.spec.containers[]?; .name == $container)) | [.metadata.namespace, .metadata.name] - | @tsv] - | first // empty - ' || true)" - - if [[ -n "$pod_row" ]]; then - read -r namespace pod_name <<<"$pod_row" - metrics="$(kubectl --context "$kube_context" get --raw "/api/v1/namespaces/$namespace/pods/$pod_name:9089/proxy/metrics" 2>/dev/null || true)" + | @tsv + ' 2>&1)"; then + last_summary="pod discovery response could not be parsed: $pod_rows" + elif [[ -z "$pod_rows" ]]; then + last_summary="no running pod for function $function_name containing container $container_name" + else all_match=true summaries=() - for index in "${!metric_names[@]}"; do - metric="${metric_names[$index]}" - comparison="${comparisons[$index]}" - expected_count="${expected_counts[$index]}" - observed_count="$(printf '%s\n' "$metrics" | connected_series_count "$metric")" - summaries+=("$metric=$observed_count") + while IFS=$'\t' read -r namespace pod_name; do + if ! metrics="$(kubectl --context "$kube_context" get --raw "/api/v1/namespaces/$namespace/pods/$pod_name:9089/proxy/metrics" 2>&1)"; then + summaries+=("$namespace/$pod_name metrics scrape failed: $metrics") + all_match=false + continue + fi + if [[ -z "$metrics" ]]; then + summaries+=("$namespace/$pod_name metrics scrape returned an empty response") + all_match=false + continue + fi + + for index in "${!metric_names[@]}"; do + metric="${metric_names[$index]}" + comparison="${comparisons[$index]}" + expected_count="${expected_counts[$index]}" + observed_count="$(printf '%s\n' "$metrics" | connected_series_count "$metric")" + summaries+=("$namespace/$pod_name $metric=$observed_count") - case "$comparison" in - exactly) - [[ "$observed_count" -eq "$expected_count" ]] || all_match=false - ;; - "at least") - [[ "$observed_count" -ge "$expected_count" ]] || all_match=false - ;; - esac - done + case "$comparison" in + exactly) + [[ "$observed_count" -eq "$expected_count" ]] || all_match=false + ;; + "at least") + [[ "$observed_count" -ge "$expected_count" ]] || all_match=false + ;; + esac + done + done <<<"$pod_rows" last_summary="${summaries[*]}" if [[ "$all_match" == true ]]; then diff --git a/tests/bdd/steps/assertion_steps.go b/tests/bdd/steps/assertion_steps.go index 0b036b912..f651efed9 100644 --- a/tests/bdd/steps/assertion_steps.go +++ b/tests/bdd/steps/assertion_steps.go @@ -99,7 +99,10 @@ func (sc *ScenarioContext) commandShouldFail() error { func (sc *ScenarioContext) commandOutputShouldContain(needle string) error { combined := combinedOutput(sc.LastResult) - resolved := dsl.Interpolate(needle) + resolved, err := resolveOutputNeedle(needle) + if err != nil { + return err + } if !strings.Contains(combined, resolved) { return fmt.Errorf("output does not contain %q", resolved) } @@ -108,7 +111,10 @@ func (sc *ScenarioContext) commandOutputShouldContain(needle string) error { func (sc *ScenarioContext) commandOutputShouldNotContain(needle string) error { combined := combinedOutput(sc.LastResult) - resolved := dsl.Interpolate(needle) + resolved, err := resolveOutputNeedle(needle) + if err != nil { + return err + } if strings.Contains(combined, resolved) { return fmt.Errorf("output contains %q", resolved) } @@ -120,9 +126,14 @@ func (sc *ScenarioContext) commandOutputShouldContainAll(table *godog.Table) err if err != nil { return err } - for _, needle := range needles { - if err := sc.commandOutputShouldContain(needle); err != nil { - return err + combined := combinedOutput(sc.LastResult) + for index, needle := range needles { + resolved, err := resolveOutputNeedle(needle) + if err != nil { + return fmt.Errorf("row %d: %w", index+1, err) + } + if !strings.Contains(combined, resolved) { + return fmt.Errorf("output does not contain %q", resolved) } } return nil @@ -133,15 +144,31 @@ func (sc *ScenarioContext) commandOutputShouldContainOneOf(table *godog.Table) e if err != nil { return err } + resolvedNeedles := make([]string, 0, len(needles)) + for index, needle := range needles { + resolved, err := resolveOutputNeedle(needle) + if err != nil { + return fmt.Errorf("row %d: %w", index+1, err) + } + resolvedNeedles = append(resolvedNeedles, resolved) + } combined := combinedOutput(sc.LastResult) - for _, needle := range needles { - if strings.Contains(combined, dsl.Interpolate(needle)) { + for _, resolved := range resolvedNeedles { + if strings.Contains(combined, resolved) { return nil } } return fmt.Errorf("output does not contain any of the %d expected values", len(needles)) } +func resolveOutputNeedle(needle string) (string, error) { + resolved := dsl.Interpolate(needle) + if strings.TrimSpace(resolved) == "" { + return "", fmt.Errorf("expected output text resolves to an empty value") + } + return resolved, nil +} + func (sc *ScenarioContext) yamlFileKeyShouldEqual(path, key, expected string) error { resolvedPath := sc.resolvePath(dsl.Interpolate(path)) got, found, err := dsl.ReadYAMLKey(resolvedPath, key) diff --git a/tests/bdd/steps/registration_steps.go b/tests/bdd/steps/registration_steps.go index 74f28dae4..4dd957827 100644 --- a/tests/bdd/steps/registration_steps.go +++ b/tests/bdd/steps/registration_steps.go @@ -30,7 +30,7 @@ import ( func registerRegistrationSteps(ctx *godog.ScenarioContext, sc *ScenarioContext) { ctx.Step(`^I successfully observe WatchStargates at "([^"]*)" with TLS authority "([^"]*)" using CA secret "([^"]*)" in namespace "([^"]*)" and context "([^"]*)" for "([^"]*)" seconds$`, sc.iSuccessfullyObserveWatchStargates) - ctx.Step(`^a pod containing container "([^"]*)" using context "([^"]*)" should report Pylon metrics within "([^"]*)":$`, sc.podShouldReportPylonMetrics) + ctx.Step(`^every Pylon for function "([^"]*)" using container "([^"]*)" and context "([^"]*)" should report metrics within "([^"]*)":$`, sc.everyPylonForFunctionShouldReportMetrics) } func (sc *ScenarioContext) iSuccessfullyObserveWatchStargates( @@ -52,8 +52,9 @@ func (sc *ScenarioContext) iSuccessfullyObserveWatchStargates( return nil } -func (sc *ScenarioContext) podShouldReportPylonMetrics( +func (sc *ScenarioContext) everyPylonForFunctionShouldReportMetrics( ctx context.Context, + functionName, containerName, kubeContext, timeout string, @@ -63,12 +64,12 @@ func (sc *ScenarioContext) podShouldReportPylonMetrics( if err != nil { return err } - command, err := dsl.PylonMetricsCommand(containerName, kubeContext, timeout, expectations) + command, err := dsl.PylonMetricsCommand(functionName, containerName, kubeContext, timeout, expectations) if err != nil { return err } if err := sc.runResolvedSuccessfully(ctx, command); err != nil { - return fmt.Errorf("pod containing container %q did not report expected Pylon metrics: %w", dsl.Interpolate(containerName), err) + return fmt.Errorf("pylon pods for function %q using container %q did not report expected metrics: %w", dsl.Interpolate(functionName), dsl.Interpolate(containerName), err) } return nil } diff --git a/tests/bdd/steps/steps_test.go b/tests/bdd/steps/steps_test.go index af21d385f..bcfed0cdb 100644 --- a/tests/bdd/steps/steps_test.go +++ b/tests/bdd/steps/steps_test.go @@ -1463,6 +1463,35 @@ func TestCommandOutputTableAssertionsInterpolateExpectedText(t *testing.T) { } } +func TestCommandOutputAssertionsRejectValuesThatInterpolateToEmpty(t *testing.T) { + sc, _ := newScenarioContext(t) + t.Setenv("BDD_EMPTY_EXPECTATION", "") + sc.LastResult = harness.Result{Stdout: "any output contains the empty string"} + + if err := sc.commandOutputShouldContain("${BDD_EMPTY_EXPECTATION}"); err == nil { + t.Fatal("expected empty single-value expectation to fail") + } + if err := sc.commandOutputShouldNotContain("${BDD_EMPTY_EXPECTATION}"); err == nil { + t.Fatal("expected empty negative expectation to fail validation") + } + + containAll := docTable(t, [][]string{ + {"text"}, + {"${BDD_EMPTY_EXPECTATION}"}, + }) + if err := sc.commandOutputShouldContainAll(containAll); err == nil { + t.Fatal("expected contain-all table with an empty resolved value to fail") + } + containOneOf := docTable(t, [][]string{ + {"text"}, + {"any output"}, + {"${BDD_EMPTY_EXPECTATION}"}, + }) + if err := sc.commandOutputShouldContainOneOf(containOneOf); err == nil { + t.Fatal("expected contain-one-of table with an empty resolved value to fail") + } +} + func TestISuccessfullyObserveWatchStargatesRunsExplicitCommand(t *testing.T) { sc, fake := newScenarioContext(t) fake.result = harness.Result{ExitCode: 0, Stdout: "{\n \"stargates\": []\n}\n"} @@ -1488,7 +1517,7 @@ func TestISuccessfullyObserveWatchStargatesRunsExplicitCommand(t *testing.T) { } } -func TestPodShouldReportPylonMetricsRunsVisibleExpectations(t *testing.T) { +func TestEveryPylonForFunctionShouldReportMetricsRunsVisibleExpectations(t *testing.T) { sc, fake := newScenarioContext(t) fake.result = harness.Result{ExitCode: 0} table := docTable(t, [][]string{ @@ -1497,10 +1526,10 @@ func TestPodShouldReportPylonMetricsRunsVisibleExpectations(t *testing.T) { {"pylon_reverse_tunnel_connected", "at least", "3"}, }) - if err := sc.podShouldReportPylonMetrics(context.Background(), "llm-worker", "k3d-ncp-local-compute-1", "10m", table); err != nil { + if err := sc.everyPylonForFunctionShouldReportMetrics(context.Background(), "bdd-registration-tls", "llm-worker", "k3d-ncp-local-compute-1", "10m", table); err != nil { t.Fatalf("observe Pylon metrics: %v", err) } - want := "bash tests/bdd/scripts/wait-pylon-metrics.sh llm-worker k3d-ncp-local-compute-1 10m pylon_registration_stream_connected exactly 5 pylon_reverse_tunnel_connected 'at least' 3" + want := "bash tests/bdd/scripts/wait-pylon-metrics.sh bdd-registration-tls llm-worker k3d-ncp-local-compute-1 10m pylon_registration_stream_connected exactly 5 pylon_reverse_tunnel_connected 'at least' 3" if len(fake.runs) != 1 || fake.runs[0].command != want { t.Fatalf("runs = %#v, want %q", fake.runs, want) } @@ -1513,7 +1542,7 @@ func TestPylonMetricTableRejectsInvalidStructureBeforeRunning(t *testing.T) { {"pylon_registration_stream_connected", "exactly", "not-a-count"}, }) - if err := sc.podShouldReportPylonMetrics(context.Background(), "llm-worker", "context", "10m", table); err == nil { + if err := sc.everyPylonForFunctionShouldReportMetrics(context.Background(), "function", "llm-worker", "context", "10m", table); err == nil { t.Fatal("expected invalid count error") } if len(fake.runs) != 0 { From 2dbaa0916c4d6cc03ae95ebffd6e2870d5b49e97 Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Fri, 28 Aug 2026 03:10:41 -0700 Subject: [PATCH 3/6] test(bdd): cover secure multi-region registration --- ...mfile-llm-registration-multiregion.feature | 162 ++++++++++++++++++ tests/bdd/godog_test.go | 95 ++++++++++ tests/bdd/scripts/install-llm-region-b.sh | 138 +++++++++++++++ 3 files changed, 395 insertions(+) create mode 100644 tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature create mode 100755 tests/bdd/scripts/install-llm-region-b.sh diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature new file mode 100644 index 000000000..011c936f1 --- /dev/null +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature @@ -0,0 +1,162 @@ +@ncp-local @multi-cluster @helmfile @pki @llm-registration @multi-region +Feature: Register an LLM worker securely with routers in two local regions + As a self-managed NVCF operator, + I want recursive router discovery to retain explicit HTTPS transport, + so that one worker can register with routable Deployment and StatefulSet routers across regions. + + Rule: A secure remote Watch URI expands the registered router topology + + Background: + Given these environment variables are set: + | name | + | NGC_API_KEY | + | NVCF_CLI | + | REPO_ROOT | + | SAMPLE_NGC_ORG | + | SAMPLE_NGC_TEAM | + And I prepare Helmfile environment "local-bdd-registration-multiregion" for stack "self-managed" from fixture "tests/bdd/fixtures/self-managed-local-bdd-multi.yaml" with values: + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.workerEndpoints.llmRequestRouterAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | + | addons.llm.requestRouter.workload.kind | Deployment | + | addons.llm.requestRouter.discovery.remoteWatchUrls[0] | https://region-b-watch.nvcf.svc.cluster.local:50071 | + | addons.llm.requestRouter.grpcTls.dnsNames[1] | region-b-watch.nvcf.svc.cluster.local | + | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | + | addons.llm.pki.dnsNames[2] | *.llm-request-router-region-b-headless.nvcf.svc.cluster.local | + | observability.profile | disabled | + And I prepare Helmfile environment "local-bdd-registration-multiregion" for stack "nvcf-compute-plane" from fixture "tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml" with values: + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | observability.profile | disabled | + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-multiregion-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential + When I run command "k3d cluster get ncp-local" + Then the command exit code should be 1 + And multi-cluster ncp-local compute clusters are running: + | ncp-local-compute-1 | + And command has succeeded: + """ + kubectl config use-context k3d-ncp-local-cp + """ + And the "nvcr-pull-secret" image pull secret exists in namespaces: + | cassandra-system | + | nats-system | + | nvcf | + | api-keys | + | ess | + | sis | + | vault-system | + | nvca-operator | + | cert-manager | + + @llm-registration-multiregion-install + Scenario: Operator installs two secure regions with distinct router workload identities + When I run command "make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-multiregion" + Then the command exit code should be 0 + And the rendered manifests in "deploy/stacks/self-managed/out" should contain: + | text | + | kind: Deployment | + | --remote-stargate-url=https://region-b-watch.nvcf.svc.cluster.local:50071 | + + When I run command "make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-registration-multiregion" + Then the command exit code should be 0 + When I run command "kubectl --context k3d-ncp-local-cp wait certificate llm-request-router-grpc-tls -n envoy-gateway-system --for=condition=Ready --timeout=5m" + Then the command exit code should be 0 + When I run command "kubectl --context k3d-ncp-local-cp rollout status deployment/llm-request-router -n nvcf --timeout=10m" + Then the command exit code should be 0 + + When I run command "tests/bdd/scripts/install-llm-region-b.sh" + Then the command exit code should be 0 + + # The initial region advertises an explicit HTTPS recursive seed while + # retaining every concrete Deployment pod identity. + When I run command: + """ + /bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); pods=$(kubectl --context k3d-ncp-local-cp get pods -n nvcf -l app.kubernetes.io/instance=llm-request-router,app.kubernetes.io/name=llm-request-router -o jsonpath="{range .items[*]}{.metadata.name}{\"\\n\"}{end}"); count=0; while IFS= read -r pod; do [ -z "$pod" ] && continue; printf "%s" "$output" | grep -Fq "$pod"; count=$((count + 1)); done <<<"$pods"; [ "$count" -eq 3 ]; printf "%s" "$output" | grep -Fq "https://region-b-watch.nvcf.svc.cluster.local:50071"; printf "region-a-deployment=%s remote-watch=https\n" "$count"' + """ + Then the command exit code should be 0 + And the command output should contain "region-a-deployment=3 remote-watch=https" + + # The remote HTTPS authority resolves to two stable StatefulSet router + # identities and never relies on a dashed-IP SRV alias. + When I run command: + """ + /bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); printf "%s" "$output" | grep -Fq "llm-request-router-region-b-0"; printf "%s" "$output" | grep -Fq "llm-request-router-region-b-1"; printf "region-b-statefulset=2 tls=https\n"' + """ + Then the command exit code should be 0 + And the command output should contain "region-b-statefulset=2 tls=https" + + When I run command: + """ + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml self-hosted --control-plane-stack deploy/stacks/self-managed --env local-bdd-registration-multiregion --control-plane-context k3d-ncp-local-cp --compute-plane-context k3d-ncp-local-compute-1 control-plane profile export --cluster-name ncp-local-cp + """ + Then the command exit code should be 0 + And file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should exist + And yaml file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should have non-empty keys: + | key | + | managementTls.caBundlePem | + | transportTls.trustBundleFingerprint | + | transportTls.trustBundlePem | + + And command has succeeded: + """ + /bin/sh -c '${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml init >/dev/null' + """ + When I run command "kubectl config use-context k3d-ncp-local-compute-1" + Then the command exit code should be 0 + When I run command: + """ + make -C deploy/stacks/nvcf-compute-plane register-cluster CLUSTER_NAME=ncp-local-compute-1 CONTROL_PLANE_PROFILE=${REPO_ROOT}/deploy/stacks/self-managed/out/control-plane-profile.yaml COMPUTE_KUBE_CONTEXT=k3d-ncp-local-compute-1 NVCF_CLI=${NVCF_CLI} NVCF_CLI_CONFIG=${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml + """ + Then the command exit code should be 0 + And the "nvcr-pull-secret" image pull secret exists in namespaces: + | nvca-operator | + When I run command: + """ + make -C deploy/stacks/nvcf-compute-plane install CLUSTER_NAME=ncp-local-compute-1 HELMFILE_ENV=local-bdd-registration-multiregion COMPUTE_KUBE_CONTEXT=k3d-ncp-local-compute-1 NVCF_CLI=${NVCF_CLI} + """ + Then the command exit code should be 0 + Then NVCFBackend "ncp-local-compute-1" in namespace "nvca-operator" using context "k3d-ncp-local-compute-1" should report agent status "healthy" within "10m" + + @llm-registration-multiregion-runtime + Scenario: Pylon recursively registers with both regions and serves an authenticated request + Given I use NVCF CLI config "${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml" + When I successfully create function "bdd-registration-multiregion" from image "nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-openai-compatible-sample:local" with CLI options: + | option | value | + | --function-type | LLM | + | --inference-url | /v1/chat/completions | + | --inference-port | 8000 | + | --health-uri | /health | + | --health-port | 8000 | + | --health-timeout | PT30S | + | --llm-model | name=openai-compatible-sample,uris=/v1/chat/completions\|/v1/embeddings,routingMethod=round_robin | + And I successfully deploy the function selected by NVCF CLI with options: + | option | value | + | --gpu | H100 | + | --instance-type | NCP.GPU.H100_1x | + | --backend | ncp-local-compute-1 | + | --regions | us-west-1 | + | --min-instances | 1 | + | --max-instances | 1 | + | --timeout | 900 | + And I successfully generate a function API key with CLI options: + | option | value | + | --description | bdd-registration-multiregion | + | --scopes | invoke_function,list_functions,queue_details,list_functions_details | + + When I run command: + """ + /bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); reverse=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 5 ] && [ "$reverse" -ge 3 ]; then printf "registration=%s reverse=%s regions=2\n" "$registration" "$reverse"; exit 0; fi; fi; sleep 5; done; exit 1' + """ + Then the command exit code should be 0 + And the command output should contain "registration=5" + And the command output should contain "regions=2" + + When I successfully invoke model "openai-compatible-sample" at "/v1/chat/completions" with timeout "120" seconds: + """ + {"messages":[{"role":"user","content":"bdd-registration-multiregion"}]} + """ + Then the command output should contain "chat.completion" + And the command output should contain "fixed 128-byte response" + And I successfully undeploy the function selected by NVCF CLI diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 970c26657..8c8849a46 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1206,6 +1206,91 @@ func TestMultiClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { assertFunctionDeploymentsUseInstanceType(t, suite.Runner.(*fakeRunner).runs, "NCP.GPU.H100_1x", 3) } +// TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps +// runs the secure recursive-discovery feature against a fake runner. The +// observations cover distinct Deployment and StatefulSet router identities, +// the HTTPS remote Watch URI, and Pylon's combined registration topology. +func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t *testing.T) { + t.Setenv("NGC_API_KEY", "test-key") + t.Setenv("SAMPLE_NGC_ORG", "test-org") + t.Setenv("SAMPLE_NGC_TEAM", "test-team") + t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") + t.Setenv("REPO_ROOT", "/repo-root-placeholder") + + const ( + regionAWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); pods=$(kubectl --context k3d-ncp-local-cp get pods -n nvcf -l app.kubernetes.io/instance=llm-request-router,app.kubernetes.io/name=llm-request-router -o jsonpath="{range .items[*]}{.metadata.name}{\"\\n\"}{end}"); count=0; while IFS= read -r pod; do [ -z "$pod" ] && continue; printf "%s" "$output" | grep -Fq "$pod"; count=$((count + 1)); done <<<"$pods"; [ "$count" -eq 3 ]; printf "%s" "$output" | grep -Fq "https://region-b-watch.nvcf.svc.cluster.local:50071"; printf "region-a-deployment=%s remote-watch=https\n" "$count"'` + regionBWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); printf "%s" "$output" | grep -Fq "llm-request-router-region-b-0"; printf "%s" "$output" | grep -Fq "llm-request-router-region-b-1"; printf "region-b-statefulset=2 tls=https\n"'` + pylonMetricsCommand = `/bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); reverse=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 5 ] && [ "$reverse" -ge 3 ]; then printf "registration=%s reverse=%s regions=2\n" "$registration" "$reverse"; exit 0; fi; fi; sleep 5; done; exit 1'` + invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" + + " --inference-url /v1/chat/completions --model-name openai-compatible-sample" + + " --request-body '{\"messages\":[{\"role\":\"user\",\"content\":\"bdd-registration-multiregion\"}]}' --timeout 120" + ) + + suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{ + "k3d cluster get ncp-local": {ExitCode: 1}, + regionAWatchCommand: { + ExitCode: 0, + Stdout: "region-a-deployment=3 remote-watch=https\n", + }, + regionBWatchCommand: { + ExitCode: 0, + Stdout: "region-b-statefulset=2 tls=https\n", + }, + pylonMetricsCommand: {ExitCode: 0, Stdout: "registration=5 reverse=3 regions=2\n"}, + invokeCommand: { + ExitCode: 0, + Stdout: "Function invocation completed!\n\nResponse:\n" + + `{"object":"chat.completion","choices":[{"message":{"content":"This is a fixed 128-byte response for routing and contract validation."}}]}` + + "\n", + }, + })) + seedHelmfileLocalBDDMultiFixture(t, suite.Config.RepoRoot) + seedComputePlaneLocalBDDMultiFixture(t, suite.Config.RepoRoot) + seedStackSecretsTemplate(t, suite.Config.RepoRoot) + writeProfileHandoffArtifact(t, suite.Config.RepoRoot) + writeMulticlusterComputeRegisterValues(t, suite.Config.RepoRoot, "nvcf-compute-plane", "ncp-local-compute-1") + writeArtifact( + t, + suite.Config.RepoRoot, + "self-managed", + "registration-multiregion-rendered.yaml", + "kind: Deployment\n"+ + "--remote-stargate-url=https://region-b-watch.nvcf.svc.cluster.local:50071\n", + ) + + sc := steps.NewScenarioContext(suite) + featurePath := mustResolveFeaturePath(t, "multi-cluster-helmfile-llm-registration-multiregion.feature") + var out strings.Builder + status := godog.TestSuite{ + Name: "multi-cluster-helmfile-llm-registration-multiregion-wiring", + ScenarioInitializer: func(ctx *godog.ScenarioContext) { + steps.RegisterAll(ctx, sc) + }, + Options: &godog.Options{ + Format: "pretty", + Paths: []string{featurePath}, + Strict: true, + Output: &out, + }, + }.Run() + if status != 0 { + t.Fatalf("godog suite status = %d\n%s", status, out.String()) + } + for _, command := range []string{regionAWatchCommand, regionBWatchCommand, pylonMetricsCommand} { + if !commandRanExactly(suite.Runner.(*fakeRunner).runs, command) { + t.Fatalf("exact multi-region observation command was not invoked: %s", command) + } + } + if !commandRanThatContainsAll( + suite.Runner.(*fakeRunner).runs, + "function create --name bdd-registration-multiregion", + "--function-type LLM", + "--llm-model", + ) { + t.Fatal("multi-region sample was not created as an LLM function") + } +} + // TestSingleClusterHelmfileUpstreamImagesFeatureFileWiresToSteps runs the // focused upstream-image feature against a fake runner. The seeded global // template contains the exact documentation blocks so the ledger-backed @@ -1971,6 +2056,16 @@ func TestMultiClusterHelmfile(t *testing.T) { runLiveFeature(t, "multi-cluster-helmfile.feature") } +// TestMultiClusterHelmfileLLMRegistrationMultiregion is the live entry point +// for secure recursive registration across two local logical regions. +// Skipped under -short. +func TestMultiClusterHelmfileLLMRegistrationMultiregion(t *testing.T) { + if testing.Short() { + t.Skip("live run skipped under -short") + } + runLiveFeature(t, "multi-cluster-helmfile-llm-registration-multiregion.feature") +} + // TestSingleClusterEKSHelmfile is the live entry point for the // single-cluster EKS Helmfile feature. Skipped under -short. func TestSingleClusterEKSHelmfile(t *testing.T) { diff --git a/tests/bdd/scripts/install-llm-region-b.sh b/tests/bdd/scripts/install-llm-region-b.sh new file mode 100755 index 000000000..9b1ae9931 --- /dev/null +++ b/tests/bdd/scripts/install-llm-region-b.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +control_context="${CONTROL_CONTEXT:-k3d-ncp-local-cp}" +compute_context="${COMPUTE_CONTEXT:-k3d-ncp-local-compute-1}" +namespace="nvcf" +region_b_release="llm-request-router-region-b" +region_b_watch_host="region-b-watch.nvcf.svc.cluster.local" +region_b_headless_host="*.llm-request-router-region-b-headless.nvcf.svc.cluster.local" +chart="${REPO_ROOT:?REPO_ROOT is required}/deploy/helm/llm-request-router/llm-request-router" + +values_json="$(helm --kube-context "${control_context}" get values llm-request-router \ + --namespace "${namespace}" --output json)" + +printf '%s' "${values_json}" | jq --arg watch_host "${region_b_watch_host}" ' + { + llmRequestRouter: ( + .llmRequestRouter + | .fullnameOverride = "llm-request-router-region-b" + | .replicaCount = 2 + | .workload.kind = "StatefulSet" + | .service.headlessName = "llm-request-router-region-b-headless" + | .kubernetes.advertisedHostnameTemplate = "{pod_name}.llm-request-router-region-b-headless.nvcf.svc.cluster.local" + | .discovery.remoteWatchUrls = [] + | .backendRouter.enabled = true + | .backendRouter.pylonGrpcDialAddress = ("https://" + $watch_host + ":50071") + | .backendRouter.pylonReverseTunnelDialAddress = ($watch_host + ":50072") + | .serviceAccount.create = false + | .serviceAccount.name = "llm-request-router" + | .pki.enabled = false + | .certificate.enabled = false + | .tls.mode = "existingSecret" + | .tls.secretName = "stargate-quic-tls" + | .image.pullPolicy = "IfNotPresent" + | .backendRouter.image.pullPolicy = "IfNotPresent" + ) + } +' | helm --kube-context "${control_context}" upgrade --install "${region_b_release}" "${chart}" \ + --namespace "${namespace}" --values - --wait --timeout 10m + +kubectl --context "${control_context}" apply -f - <&2 + exit 1 +fi + +kubectl --context "${compute_context}" apply -f - < Date: Fri, 28 Aug 2026 16:15:52 -0400 Subject: [PATCH 4/6] test(bdd): fix secure multi-region coverage (#1322) Signed-off-by: Mike Camp --- ...mfile-llm-registration-multiregion.feature | 8 +- tests/bdd/godog_test.go | 48 ++++++++- tests/bdd/region_b_script_test.go | 101 ++++++++++++++++++ tests/bdd/scripts/install-llm-region-b.sh | 4 +- 4 files changed, 153 insertions(+), 8 deletions(-) create mode 100644 tests/bdd/region_b_script_test.go diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature index 011c936f1..d06b58af0 100644 --- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature @@ -23,7 +23,8 @@ Feature: Register an LLM worker securely with routers in two local regions | addons.llm.requestRouter.discovery.remoteWatchUrls[0] | https://region-b-watch.nvcf.svc.cluster.local:50071 | | addons.llm.requestRouter.grpcTls.dnsNames[1] | region-b-watch.nvcf.svc.cluster.local | | addons.llm.requestRouter.backendRouter.pylonGrpcDialAddress | https://llm-request-router.nvcf.svc.cluster.local:50071 | - | addons.llm.pki.dnsNames[2] | *.llm-request-router-region-b-headless.nvcf.svc.cluster.local | + | addons.llm.pki.dnsNames[2] | region-b-watch.nvcf.svc.cluster.local | + | addons.llm.pki.dnsNames[3] | *.llm-request-router-region-b-headless.nvcf.svc.cluster.local | | observability.profile | disabled | And I prepare Helmfile environment "local-bdd-registration-multiregion" for stack "nvcf-compute-plane" from fixture "tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml" with values: | global.imagePullSecrets[0].name | nvcr-pull-secret | @@ -63,6 +64,9 @@ Feature: Register an LLM worker securely with routers in two local regions Then the command exit code should be 0 When I run command "kubectl --context k3d-ncp-local-cp wait certificate llm-request-router-grpc-tls -n envoy-gateway-system --for=condition=Ready --timeout=5m" Then the command exit code should be 0 + When I run command "kubectl --context k3d-ncp-local-cp get certificate llm-request-router-grpc-tls -n envoy-gateway-system -o jsonpath={.spec.dnsNames}" + Then the command exit code should be 0 + And the command output should contain "region-b-watch.nvcf.svc.cluster.local" When I run command "kubectl --context k3d-ncp-local-cp rollout status deployment/llm-request-router -n nvcf --timeout=10m" Then the command exit code should be 0 @@ -82,7 +86,7 @@ Feature: Register an LLM worker securely with routers in two local regions # identities and never relies on a dashed-IP SRV alias. When I run command: """ - /bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); printf "%s" "$output" | grep -Fq "llm-request-router-region-b-0"; printf "%s" "$output" | grep -Fq "llm-request-router-region-b-1"; printf "region-b-statefulset=2 tls=https\n"' + /bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); identities=$(printf "%s\n" "$output" | grep -Eo "llm-request-router-region-b-[0-9]+" | sort -u || true); expected=$(printf "llm-request-router-region-b-0\nllm-request-router-region-b-1\n"); [ "$identities" = "$expected" ]; count=$(printf "%s\n" "$identities" | grep -c .); [ "$count" -eq 2 ]; ! printf "%s" "$output" | grep -Eq "([0-9]{1,3}-){3}[0-9]{1,3}\."; printf "region-b-statefulset=%s tls=https\n" "$count"' """ Then the command exit code should be 0 And the command output should contain "region-b-statefulset=2 tls=https" diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 8c8849a46..969ecd9c3 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1217,14 +1217,18 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") t.Setenv("REPO_ROOT", "/repo-root-placeholder") + //revive:disable:line-length-limit Exact feature commands must remain byte-for-byte identical. const ( - regionAWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); pods=$(kubectl --context k3d-ncp-local-cp get pods -n nvcf -l app.kubernetes.io/instance=llm-request-router,app.kubernetes.io/name=llm-request-router -o jsonpath="{range .items[*]}{.metadata.name}{\"\\n\"}{end}"); count=0; while IFS= read -r pod; do [ -z "$pod" ] && continue; printf "%s" "$output" | grep -Fq "$pod"; count=$((count + 1)); done <<<"$pods"; [ "$count" -eq 3 ]; printf "%s" "$output" | grep -Fq "https://region-b-watch.nvcf.svc.cluster.local:50071"; printf "region-a-deployment=%s remote-watch=https\n" "$count"'` - regionBWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); printf "%s" "$output" | grep -Fq "llm-request-router-region-b-0"; printf "%s" "$output" | grep -Fq "llm-request-router-region-b-1"; printf "region-b-statefulset=2 tls=https\n"'` - pylonMetricsCommand = `/bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); reverse=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 5 ] && [ "$reverse" -ge 3 ]; then printf "registration=%s reverse=%s regions=2\n" "$registration" "$reverse"; exit 0; fi; fi; sleep 5; done; exit 1'` - invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" + + regionAWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); pods=$(kubectl --context k3d-ncp-local-cp get pods -n nvcf -l app.kubernetes.io/instance=llm-request-router,app.kubernetes.io/name=llm-request-router -o jsonpath="{range .items[*]}{.metadata.name}{\"\\n\"}{end}"); count=0; while IFS= read -r pod; do [ -z "$pod" ] && continue; printf "%s" "$output" | grep -Fq "$pod"; count=$((count + 1)); done <<<"$pods"; [ "$count" -eq 3 ]; printf "%s" "$output" | grep -Fq "https://region-b-watch.nvcf.svc.cluster.local:50071"; printf "region-a-deployment=%s remote-watch=https\n" "$count"'` + regionBWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); identities=$(printf "%s\n" "$output" | grep -Eo "llm-request-router-region-b-[0-9]+" | sort -u || true); expected=$(printf "llm-request-router-region-b-0\nllm-request-router-region-b-1\n"); [ "$identities" = "$expected" ]; count=$(printf "%s\n" "$identities" | grep -c .); [ "$count" -eq 2 ]; ! printf "%s" "$output" | grep -Eq "([0-9]{1,3}-){3}[0-9]{1,3}\."; printf "region-b-statefulset=%s tls=https\n" "$count"'` + pylonMetricsCommand = `/bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); reverse=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 5 ] && [ "$reverse" -ge 3 ]; then printf "registration=%s reverse=%s regions=2\n" "$registration" "$reverse"; exit 0; fi; fi; sleep 5; done; exit 1'` + grpcCertificateCommand = "kubectl --context k3d-ncp-local-cp get certificate llm-request-router-grpc-tls" + + " -n envoy-gateway-system -o jsonpath={.spec.dnsNames}" + invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" + " --inference-url /v1/chat/completions --model-name openai-compatible-sample" + " --request-body '{\"messages\":[{\"role\":\"user\",\"content\":\"bdd-registration-multiregion\"}]}' --timeout 120" ) + //revive:enable:line-length-limit suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{ "k3d cluster get ncp-local": {ExitCode: 1}, @@ -1236,6 +1240,10 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t ExitCode: 0, Stdout: "region-b-statefulset=2 tls=https\n", }, + grpcCertificateCommand: { + ExitCode: 0, + Stdout: "[llm-request-router.nvcf.svc.cluster.local region-b-watch.nvcf.svc.cluster.local]", + }, pylonMetricsCommand: {ExitCode: 0, Stdout: "registration=5 reverse=3 regions=2\n"}, invokeCommand: { ExitCode: 0, @@ -1276,7 +1284,12 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t if status != 0 { t.Fatalf("godog suite status = %d\n%s", status, out.String()) } - for _, command := range []string{regionAWatchCommand, regionBWatchCommand, pylonMetricsCommand} { + for _, command := range []string{ + grpcCertificateCommand, + regionAWatchCommand, + regionBWatchCommand, + pylonMetricsCommand, + } { if !commandRanExactly(suite.Runner.(*fakeRunner).runs, command) { t.Fatalf("exact multi-region observation command was not invoked: %s", command) } @@ -1289,6 +1302,31 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t ) { t.Fatal("multi-region sample was not created as an LLM function") } + + environmentPath, err := dsl.HelmfileEnvironmentPath( + suite.Config.RepoRoot, + "self-managed", + "local-bdd-registration-multiregion", + ) + if err != nil { + t.Fatalf("resolve multi-region environment: %v", err) + } + for _, expectation := range []struct { + key string + want string + }{ + {key: "addons.llm.requestRouter.grpcTls.dnsNames[1]", want: "region-b-watch.nvcf.svc.cluster.local"}, + {key: "addons.llm.pki.dnsNames[2]", want: "region-b-watch.nvcf.svc.cluster.local"}, + {key: "addons.llm.pki.dnsNames[3]", want: "*.llm-request-router-region-b-headless.nvcf.svc.cluster.local"}, + } { + got, found, readErr := dsl.ReadYAMLKey(environmentPath, expectation.key) + if readErr != nil { + t.Fatalf("read %s: %v", expectation.key, readErr) + } + if !found || got != expectation.want { + t.Fatalf("%s = %q, found %t, want %q", expectation.key, got, found, expectation.want) + } + } } // TestSingleClusterHelmfileUpstreamImagesFeatureFileWiresToSteps runs the diff --git a/tests/bdd/region_b_script_test.go b/tests/bdd/region_b_script_test.go new file mode 100644 index 000000000..23b6e3044 --- /dev/null +++ b/tests/bdd/region_b_script_test.go @@ -0,0 +1,101 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bdd_tmp + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestInstallLLMRegionBCreatesWatchAliasInBothClusters(t *testing.T) { + binDir := t.TempDir() + applyDir := t.TempDir() + + helmScript := `#!/usr/bin/env bash +set -euo pipefail +case " $* " in + *" get values "*) printf '{"llmRequestRouter":{}}\n' ;; + *) cat >/dev/null ;; +esac +` + kubectlScript := `#!/usr/bin/env bash +set -euo pipefail +context="" +previous="" +for argument in "$@"; do + if [[ "${previous}" == "--context" ]]; then + context="${argument}" + fi + previous="${argument}" +done +case " $* " in + *" get endpoints llm-request-router "*) printf '192.0.2.10' ;; + *" apply -f - "*) + cat >>"${FAKE_APPLY_DIR}/${context}.yaml" + printf '\n---\n' >>"${FAKE_APPLY_DIR}/${context}.yaml" + ;; +esac +` + jqScript := `#!/usr/bin/env bash +set -euo pipefail +cat +` + for name, body := range map[string]string{ + "helm": helmScript, + "jq": jqScript, + "kubectl": kubectlScript, + } { + if err := os.WriteFile(filepath.Join(binDir, name), []byte(body), 0o755); err != nil { + t.Fatalf("write fake %s: %v", name, err) + } + } + + cmd := exec.Command("bash", "scripts/install-llm-region-b.sh") + cmd.Env = append(os.Environ(), + "CONTROL_CONTEXT=bdd-control", + "COMPUTE_CONTEXT=bdd-compute", + "FAKE_APPLY_DIR="+applyDir, + "PATH="+binDir+":"+os.Getenv("PATH"), + "REPO_ROOT="+t.TempDir(), + ) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("install region B: %v\n%s", err, output) + } + + for _, context := range []string{"bdd-control", "bdd-compute"} { + manifestPath := filepath.Join(applyDir, context+".yaml") + manifest, err := os.ReadFile(manifestPath) + if err != nil { + t.Fatalf("read %s aliases: %v", context, err) + } + for _, want := range []string{ + "kind: Service\nmetadata:\n name: region-b-watch", + "kind: Endpoints\nmetadata:\n name: region-b-watch", + "- ip: 192.0.2.10", + "name: llm-grpc", + "name: llm-quic", + } { + if !strings.Contains(string(manifest), want) { + t.Fatalf("%s aliases missing %q:\n%s", context, want, manifest) + } + } + } +} diff --git a/tests/bdd/scripts/install-llm-region-b.sh b/tests/bdd/scripts/install-llm-region-b.sh index 9b1ae9931..26a7e03f0 100755 --- a/tests/bdd/scripts/install-llm-region-b.sh +++ b/tests/bdd/scripts/install-llm-region-b.sh @@ -98,7 +98,8 @@ if [[ -z "${control_plane_ip}" ]]; then exit 1 fi -kubectl --context "${compute_context}" apply -f - < Date: Sun, 30 Aug 2026 00:03:36 -0700 Subject: [PATCH 5/6] test(bdd): use shared Pylon metrics assertion Replace embedded Pylon polling with the shared topology assertion while leaving regional router identity checks explicit in the feature. Refs #1307 Refs #1358 Signed-off-by: Stephanie Baum --- ...elmfile-llm-registration-multiregion.feature | 17 ++++++++--------- tests/bdd/godog_test.go | 15 +++++++++++---- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature index d06b58af0..df2351907 100644 --- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature @@ -149,18 +149,17 @@ Feature: Register an LLM worker securely with routers in two local regions | --description | bdd-registration-multiregion | | --scopes | invoke_function,list_functions,queue_details,list_functions_details | - When I run command: - """ - /bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); reverse=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 5 ] && [ "$reverse" -ge 3 ]; then printf "registration=%s reverse=%s regions=2\n" "$registration" "$reverse"; exit 0; fi; fi; sleep 5; done; exit 1' - """ - Then the command exit code should be 0 - And the command output should contain "registration=5" - And the command output should contain "regions=2" + Then a pod containing container "llm-worker" using context "k3d-ncp-local-compute-1" should report Pylon metrics within "10m": + | metric | comparison | count | + | pylon_registration_stream_connected | exactly | 5 | + | pylon_reverse_tunnel_connected | at least | 3 | When I successfully invoke model "openai-compatible-sample" at "/v1/chat/completions" with timeout "120" seconds: """ {"messages":[{"role":"user","content":"bdd-registration-multiregion"}]} """ - Then the command output should contain "chat.completion" - And the command output should contain "fixed 128-byte response" + Then the command output should contain all: + | text | + | chat.completion | + | fixed 128-byte response | And I successfully undeploy the function selected by NVCF CLI diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 969ecd9c3..ea48da807 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1219,9 +1219,12 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t //revive:disable:line-length-limit Exact feature commands must remain byte-for-byte identical. const ( - regionAWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); pods=$(kubectl --context k3d-ncp-local-cp get pods -n nvcf -l app.kubernetes.io/instance=llm-request-router,app.kubernetes.io/name=llm-request-router -o jsonpath="{range .items[*]}{.metadata.name}{\"\\n\"}{end}"); count=0; while IFS= read -r pod; do [ -z "$pod" ] && continue; printf "%s" "$output" | grep -Fq "$pod"; count=$((count + 1)); done <<<"$pods"; [ "$count" -eq 3 ]; printf "%s" "$output" | grep -Fq "https://region-b-watch.nvcf.svc.cluster.local:50071"; printf "region-a-deployment=%s remote-watch=https\n" "$count"'` - regionBWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); identities=$(printf "%s\n" "$output" | grep -Eo "llm-request-router-region-b-[0-9]+" | sort -u || true); expected=$(printf "llm-request-router-region-b-0\nllm-request-router-region-b-1\n"); [ "$identities" = "$expected" ]; count=$(printf "%s\n" "$identities" | grep -c .); [ "$count" -eq 2 ]; ! printf "%s" "$output" | grep -Eq "([0-9]{1,3}-){3}[0-9]{1,3}\."; printf "region-b-statefulset=%s tls=https\n" "$count"'` - pylonMetricsCommand = `/bin/sh -c 'set -eu; for attempt in $(seq 1 120); do row=$(kubectl --context k3d-ncp-local-compute-1 get pods -A -o json | jq -r "[.items[] | select(any(.spec.containers[]?; .name == \"llm-worker\")) | [.metadata.namespace,.metadata.name] | @tsv] | first // empty"); if [ -n "$row" ]; then ns=$(printf "%s" "$row" | cut -f1); pod=$(printf "%s" "$row" | cut -f2); metrics=$(kubectl --context k3d-ncp-local-compute-1 get --raw "/api/v1/namespaces/$ns/pods/$pod:9089/proxy/metrics" 2>/dev/null || true); registration=$(printf "%s\n" "$metrics" | grep -c "^pylon_registration_stream_connected.* 1$" || true); reverse=$(printf "%s\n" "$metrics" | grep -c "^pylon_reverse_tunnel_connected.* 1$" || true); if [ "$registration" -eq 5 ] && [ "$reverse" -ge 3 ]; then printf "registration=%s reverse=%s regions=2\n" "$registration" "$reverse"; exit 0; fi; fi; sleep 5; done; exit 1'` + regionAWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); pods=$(kubectl --context k3d-ncp-local-cp get pods -n nvcf -l app.kubernetes.io/instance=llm-request-router,app.kubernetes.io/name=llm-request-router -o jsonpath="{range .items[*]}{.metadata.name}{\"\\n\"}{end}"); count=0; while IFS= read -r pod; do [ -z "$pod" ] && continue; printf "%s" "$output" | grep -Fq "$pod"; count=$((count + 1)); done <<<"$pods"; [ "$count" -eq 3 ]; printf "%s" "$output" | grep -Fq "https://region-b-watch.nvcf.svc.cluster.local:50071"; printf "region-a-deployment=%s remote-watch=https\n" "$count"'` + regionBWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); identities=$(printf "%s\n" "$output" | grep -Eo "llm-request-router-region-b-[0-9]+" | sort -u || true); expected=$(printf "llm-request-router-region-b-0\nllm-request-router-region-b-1\n"); [ "$identities" = "$expected" ]; count=$(printf "%s\n" "$identities" | grep -c .); [ "$count" -eq 2 ]; ! printf "%s" "$output" | grep -Eq "([0-9]{1,3}-){3}[0-9]{1,3}\."; printf "region-b-statefulset=%s tls=https\n" "$count"'` + pylonMetricsCommand = "bash tests/bdd/scripts/wait-pylon-metrics.sh" + + " llm-worker k3d-ncp-local-compute-1 10m" + + " pylon_registration_stream_connected exactly 5" + + " pylon_reverse_tunnel_connected 'at least' 3" grpcCertificateCommand = "kubectl --context k3d-ncp-local-cp get certificate llm-request-router-grpc-tls" + " -n envoy-gateway-system -o jsonpath={.spec.dnsNames}" invokeCommand = "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" + @@ -1244,7 +1247,11 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t ExitCode: 0, Stdout: "[llm-request-router.nvcf.svc.cluster.local region-b-watch.nvcf.svc.cluster.local]", }, - pylonMetricsCommand: {ExitCode: 0, Stdout: "registration=5 reverse=3 regions=2\n"}, + pylonMetricsCommand: { + ExitCode: 0, + Stdout: "pylon_registration_stream_connected=5\n" + + "pylon_reverse_tunnel_connected=3\n", + }, invokeCommand: { ExitCode: 0, Stdout: "Function invocation completed!\n\nResponse:\n" + From 34b2e392f0a846f109aa683f57246272495827f3 Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Sun, 30 Aug 2026 00:42:07 -0700 Subject: [PATCH 6/6] test(bdd): target multi-region Pylons by function Refs #1292 Signed-off-by: Stephanie Baum --- .../multi-cluster-helmfile-llm-registration-multiregion.feature | 2 +- tests/bdd/godog_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature b/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature index df2351907..7ac7dadb3 100644 --- a/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature +++ b/tests/bdd/features/multi-cluster-helmfile-llm-registration-multiregion.feature @@ -149,7 +149,7 @@ Feature: Register an LLM worker securely with routers in two local regions | --description | bdd-registration-multiregion | | --scopes | invoke_function,list_functions,queue_details,list_functions_details | - Then a pod containing container "llm-worker" using context "k3d-ncp-local-compute-1" should report Pylon metrics within "10m": + Then every Pylon for function "bdd-registration-multiregion" using container "llm-worker" and context "k3d-ncp-local-compute-1" should report metrics within "10m": | metric | comparison | count | | pylon_registration_stream_connected | exactly | 5 | | pylon_reverse_tunnel_connected | at least | 3 | diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index ea48da807..0d207012c 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -1222,7 +1222,7 @@ func TestMultiClusterHelmfileLLMRegistrationMultiregionFeatureFileWiresToSteps(t regionAWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); pods=$(kubectl --context k3d-ncp-local-cp get pods -n nvcf -l app.kubernetes.io/instance=llm-request-router,app.kubernetes.io/name=llm-request-router -o jsonpath="{range .items[*]}{.metadata.name}{\"\\n\"}{end}"); count=0; while IFS= read -r pod; do [ -z "$pod" ] && continue; printf "%s" "$output" | grep -Fq "$pod"; count=$((count + 1)); done <<<"$pods"; [ "$count" -eq 3 ]; printf "%s" "$output" | grep -Fq "https://region-b-watch.nvcf.svc.cluster.local:50071"; printf "region-a-deployment=%s remote-watch=https\n" "$count"'` regionBWatchCommand = `/bin/bash -c 'set -eu; output=$(grpcurl -max-time 3 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority region-b-watch.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1 || true); identities=$(printf "%s\n" "$output" | grep -Eo "llm-request-router-region-b-[0-9]+" | sort -u || true); expected=$(printf "llm-request-router-region-b-0\nllm-request-router-region-b-1\n"); [ "$identities" = "$expected" ]; count=$(printf "%s\n" "$identities" | grep -c .); [ "$count" -eq 2 ]; ! printf "%s" "$output" | grep -Eq "([0-9]{1,3}-){3}[0-9]{1,3}\."; printf "region-b-statefulset=%s tls=https\n" "$count"'` pylonMetricsCommand = "bash tests/bdd/scripts/wait-pylon-metrics.sh" + - " llm-worker k3d-ncp-local-compute-1 10m" + + " bdd-registration-multiregion llm-worker k3d-ncp-local-compute-1 10m" + " pylon_registration_stream_connected exactly 5" + " pylon_reverse_tunnel_connected 'at least' 3" grpcCertificateCommand = "kubectl --context k3d-ncp-local-cp get certificate llm-request-router-grpc-tls" +