From 71e472f4b61913d9971ce23369ae6d186b36c872 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 17 Aug 2026 21:30:08 +0000 Subject: [PATCH 01/26] charts: expose the router at its node IP, GKE-style router.hostPort binds 4501 TCP+UDP on the pod's node and derives the announced multiaddrs from status.hostIP via the downward API, for TCP and QUIC both. The router appends its own PeerID when leasing, so clients get dialable addresses. --allow-loopback is now gated on router.allowLoopback instead of always passed, and the Service drops its TCP-only nodePort block. router.nodeSelector (empty by default) lets the router be pinned away from sam-node workers: with hostPort set the router announces its own node's IP, and a sam-node scheduled onto that same node cannot reach the hostPort in kind. --- charts/sam-mesh/templates/router-service.yaml | 3 --- .../sam-mesh/templates/router-statefulset.yaml | 16 ++++++++++++++++ charts/sam-mesh/values.yaml | 8 +++++++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/charts/sam-mesh/templates/router-service.yaml b/charts/sam-mesh/templates/router-service.yaml index 0735681b..708fc098 100644 --- a/charts/sam-mesh/templates/router-service.yaml +++ b/charts/sam-mesh/templates/router-service.yaml @@ -11,9 +11,6 @@ spec: - name: p2p-tcp port: {{ .Values.router.service.port }} targetPort: 4501 - {{- if and (eq .Values.router.service.type "NodePort") .Values.router.service.nodePort }} - nodePort: {{ .Values.router.service.nodePort }} - {{- end }} protocol: TCP - name: p2p-udp port: {{ .Values.router.service.port }} diff --git a/charts/sam-mesh/templates/router-statefulset.yaml b/charts/sam-mesh/templates/router-statefulset.yaml index 94def123..f07b0fd4 100644 --- a/charts/sam-mesh/templates/router-statefulset.yaml +++ b/charts/sam-mesh/templates/router-statefulset.yaml @@ -24,6 +24,10 @@ spec: app: {{ include "sam-mesh.fullname" . }}-router spec: serviceAccountName: {{ include "sam-mesh.fullname" . }}-router-sa + {{- with .Values.router.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.router.podSecurityContext }} securityContext: {{- toYaml . | nindent 8 }} @@ -76,6 +80,13 @@ spec: name: {{ include "sam-mesh.fullname" . }}-router-token key: token {{- end }} + {{- if .Values.router.hostPort }} + # kubelet expands $(HOST_IP) in the args below. + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- end }} args: - "--control-plane=http://{{ include "sam-mesh.fullname" . }}-control-plane:{{ .Values.controlPlane.service.port }}" - "--listen=/ip4/0.0.0.0/tcp/4501" @@ -84,11 +95,16 @@ spec: {{- range .Values.router.externalAddrs }} - "--external-addr={{ . }}" {{- end }} + {{- else if .Values.router.hostPort }} + - "--external-addr=/ip4/$(HOST_IP)/tcp/{{ .Values.router.hostPort }}" + - "--external-addr=/ip4/$(HOST_IP)/udp/{{ .Values.router.hostPort }}/quic-v1" {{- else }} - "--external-addr=/dns4/{{ include "sam-mesh.fullname" . }}-router/tcp/4501" {{- end }} - "--keys-path=/data/router.key" + {{- if .Values.router.allowLoopback }} - "--allow-loopback" + {{- end }} - "--log-level={{ .Values.router.logLevel }}" {{- if .Values.router.useOidcToken }} - "--jwt-path=/var/run/secrets/tokens/sam-token" diff --git a/charts/sam-mesh/values.yaml b/charts/sam-mesh/values.yaml index 96386274..dc54b6a2 100644 --- a/charts/sam-mesh/values.yaml +++ b/charts/sam-mesh/values.yaml @@ -61,6 +61,13 @@ router: repository: sam-router useOidcToken: false logLevel: info + # Binds 4501 TCP+UDP on the router's own node and announces /ip4//... for both + # transports. One router per node. + hostPort: null + nodeSelector: {} + # Overrides the announced multiaddrs; derived from hostPort when left empty. + externalAddrs: [] + allowLoopback: false resources: requests: cpu: 100m @@ -71,7 +78,6 @@ router: service: type: ClusterIP port: 4501 - nodePort: null console: enabled: true From e9a7af49ef901bcb3fec1dd5d58a1c222c9af38d Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 17 Aug 2026 21:30:08 +0000 Subject: [PATCH 02/26] charts: make sam:role:node's allowed services configurable The hardcoded five-entry list came from the e2e test fixtures and was already stale for every service under development/examples/ except calc-mcp. Expose it as bootstrap.nodeServices, defaulting to ["*"]. Two workarounds it forced are now gone: the sam-admin binding that enumerated per-node kind ServiceAccounts (local-node-sa never existed), and the local allow policies in everything-mcp's node config. --- charts/sam-mesh/templates/bootstrap-job.yaml | 5 ++--- charts/sam-mesh/values.yaml | 2 ++ development/examples/calc-mcp/sam-node-config.yaml | 4 +--- development/examples/everything-mcp/sam-node-config.yaml | 4 +--- development/examples/greeter-mcp/sam-node-config.yaml | 2 +- 5 files changed, 7 insertions(+), 10 deletions(-) diff --git a/charts/sam-mesh/templates/bootstrap-job.yaml b/charts/sam-mesh/templates/bootstrap-job.yaml index b49f3238..e93e06ad 100644 --- a/charts/sam-mesh/templates/bootstrap-job.yaml +++ b/charts/sam-mesh/templates/bootstrap-job.yaml @@ -81,9 +81,8 @@ spec: {{- $bindings := .Values.bootstrap.bindings -}} {{- if not $bindings -}} {{- $defaultRouterSA := printf "user:system:serviceaccount:%s:%s-router-sa" .Release.Namespace (include "sam-mesh.fullname" .) -}} - {{- $bindings = list + {{- $bindings = list (dict "role" "sam:role:router" "members" (list "group:routers" $defaultRouterSA)) - (dict "role" "sam-admin" "members" (list (printf "user:system:serviceaccount:%s:node-a-sa" .Release.Namespace) (printf "user:system:serviceaccount:%s:node-b-sa" .Release.Namespace) (printf "user:system:serviceaccount:%s:node-c-sa" .Release.Namespace) (printf "user:system:serviceaccount:%s:local-node-sa" .Release.Namespace))) (dict "role" "sam:role:sambox" "members" (list (printf "user:system:serviceaccount:%s:sam-box-sa" .Release.Namespace))) (dict "role" "sam:role:node" "members" (list "group:data-scientist" "group:users")) -}} @@ -96,7 +95,7 @@ spec: {"name": "sam-admin", "allowed_services": ["*"], "allowed_targets": ["*"]}, {"name": "sam:role:sambox", "allowed_services": ["*"], "allowed_targets": ["*"]}, {"name": "sam:role:router", "allowed_services": ["*"], "allowed_targets": ["*"]}, - {"name": "sam:role:node", "allowed_services": ["mcp://calculator", "mcp://db-agent", "mcp://http-tool", "mcp://stdio-tool", "system://sam.catalog"], "allowed_targets": ["*"]} + {"name": "sam:role:node", "allowed_services": {{ toJson .Values.bootstrap.nodeServices }}, "allowed_targets": ["*"]} ], "bindings": {{ toJson $bindings }} }' \ diff --git a/charts/sam-mesh/values.yaml b/charts/sam-mesh/values.yaml index dc54b6a2..2b0c2884 100644 --- a/charts/sam-mesh/values.yaml +++ b/charts/sam-mesh/values.yaml @@ -112,4 +112,6 @@ dex: bootstrap: bindings: [] + # Services sam:role:node may call + nodeServices: ["*"] diff --git a/development/examples/calc-mcp/sam-node-config.yaml b/development/examples/calc-mcp/sam-node-config.yaml index fad29c57..1abd60e8 100644 --- a/development/examples/calc-mcp/sam-node-config.yaml +++ b/development/examples/calc-mcp/sam-node-config.yaml @@ -1,8 +1,6 @@ version: "v1alpha1" attenuation: - # Allow cross-node MCP only when the caller's biscuit carries a - # node($peer) fact — i.e. it was issued by our control plane upon enrollment. - policies: + policies: [] services: - type: "mcp" name: "calculator" diff --git a/development/examples/everything-mcp/sam-node-config.yaml b/development/examples/everything-mcp/sam-node-config.yaml index af7e33f7..3107e5df 100644 --- a/development/examples/everything-mcp/sam-node-config.yaml +++ b/development/examples/everything-mcp/sam-node-config.yaml @@ -1,8 +1,6 @@ version: "v1alpha1" attenuation: - policies: - - 'allow if service("mcp", "everything");' - - 'allow if service("system", "sam.catalog");' + policies: [] services: - type: "mcp" name: "everything" diff --git a/development/examples/greeter-mcp/sam-node-config.yaml b/development/examples/greeter-mcp/sam-node-config.yaml index 4b52f7b2..12722585 100644 --- a/development/examples/greeter-mcp/sam-node-config.yaml +++ b/development/examples/greeter-mcp/sam-node-config.yaml @@ -1,6 +1,6 @@ version: "v1alpha1" attenuation: - policies: + policies: [] services: - type: "mcp" name: "greeter" From f2b76a0fd585a7b50d86b9bdf0aa5394ac6a2692 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 17 Aug 2026 21:30:24 +0000 Subject: [PATCH 03/26] charts: front the mesh with Gateway API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One Gateway and one HTTPRoute behind gateway.enabled: the cloud's 8 exact control-plane paths, a separately-gated dev /admin rule, and the console under gateway.consolePath shaped exactly like the cloud deployment — the bare prefix 302s to / and a URLRewrite filter strips it before the console. URLRewrite is Extended conformance, so the provider must support it. listeners, hostnames, addresses and annotations pass through to the Gateway API objects verbatim, so TLS, static addresses and provider annotations are all expressible instead of hardcoded. gateway.className has no default and is required when enabled: the class is provider-specific, so a wrong default is worse than a clear error. --- charts/sam-mesh/templates/gateway.yaml | 102 +++++++++++++++++++++++++ charts/sam-mesh/values.yaml | 27 +++++++ 2 files changed, 129 insertions(+) create mode 100644 charts/sam-mesh/templates/gateway.yaml diff --git a/charts/sam-mesh/templates/gateway.yaml b/charts/sam-mesh/templates/gateway.yaml new file mode 100644 index 00000000..8d6105e3 --- /dev/null +++ b/charts/sam-mesh/templates/gateway.yaml @@ -0,0 +1,102 @@ +{{- if .Values.gateway.enabled -}} +{{- $fullName := include "sam-mesh.fullname" . -}} +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: {{ $fullName }}-gateway + labels: + {{- include "sam-mesh.labels" . | nindent 4 }} + {{- with .Values.gateway.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + gatewayClassName: {{ required "gateway.className is required when gateway.enabled is true" .Values.gateway.className }} + {{- with .Values.gateway.addresses }} + addresses: + {{- toYaml . | nindent 2 }} + {{- end }} + listeners: + {{- toYaml .Values.gateway.listeners | nindent 2 }} +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ $fullName }}-route + labels: + {{- include "sam-mesh.labels" . | nindent 4 }} +spec: + parentRefs: + - name: {{ $fullName }}-gateway + {{- with .Values.gateway.hostnames }} + hostnames: + {{- toYaml . | nindent 2 }} + {{- end }} + rules: + # The enrollment surface, matching the cloud's allow-list exactly. + - matches: + - path: + type: Exact + value: /register + - path: + type: Exact + value: /info + - path: + type: Exact + value: /keys + - path: + type: Exact + value: /routers/lease + - path: + type: Exact + value: /policies + - path: + type: Exact + value: /enroll + - path: + type: Exact + value: /enroll/status + - path: + type: Exact + value: /refresh + backendRefs: + - name: {{ $fullName }}-control-plane + port: {{ .Values.controlPlane.service.port }} + {{- if .Values.gateway.adminRoute }} + - matches: + - path: + type: PathPrefix + value: /admin + backendRefs: + - name: {{ $fullName }}-control-plane + port: {{ .Values.controlPlane.service.port }} + {{- end }} + {{- if and .Values.console.enabled .Values.gateway.consolePath }} + # The cloud deployment's console shape: the bare prefix 302s to /, and + # URLRewrite (Extended conformance) strips it before the console sees the request. + - matches: + - path: + type: Exact + value: {{ .Values.gateway.consolePath }} + filters: + - type: RequestRedirect + requestRedirect: + path: + type: ReplaceFullPath + replaceFullPath: {{ .Values.gateway.consolePath }}/ + statusCode: 302 + - matches: + - path: + type: PathPrefix + value: {{ .Values.gateway.consolePath }}/ + filters: + - type: URLRewrite + urlRewrite: + path: + type: ReplacePrefixMatch + replacePrefixMatch: / + backendRefs: + - name: {{ $fullName }}-console + port: {{ .Values.console.service.port }} + {{- end }} +{{- end }} diff --git a/charts/sam-mesh/values.yaml b/charts/sam-mesh/values.yaml index 2b0c2884..af1fd446 100644 --- a/charts/sam-mesh/values.yaml +++ b/charts/sam-mesh/values.yaml @@ -2,6 +2,33 @@ global: imageTag: local imagePullPolicy: IfNotPresent +# One Gateway fronting the mesh: the control plane's enrollment surface plus the console +# under gateway.consolePath, mirroring the cloud deployment. +gateway: + enabled: false + # Required when enabled. No default: the GatewayClass is provider-specific, e.g. + # cloud-provider-kind in kind, gke-l7-global-external-managed on GKE. + className: "" + # Routes the control plane's /admin API. Dev only; the cloud leaves it unrouted. + adminRoute: false + # External prefix the console is served under. The bare prefix 302s to / and a + # URLRewrite filter (Extended conformance) strips it before the console sees the + # request, as on the cloud. Empty leaves the console unrouted. + consolePath: /console + # listeners, hostnames, addresses and annotations pass through to the Gateway API + # objects verbatim, so anything the spec allows is expressible. Plain HTTP on 80 + # matching any host suits a local cluster. + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: Same + hostnames: [] + addresses: [] + annotations: {} + controlPlane: replicaCount: 2 image: From 0bffff7b4b110f6e30d0a0eb4e6780467c9c9d15 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 17 Aug 2026 21:30:24 +0000 Subject: [PATCH 04/26] charts: drop the bundled Dex Dex is an independent component; bundling it coupled the chart to one dev IdP's config shape. The chart now ships no identity provider: controlPlane.oidcIssuer loses its dex default and must be set, and the kind lane deploys its own throwaway Dex from development/kind/dex.yaml instead. --- charts/sam-mesh/README.md | 70 +++++++++++++-- charts/sam-mesh/templates/dex-deployment.yaml | 86 ------------------- charts/sam-mesh/values.yaml | 19 +--- 3 files changed, 64 insertions(+), 111 deletions(-) delete mode 100644 charts/sam-mesh/templates/dex-deployment.yaml diff --git a/charts/sam-mesh/README.md b/charts/sam-mesh/README.md index e271237b..4399b93e 100644 --- a/charts/sam-mesh/README.md +++ b/charts/sam-mesh/README.md @@ -1,8 +1,8 @@ # sam-mesh Helm chart -Deploys a self-contained SAM mesh (control plane, router, console, and -optionally an in-cluster Postgres + Dex) for local development, testing, or -self-hosting your own hub. +Deploys a self-contained SAM mesh (control plane, router, console, and an +in-cluster Postgres) for local development, testing, or self-hosting your +own hub. > For large-scale production deployments (GKE/EKS/AKS) using externally > managed Postgres/DNS/OIDC, see the @@ -12,9 +12,17 @@ self-hosting your own hub. ## Install ```bash -helm upgrade --install sam-mesh ./charts/sam-mesh --namespace sam --create-namespace +helm upgrade --install sam-mesh ./charts/sam-mesh --namespace sam --create-namespace \ + --set controlPlane.oidcIssuer= ``` +`controlPlane.oidcIssuer` is required: the chart bundles no identity +provider, and the control plane refuses to start without an issuer. Point it +at your own OIDC provider (Google, Okta, a Dex you run, the cluster's own +issuer for ServiceAccount Workload Identity Federation, …). The `kind` dev +environment (`make kind-up`) deploys its own throwaway Dex from +`development/kind/dex.yaml` and wires it in for you. + At the end of `helm install`/`helm upgrade`, the chart prints the exact `kubectl` command to retrieve your generated secrets (see below) — read the NOTES output before doing anything else. @@ -59,9 +67,53 @@ administrator to approve each enrollment via `/admin/enrollments` before a node can join — see the [Control Plane Configuration guide](https://sam-mesh.dev/docs/user/control-plane-configuration/#6-headless-node-enrollment-bootstrap-token-flow). -## Dex (`dex.enabled`) +## Gateway API (`gateway.enabled`) + +Disabled by default. When enabled the chart creates one `Gateway` fronting +the mesh, with one `HTTPRoute`. `gateway.className` is then **required**, +with no default, because the right GatewayClass is provider-specific +(`cloud-provider-kind` in kind, `gke-l7-global-external-managed` on GKE, +`istio`, `envoy-gateway`, …). + +The route exposes only the control plane's enrollment surface (`/register`, +`/info`, `/keys`, `/routers/lease`, `/policies`, `/enroll`, `/enroll/status`, +`/refresh`) and the console under `gateway.consolePath`; everything else, +including `/admin` and `/user`, is unrouted. `gateway.adminRoute: true` +additionally routes `/admin` — a dev convenience, leave it off in production. + +The console rules mirror the cloud deployment exactly: the bare prefix +(`/console`) is answered with a 302 to `/console/`, and a `URLRewrite` filter +strips the prefix before the request reaches the console. `URLRewrite` is +**Extended** (not core) Gateway API conformance, so the provider must support +it. Set `gateway.consolePath: ""` to leave the console unrouted. + +`listeners`, `hostnames`, `addresses` and `annotations` are passed through to +the Gateway API objects verbatim, so anything the spec allows is expressible. +They default to one plain-HTTP listener on port 80 matching every host, which +suits a local cluster. For example, on GKE: + +```yaml +gateway: + enabled: true + className: gke-l7-global-external-managed + listeners: + - name: https + protocol: HTTPS + port: 443 + tls: + certificateRefs: + - name: sam-mesh-tls + allowedRoutes: + namespaces: + from: Same + hostnames: [hub.example.com] + addresses: + - type: NamedAddress + value: sam-hub-ip +``` + +## OIDC login for the console -Disabled by default. The bundled Dex is only meant for local/dev OIDC login -(username/password test users); real deployments should point -`controlPlane.oidcIssuer` at your own identity provider instead of enabling -this. +There is no bundled Dex. Point `controlPlane.oidcIssuer` at your identity +provider and register `https:///auth/callback` as +a redirect URI for the OIDC client the control plane reports. diff --git a/charts/sam-mesh/templates/dex-deployment.yaml b/charts/sam-mesh/templates/dex-deployment.yaml deleted file mode 100644 index 8f6a65a7..00000000 --- a/charts/sam-mesh/templates/dex-deployment.yaml +++ /dev/null @@ -1,86 +0,0 @@ -{{- if .Values.dex.enabled -}} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "sam-mesh.fullname" . }}-dex-config - labels: - {{- include "sam-mesh.labels" . | nindent 4 }} -data: - config.yaml: | - issuer: http://{{ include "sam-mesh.fullname" . }}-dex:5556/dex - storage: - type: memory - web: - http: 0.0.0.0:5556 - oauth2: - skipApprovalScreen: true - staticClients: - - id: sam-console - redirectURIs: - - http://localhost:9092/auth/callback - - http://127.0.0.1:9092/auth/callback - name: 'SAM Console' - public: true - enablePasswordDB: true - staticPasswords: - - email: admin@example.com - # bcrypt hash for 'password' - hash: "$2a$10$TUOgJVdtu0AL6Y6TcNBQXOSjRWRKhIkC0kXDzN9kOkPB4hg2aFPE2" - username: admin - userID: "0898fa90-3453-4311-8566-020953250212" ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "sam-mesh.fullname" . }}-dex - labels: - {{- include "sam-mesh.labels" . | nindent 4 }} -spec: - replicas: 1 - selector: - matchLabels: - app: {{ include "sam-mesh.fullname" . }}-dex - template: - metadata: - labels: - app: {{ include "sam-mesh.fullname" . }}-dex - spec: - nodeSelector: - sam-role: control-plane - containers: - - name: dex - image: "{{ .Values.dex.image.repository }}:{{ .Values.dex.image.tag }}" - imagePullPolicy: {{ .Values.global.imagePullPolicy }} - args: - - dex - - serve - - /etc/dex/cfg/config.yaml - ports: - - containerPort: 5556 - name: http - volumeMounts: - - name: config - mountPath: /etc/dex/cfg - volumes: - - name: config - configMap: - name: {{ include "sam-mesh.fullname" . }}-dex-config ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "sam-mesh.fullname" . }}-dex - labels: - {{- include "sam-mesh.labels" . | nindent 4 }} -spec: - type: NodePort - selector: - app: {{ include "sam-mesh.fullname" . }}-dex - ports: - - name: http - port: 5556 - targetPort: 5556 - {{- if .Values.dex.nodePort }} - nodePort: {{ .Values.dex.nodePort }} - {{- end }} -{{- end }} diff --git a/charts/sam-mesh/values.yaml b/charts/sam-mesh/values.yaml index af1fd446..33e6217e 100644 --- a/charts/sam-mesh/values.yaml +++ b/charts/sam-mesh/values.yaml @@ -41,7 +41,9 @@ controlPlane: # Only skips TLS verification for the OIDC issuer; leave false unless # pointing at a local/dev issuer without a valid certificate. insecureSkipTlsVerify: false - oidcIssuer: "http://sam-mesh-dex:5556/dex" + # Required: the chart bundles no identity provider. Point it at your OIDC issuer + # (comma-separated for more than one); the control plane refuses to start without one. + oidcIssuer: "" allowedAudiences: "sam-mesh-audience,sam-control-plane-audience" resources: requests: @@ -123,22 +125,7 @@ console: port: 8081 nodePort: null -dex: - enabled: false - nodePort: 30556 - image: - repository: dexidp/dex - tag: v2.39.0 - google: - clientId: "" - clientSecret: "" - github: - clientId: "" - clientSecret: "" - cliOAuthSecret: "" - bootstrap: bindings: [] # Services sam:role:node may call nodeServices: ["*"] - From aade64fec92b50947ea9267e29a77609c440ecf6 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 17 Aug 2026 21:30:39 +0000 Subject: [PATCH 05/26] development/kind: deploy the mesh behind the gateway surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run.sh now fronts the mesh with cloud-provider-kind: the control plane and console share one gateway address, Dex gets its own, and the router is reached at its node IP — no NodePorts, no extraPortMappings. Dex is deployed from dex.yaml in two passes, since its issuer and the console callback URL are the gateway addresses that only exist once the gateways do; the console is restarted after the final control-plane rollout because it reads the issuer from /info once, at startup. The local node enrolls with a bootstrap token over the gateway, the way a real external node joins. The cluster shrinks to two sam-node workers; add labelled workers to kind-config.yaml to host more examples at once. make kind-down delegates to run.sh so cloud-provider-kind and its envoy containers are torn down in dependency order. --- Makefile | 2 +- development/kind/dex.yaml | 107 ++++++++++++++ development/kind/kind-config.yaml | 24 --- development/kind/mesh-config.e2e.yaml | 3 - development/kind/mesh-config.yaml | 7 +- development/kind/run-local-node.sh | 38 +++-- development/kind/run.sh | 202 ++++++++++++++++++++------ 7 files changed, 292 insertions(+), 91 deletions(-) create mode 100644 development/kind/dex.yaml diff --git a/Makefile b/Makefile index 58fc5b4c..f5c6a237 100644 --- a/Makefile +++ b/Makefile @@ -118,7 +118,7 @@ kind-logs: ./development/kind/run.sh -l kind-down: - kind delete cluster --name sam-kind + ./development/kind/run.sh -d .PHONY: kind-local-node kind-local-node: diff --git a/development/kind/dex.yaml b/development/kind/dex.yaml new file mode 100644 index 00000000..b068d31b --- /dev/null +++ b/development/kind/dex.yaml @@ -0,0 +1,107 @@ +# Dev-only Dex for the kind mesh. Dex is an independent component, so it lives here, not +# in the Helm chart. run.sh applies this with envsubst, filling ${DEX_ISSUER} and +# ${CONSOLE_REDIRECT_URI} once the gateway addresses exist. +apiVersion: v1 +kind: ConfigMap +metadata: + name: sam-mesh-dex-config +data: + config.yaml: | + issuer: ${DEX_ISSUER} + storage: + type: memory + web: + http: 0.0.0.0:5556 + oauth2: + skipApprovalScreen: true + staticClients: + - id: sam-console + redirectURIs: + - ${CONSOLE_REDIRECT_URI} + name: 'SAM Console' + public: true + enablePasswordDB: true + staticPasswords: + # The control plane auto-registers a first-time OIDC login with the "user" role, so + # this account is not an admin however it is named. + - email: user@example.com + # bcrypt hash for 'password' + hash: "$2a$10$TUOgJVdtu0AL6Y6TcNBQXOSjRWRKhIkC0kXDzN9kOkPB4hg2aFPE2" + username: user + userID: "0898fa90-3453-4311-8566-020953250212" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sam-mesh-dex +spec: + replicas: 1 + selector: + matchLabels: + app: sam-mesh-dex + template: + metadata: + labels: + app: sam-mesh-dex + spec: + nodeSelector: + sam-role: control-plane + containers: + - name: dex + image: dexidp/dex:v2.39.0 + args: + - dex + - serve + - /etc/dex/cfg/config.yaml + ports: + - containerPort: 5556 + name: http + volumeMounts: + - name: config + mountPath: /etc/dex/cfg + volumes: + - name: config + configMap: + name: sam-mesh-dex-config +--- +apiVersion: v1 +kind: Service +metadata: + name: sam-mesh-dex +spec: + selector: + app: sam-mesh-dex + ports: + - name: http + port: 5556 + targetPort: 5556 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: sam-mesh-dex-gateway +spec: + gatewayClassName: cloud-provider-kind + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: Same +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: sam-mesh-dex-route +spec: + parentRefs: + - name: sam-mesh-dex-gateway + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: sam-mesh-dex + port: 5556 diff --git a/development/kind/kind-config.yaml b/development/kind/kind-config.yaml index 58f08211..80906475 100644 --- a/development/kind/kind-config.yaml +++ b/development/kind/kind-config.yaml @@ -2,21 +2,6 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - # Expose the control plane and router to the host (via the sam-control-plane-nodeport NodePort) so a - # locally-built sam-node can enroll: host:9090->HTTP, host:4001->libp2p TCP. - extraPortMappings: - - containerPort: 30090 - hostPort: 9090 - protocol: TCP - - containerPort: 30401 - hostPort: 4001 - protocol: TCP - - containerPort: 30081 - hostPort: 9092 - protocol: TCP - - containerPort: 30556 - hostPort: 5556 - protocol: TCP - role: worker labels: sam-role: control-plane @@ -26,12 +11,3 @@ nodes: - role: worker labels: sam-role: node-b - - role: worker - labels: - sam-role: node-c - - role: worker - labels: - sam-role: node-d - - role: worker - labels: - sam-role: node-e diff --git a/development/kind/mesh-config.e2e.yaml b/development/kind/mesh-config.e2e.yaml index 3f8bf904..86753d6c 100644 --- a/development/kind/mesh-config.e2e.yaml +++ b/development/kind/mesh-config.e2e.yaml @@ -2,6 +2,3 @@ # mcp://calculator/add. Select it with MESH_CONFIG=development/kind/mesh-config.e2e.yaml. node-a: node-b: calc-mcp -node-c: -node-d: -node-e: diff --git a/development/kind/mesh-config.yaml b/development/kind/mesh-config.yaml index 6f6e4587..58430ebb 100644 --- a/development/kind/mesh-config.yaml +++ b/development/kind/mesh-config.yaml @@ -1,11 +1,8 @@ # node -> service. A blank value means a bare node (no service, e.g. a caller). # Set a value to host a service on that node; the value is a folder path under # development/examples/, e.g: -# node-b: calc-mcp -# node-c: code-reviewer-pool/reviewer +# node-a: calc-mcp +# node-b: code-reviewer-pool/reviewer # All nodes ship bare by default — assign services to suit what you're testing. node-a: node-b: -node-c: -node-d: -node-e: diff --git a/development/kind/run-local-node.sh b/development/kind/run-local-node.sh index f56a31a2..469d746b 100755 --- a/development/kind/run-local-node.sh +++ b/development/kind/run-local-node.sh @@ -1,8 +1,7 @@ #!/usr/bin/env bash -# Enroll a locally-built ./bin/sam-node into the kind mesh for local testing. -# The control plane (exposed via sam-control-plane-nodeport + kind extraPortMappings) is reached at -# 127.0.0.1:9090 (HTTP enroll) and 127.0.0.1:4001 (libp2p TCP); the node relays -# peer traffic through the router. Extra args pass through, e.g. to host a service: +# Enroll a locally-built ./bin/sam-node into the kind mesh the way a real external node joins: +# a bootstrap token over the control plane's gateway address, peer traffic through the router's +# node-IP multiaddrs. Extra args pass through, e.g. to host a service: # ARGS="--config development/examples/calc-mcp/sam-node-config.yaml" set -euo pipefail @@ -14,25 +13,38 @@ PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "${PROJECT_ROOT}" # Prereqs -for bin in kubectl kind; do +for bin in kubectl kind curl jq; do command -v "$bin" >/dev/null 2>&1 || { echo "missing prerequisite: $bin" >&2; exit 1; } done kind get clusters 2>/dev/null | grep -qx "${CLUSTER}" || { echo "kind cluster '${CLUSTER}' not found; start it first: make kind-up" >&2; exit 1; } [[ -x ./bin/sam-node ]] || { echo "./bin/sam-node not found; build it first: make build" >&2; exit 1; } -# Identity + token -kubectl --context "${KCTX}" -n "${NAMESPACE}" create serviceaccount local-node-sa \ - >/dev/null 2>&1 || true -JWT="$(kubectl --context "${KCTX}" -n "${NAMESPACE}" create token local-node-sa \ - --audience=sam-mesh-audience --duration=1h)" +MAIN_IP="$(kubectl --context "${KCTX}" -n "${NAMESPACE}" get gateway sam-mesh-gateway \ + -o jsonpath='{.status.addresses[0].value}' 2>/dev/null || true)" +[[ -n "${MAIN_IP}" ]] || { + echo "gateway 'sam-mesh-gateway' has no LoadBalancer address; is cloud-provider-kind running?" >&2 + exit 1; } +CONTROL_PLANE_URL="http://${MAIN_IP}" -echo "Enrolling local ./bin/sam-node into the mesh control plane at http://127.0.0.1:9090…" +# Mint an enrollment credential the way an operator would, through the dev-routed /admin API. +ADMIN_TOKEN="$(kubectl --context "${KCTX}" -n "${NAMESPACE}" get secret sam-mesh-secrets \ + -o jsonpath='{.data.admin-token}' | base64 -d)" +TOKEN_RESPONSE="$(curl -s -X POST "${CONTROL_PLANE_URL}/admin/bootstrap-tokens" \ + -H "Authorization: Bearer ${ADMIN_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"role":"sam:role:node","ttl_hours":1,"max_usages":1,"description":"local sam-node"}' || true)" +BOOTSTRAP_TOKEN="$(printf '%s' "${TOKEN_RESPONSE}" | jq -r '.token // empty' 2>/dev/null || true)" +[[ -n "${BOOTSTRAP_TOKEN}" ]] || { + echo "could not mint a bootstrap token at ${CONTROL_PLANE_URL}/admin/bootstrap-tokens: ${TOKEN_RESPONSE}" >&2 + exit 1; } + +echo "Enrolling local ./bin/sam-node into the mesh control plane at ${CONTROL_PLANE_URL}…" echo " MCP/sidecar API on 127.0.0.1:9099" export SAM_API_TOKEN=devtoken exec ./bin/sam-node run \ - --control-plane http://127.0.0.1:9090 \ - --jwt "${JWT}" \ + --control-plane "${CONTROL_PLANE_URL}" \ + --bootstrap-token "${BOOTSTRAP_TOKEN}" \ --listen /ip4/0.0.0.0/tcp/0 \ --bind-addr 127.0.0.1:9099 \ --discovery-interval 200ms \ diff --git a/development/kind/run.sh b/development/kind/run.sh index 3712b28a..35b94939 100755 --- a/development/kind/run.sh +++ b/development/kind/run.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -# Kind dev mesh: a control plane and router plus the nodes declared in mesh-config.yaml, each pinned -# to its own k8s node, with live per-pod logs in named tmux panes. +# Kind dev mesh: a control plane and router plus the nodes from mesh-config.yaml, each pinned to its +# own k8s node, with live per-pod logs in named tmux panes. Control plane, console and Dex are reached +# over Gateway API LoadBalancer addresses from cloud-provider-kind (started here); the router at its node IP. set -euo pipefail CLUSTER="sam-kind" @@ -10,6 +11,13 @@ KCTX="kind-${CLUSTER}" IMAGE_TAG="local" CONTROL_PLANE_URL="http://sam-mesh-control-plane:8080" HELM="helm" +# Serves the gateway LoadBalancer IPs. It installs the Gateway API CRDs and the +# cloud-provider-kind GatewayClass itself, so the cluster needs no CRD step. +CPK_CONTAINER="cloud-provider-kind" +CPK_IMAGE="registry.k8s.io/cloud-provider-kind/cloud-controller-manager:v0.11.1" +# The console shares the control plane's gateway under this prefix, as it does on the +# cloud: the route rewrites the prefix away before the console sees the request. +CONSOLE_BASE_PATH="/console" @@ -39,6 +47,77 @@ check_prereqs() { fi } +# cloud-provider-kind names each gateway's envoy container after a hash of the gateway, so +# one left behind by a deleted cluster is adopted by the next run and reported Programmed +# while its config stream is dead and it serves nothing. +remove_lb_containers() { + local containers + containers="$(docker ps -aq -f "label=io.x-k8s.cloud-provider-kind.cluster=${CLUSTER}")" + if [[ -n "${containers}" ]]; then + # shellcheck disable=SC2086 # one container id per argument + docker rm -f ${containers} >/dev/null + fi +} + +# Must run after the cluster, so the 'kind' docker network exists. +start_cloud_provider_kind() { + remove_lb_containers + if [[ -n "$(docker ps -q -f "name=^${CPK_CONTAINER}$")" ]]; then + echo "cloud-provider-kind already running" + return + fi + docker rm -f "${CPK_CONTAINER}" >/dev/null 2>&1 || true + docker run -d --name "${CPK_CONTAINER}" --network kind \ + -v /var/run/docker.sock:/var/run/docker.sock "${CPK_IMAGE}" >/dev/null +} + +# Stop cloud-provider-kind before its containers, or it recreates them, and remove them before +# deleting the cluster, or deleting it makes cloud-provider-kind race us to the same containers. +teardown() { + docker rm -f "${CPK_CONTAINER}" >/dev/null 2>&1 || true + remove_lb_containers + kind delete cluster --name "${CLUSTER}" +} + +# gateway_ip : the LoadBalancer address cloud-provider-kind assigned, waited for. +gateway_ip() { + local gateway="$1" ip="" + for _ in $(seq 1 60); do + ip="$(kubectl --context "${KCTX}" -n "${NAMESPACE}" get gateway "${gateway}" \ + -o jsonpath='{.status.addresses[0].value}' 2>/dev/null || true)" + [[ -n "$ip" ]] && { echo "$ip"; return 0; } + sleep 2 + done + echo "gateway '${gateway}' never got a LoadBalancer address; is cloud-provider-kind running?" >&2 + return 1 +} + +# deploy_chart [extra --set flags]: later flags win, so phase 2 overrides phase 1. +# TLS verification is skipped because one issuer is the kind cluster's own API server, +# served with a self-signed cert. +deploy_chart() { + "${HELM}" --kube-context "${KCTX}" upgrade --install sam-mesh "${PROJECT_ROOT}/charts/sam-mesh" --timeout 10m \ + --namespace "${NAMESPACE}" \ + --set global.imageTag="${IMAGE_TAG}" \ + --set controlPlane.oidcIssuer="${CONTROL_PLANE_ISSUERS//,/\\,}" \ + --set controlPlane.allowedAudiences="${ALLOWED_AUDIENCES//,/\\,}" \ + --set controlPlane.insecureSkipTlsVerify=true \ + --set gateway.enabled=true \ + --set gateway.className=cloud-provider-kind \ + --set gateway.adminRoute=true \ + --set router.hostPort=4501 \ + --set router.nodeSelector.sam-role=control-plane \ + --set router.useOidcToken=true \ + "$@" +} + +# apply_dex : Dex is an independent component, deployed +# from dex.yaml rather than the chart, with the URLs of the moment substituted in. +apply_dex() { + DEX_ISSUER="$1" CONSOLE_REDIRECT_URI="$2" \ + envsubst '${DEX_ISSUER} ${CONSOLE_REDIRECT_URI}' < "${SCRIPT_DIR}/dex.yaml" \ + | kubectl --context "${KCTX}" -n "${NAMESPACE}" apply -f - +} render_and_apply() { local node="$1" svc="$2" @@ -107,8 +186,8 @@ read_mesh_nodes() { ### MAIN ### -if [[ $# -gt 0 && "$1" != "-s" && "$1" != "-l" ]]; then - echo "usage: $(basename "$0") [-s]" >&2 +if [[ $# -gt 0 && "$1" != "-s" && "$1" != "-l" && "$1" != "-d" ]]; then + echo "usage: $(basename "$0") [-s|-l|-d]" >&2 exit 1 fi @@ -118,11 +197,19 @@ if [[ "${1:-}" == "-l" ]]; then exit 0 fi +if [[ "${1:-}" == "-d" ]]; then + teardown + exit 0 +fi + check_prereqs "${1:-}" echo "== Creating kind cluster '${CLUSTER}' ==" kind create cluster --name "${CLUSTER}" --config "${SCRIPT_DIR}/kind-config.yaml" +echo "== Starting cloud-provider-kind ==" +start_cloud_provider_kind + echo "== Building sam images ==" make docker-build-control-plane docker-build-router docker-build-node docker-build-sam-console echo "== Loading sam images into kind ==" @@ -134,48 +221,68 @@ read_mesh_nodes ISSUER="$(kubectl --context "${KCTX}" get --raw /.well-known/openid-configuration | jq -r .issuer)" [[ -n "$ISSUER" ]] || { echo "could not determine cluster OIDC issuer" >&2; exit 1; } -OIDC_ISSUER="${OIDC_ISSUER:-http://sam-mesh-dex:5556/dex}" -OIDC_CLIENT_ID="${OIDC_CLIENT_ID:-sam-console}" -OIDC_CLIENT_SECRET="${OIDC_CLIENT_SECRET:-}" -OIDC_REDIRECT_URL="${OIDC_REDIRECT_URL:-}" - +# Dex's own address isn't known until its gateway has one, so phase 1 trusts only the +# cluster issuer and phase 2 prepends Dex. CONTROL_PLANE_ISSUERS="${ISSUER}" -if [[ -n "${OIDC_ISSUER}" ]]; then - CONTROL_PLANE_ISSUERS="${OIDC_ISSUER},${ISSUER}" -fi -ALLOWED_AUDIENCES="sam-mesh-audience,sam-control-plane-audience" -if [[ -n "${OIDC_CLIENT_ID}" ]]; then - ALLOWED_AUDIENCES="${OIDC_CLIENT_ID},${ALLOWED_AUDIENCES}" -fi +# The first audience is what the control plane reports as the OIDC client id, so it has to +# match Dex's static client. +ALLOWED_AUDIENCES="sam-console,sam-mesh-audience,sam-control-plane-audience" -export NAMESPACE ISSUER IMAGE_TAG OIDC_ISSUER OIDC_CLIENT_ID OIDC_CLIENT_SECRET OIDC_REDIRECT_URL CONTROL_PLANE_ISSUERS ALLOWED_AUDIENCES +# Only the node template's envsubst reads these from the environment. +export NAMESPACE IMAGE_TAG echo "== Applying namespace and RBAC cluster rules ==" envsubst '${NAMESPACE}' < "${SCRIPT_DIR}/00-namespace-rbac.yaml" | kubectl --context "${KCTX}" apply -f - echo "== Deploying SAM Mesh via Helm ==" -"${HELM}" --kube-context "${KCTX}" upgrade --install sam-mesh "${PROJECT_ROOT}/charts/sam-mesh" --timeout 10m \ - --namespace "${NAMESPACE}" \ - --set global.imageTag="${IMAGE_TAG}" \ - --set controlPlane.oidcIssuer="${CONTROL_PLANE_ISSUERS//,/\\,}" \ - --set controlPlane.allowedAudiences="${ALLOWED_AUDIENCES//,/\\,}" \ - --set controlPlane.service.type=NodePort \ - --set controlPlane.service.nodePort=30090 \ - --set router.service.type=NodePort \ - --set router.service.nodePort=30401 \ - --set router.useOidcToken=true \ - --set router.externalAddrs[0]="/dns4/sam-mesh-router/tcp/4501" \ - --set router.externalAddrs[1]="/ip4/127.0.0.1/tcp/4001" \ - --set console.service.type=NodePort \ - --set console.service.nodePort=30081 \ - --set dex.enabled=true \ - --set controlPlane.insecureSkipTlsVerify=true # ISSUER is the kind cluster's own API server, served with a self-signed cert +deploy_chart + +# The real URLs aren't known until Dex's gateway exists, so deploy it with placeholders +# first and rewire it below. +echo "== Deploying Dex ==" +apply_dex "http://sam-mesh-dex:5556/dex" "http://127.0.0.1${CONSOLE_BASE_PATH}/auth/callback" + +# The OIDC URLs are the gateway addresses, which only exist once the gateways do — so +# resolve them, then redeploy with the URLs everything must agree on. +echo "== Waiting for gateway LoadBalancer addresses ==" +MAIN_IP="$(gateway_ip sam-mesh-gateway)" +DEX_IP="$(gateway_ip sam-mesh-dex-gateway)" +CONSOLE_URL="http://${MAIN_IP}${CONSOLE_BASE_PATH}/" +echo "control plane: http://${MAIN_IP} console: ${CONSOLE_URL} dex: http://${DEX_IP}" + +OIDC_ISSUER="http://${DEX_IP}/dex" + +echo "== Wiring the OIDC URLs into Dex ==" +apply_dex "${OIDC_ISSUER}" "http://${MAIN_IP}${CONSOLE_BASE_PATH}/auth/callback" +# A config-only change doesn't roll the pods, and Dex reads its config at startup. +kubectl --context "${KCTX}" -n "${NAMESPACE}" rollout restart deployment/sam-mesh-dex +kubectl --context "${KCTX}" -n "${NAMESPACE}" rollout status deployment/sam-mesh-dex --timeout=180s + +# The control plane discovers the issuer at startup and refuses to start if it can't, so +# prove a pod can reach the gateway address before pinning the mesh to it. +echo "== Checking Dex discovery from inside the cluster ==" +# A pod left behind by an interrupted run fails the next one with AlreadyExists, which +# reads exactly like a routing failure. +kubectl --context "${KCTX}" -n "${NAMESPACE}" delete pod dex-discovery-check --ignore-not-found >/dev/null +kubectl --context "${KCTX}" -n "${NAMESPACE}" run dex-discovery-check \ + --rm -i --restart=Never --quiet --image=curlimages/curl:8.6.0 -- \ + curl -sf --retry 10 --retry-delay 2 --retry-connrefused --connect-timeout 5 --max-time 20 \ + "${OIDC_ISSUER}/.well-known/openid-configuration" >/dev/null || { + echo "cannot reach ${OIDC_ISSUER} from inside the cluster; pods must be able to route to the cloud-provider-kind LoadBalancer addresses" >&2 + exit 1 + } + +echo "== Wiring the OIDC URLs into the control plane ==" +CONTROL_PLANE_ISSUERS="${OIDC_ISSUER},${ISSUER}" +deploy_chart echo "== Waiting for database to be ready ==" kubectl --context "${KCTX}" -n "${NAMESPACE}" wait --for=condition=ready --timeout=180s pod -l app=sam-mesh-db echo "== Waiting for control plane to be ready ==" -kubectl --context "${KCTX}" -n "${NAMESPACE}" wait --for=condition=available --timeout=180s deployment/sam-mesh-control-plane +# rollout status, not wait --for=available: every pod must serve the Dex issuer before +# the console reads it below. +kubectl --context "${KCTX}" -n "${NAMESPACE}" rollout status deployment/sam-mesh-control-plane --timeout=180s # Policy seeding is handled automatically by the Helm chart bootstrap job, we just wait for it to complete. echo "== Waiting for bootstrap job to complete ==" @@ -183,10 +290,12 @@ kubectl --context "${KCTX}" -n "${NAMESPACE}" wait --for=condition=complete --ti echo "== Waiting for router to be ready ==" kubectl --context "${KCTX}" -n "${NAMESPACE}" wait --for=condition=ready --timeout=180s pod -l app=sam-mesh-router -echo "== Waiting for console to be ready ==" -kubectl --context "${KCTX}" -n "${NAMESPACE}" wait --for=condition=available --timeout=180s deployment/sam-mesh-console -echo "== Waiting for Dex to be ready ==" -kubectl --context "${KCTX}" -n "${NAMESPACE}" wait --for=condition=available --timeout=180s deployment/sam-mesh-dex + +# The console discovers the issuer from the control plane's /info once, at startup, so +# restart it now that the control plane serves the Dex issuer. +echo "== Restarting the console with the final issuer ==" +kubectl --context "${KCTX}" -n "${NAMESPACE}" rollout restart deployment/sam-mesh-console +kubectl --context "${KCTX}" -n "${NAMESPACE}" rollout status deployment/sam-mesh-console --timeout=180s echo "== Applying sam-nodes ==" @@ -201,13 +310,16 @@ for node in "${NODES[@]}"; do done echo -echo "Mesh up. To call a node's MCP API, port-forward it in another shell, e.g.:" -echo " kubectl --context ${KCTX} -n ${NAMESPACE} port-forward deploy/node-a 9091:8080" -echo "then:" -echo " ./bin/mcp-client -url http://127.0.0.1:9091/mcp -token devtoken -tool find_remote_tools -args '{}'" -echo "" -echo "You can access the SAM Web Console at:" -echo " http://localhost:9092/" +echo "Mesh up." +echo " console: ${CONSOLE_URL}" +echo " control plane: http://${MAIN_IP}" +echo " dex: ${OIDC_ISSUER}" +echo +echo "To drive the mesh, enroll a local node in another shell (it stays in the foreground):" +echo " make build && make kind-local-node" +echo "then call its MCP API on 127.0.0.1:9099:" +echo " ./bin/mcp-client -url http://127.0.0.1:9099/mcp -token devtoken -tool get_mesh_info -args '{}'" +echo " ./bin/mcp-client -url http://127.0.0.1:9099/mcp -token devtoken -tool find_remote_tools -args '{}'" if [[ "${1:-}" != "-s" ]]; then show_cluster_logs From a0cffcccdd5820ecc5b3a4f4ed5aef373e337969 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 17 Aug 2026 21:30:39 +0000 Subject: [PATCH 06/26] tests/e2e: pin the chart values the bats lane asserts on router.externalAddrs: node containers resolve sam-router per-network via --add-host, so the relay test's isolated network works only while the announced multiaddr carries that name; a node-IP announcement is unroutable from there. bootstrap.nodeServices: policy.bats' negative path proves the control plane denies an ungranted service, which only works while sam:role:node has a finite list. The chart default is now ["*"], so the lane declares the grants it tests. --- tests/e2e/lib/container_mesh.bash | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/e2e/lib/container_mesh.bash b/tests/e2e/lib/container_mesh.bash index 8f066084..22fb3bf3 100644 --- a/tests/e2e/lib/container_mesh.bash +++ b/tests/e2e/lib/container_mesh.bash @@ -363,6 +363,10 @@ if [[ -z "${MESH_HELPERS_LOADED:-}" ]]; then fi fi + # router.externalAddrs: nodes resolve sam-router per-network via --add-host, so the router must + # announce that name. Announcing a node IP instead is unroutable from an isolated test network. + # bootstrap.nodeServices: policy.bats asserts the control plane denies an ungranted service, so + # this lane pins the grants it tests rather than inheriting the chart default. "${helm_bin}" --kube-context="${KUBECONTEXT}" upgrade --install sam ./charts/sam-mesh \ --namespace default \ --set fullnameOverride="sam" \ @@ -375,7 +379,9 @@ if [[ -z "${MESH_HELPERS_LOADED:-}" ]]; then --set controlPlane.hostPort=8080 \ --set router.useOidcToken=false \ --set router.hostPort=4501 \ - --set console.enabled=false + --set console.enabled=false \ + --set 'router.externalAddrs={/dns4/sam-router/tcp/4501}' \ + --set 'bootstrap.nodeServices={mcp://calculator,mcp://db-agent,mcp://http-tool,mcp://stdio-tool,system://sam.catalog}' mesh_wait_for_rollout statefulset/sam-db mesh_wait_for_rollout deployment/sam-control-plane From 7672f211cdf4f31bac4b4fbf8631eab6ae4ca97e Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 17 Aug 2026 21:30:39 +0000 Subject: [PATCH 07/26] docs: describe the gateway-fronted kind mesh The kind guide now matches the gateway lane: LoadBalancer addresses from cloud-provider-kind, the console under /console on the control plane's address, Dex outside the chart, and a bootstrap-token local node. The use-case docs pick up the two-node default cluster. --- .../docs/development/kubernetes-deployment.md | 36 +++++++++++-------- site/content/docs/use-cases/gemini-buddy.md | 3 -- .../content/docs/use-cases/warm-agent-pool.md | 18 +++++++++- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/site/content/docs/development/kubernetes-deployment.md b/site/content/docs/development/kubernetes-deployment.md index 19f682ee..9f36f6ac 100644 --- a/site/content/docs/development/kubernetes-deployment.md +++ b/site/content/docs/development/kubernetes-deployment.md @@ -2,7 +2,7 @@ title: "Kubernetes Deployment and Local Testing Guide" linkTitle: "Kubernetes Deployment and Local Testing Guide" --- -This guide explains how to deploy the SAM control plane and router in a Kubernetes cluster and how to test it locally with `kind` — using the bundled `make kind-*` targets for a one-command mesh, or a manual setup with `cloud-provider-kind`. +This guide explains how to deploy the SAM control plane and router in a Kubernetes cluster and how to test it locally with `kind` — using the bundled `make kind-*` targets for a one-command mesh, or a manual setup. Both paths need `cloud-provider-kind`. > [!TIP] > This guide focuses on local development sandboxing. For production-grade Kubernetes deployments (GKE, EKS, AKS), see the [Production Kubernetes Deployment](../../user/kubernetes-deployment/) guide. @@ -19,19 +19,26 @@ The repository ships a one-command local mesh under `development/kind/`, driven make kind-up ``` -This creates a `sam-kind` cluster (one control-plane plus workers for the control plane, router, and `node-a` through `node-e`), builds the `sam-control-plane:local`, `sam-router:local`, and `sam-node:local` images, loads them into the cluster, and deploys: +This creates a `sam-kind` cluster (one control-plane plus three workers — one labeled `sam-role: control-plane`, and one each for `node-a` and `node-b`), builds the `sam-control-plane:local`, `sam-router:local`, and `sam-node:local` images, loads them into the cluster, and deploys: - The **control plane**, configured to trust the cluster's own OIDC issuer. -- Five **nodes** declared in `development/kind/mesh-config.yaml` (`node-a` through `node-e`), all **bare** by default — assign services to suit what you're testing. +- Two **nodes** declared in `development/kind/mesh-config.yaml` (`node-a` and `node-b`), both **bare** by default — assign services to suit what you're testing. -Nodes authenticate to the control plane via **Workload Identity Federation** (projected ServiceAccount tokens), so no static secrets or mock OIDC provider are needed. The control plane is exposed to the host on `127.0.0.1:9090` (HTTP enroll) and `127.0.0.1:4001` (libp2p) via a NodePort and the cluster's `extraPortMappings` — `cloud-provider-kind` is not required. +In-cluster nodes authenticate to the control plane via **Workload Identity Federation** (projected ServiceAccount tokens), so no static secrets or mock OIDC provider are needed. + +The mesh is exposed through Gateway API LoadBalancer addresses, so there are no port-forwards and no `extraPortMappings`. `run.sh` runs **`cloud-provider-kind`** as a container (a hard prerequisite — it needs the docker socket) to serve those addresses, and prints them when the mesh is up: + +- The **control plane** on its own address, routing the 8 exact enrollment paths, plus a dev-only `/admin` route. +- The **console** at `/console/` on that same address, shaped like the production deployment: `/console` 302s to `/console/`, and a `URLRewrite` filter strips the prefix before the console sees the request. +- **Dex** on its own address, deployed from `development/kind/dex.yaml` rather than the chart — Dex is an independent component the chart no longer bundles. +- The **router** at its own node's IP on port 4501, TCP **and** QUIC, announced from `status.hostIP`. Once everything is up, `make kind-up` opens a tmux session with live per-pod logs (control plane, router and each node in its own pane). Manage the mesh with: ```bash make kind-up ARGS=-s # bring the mesh up without attaching the log view make kind-logs # (re)attach the live-logs tmux session -make kind-down # delete the sam-kind cluster +make kind-down # delete the sam-kind cluster and stop cloud-provider-kind ``` ### Mesh Layout (`mesh-config.yaml`) @@ -40,17 +47,16 @@ The nodes that make up the dev mesh are declared in `development/kind/mesh-confi ```yaml # node -> service. A blank value means a bare node (no service, e.g. a caller). -# The service value is a folder path under development/examples/, e.g: -# node-b: calc-mcp -# node-c: code-reviewer-pool/reviewer +# Set a value to host a service on that node; the value is a folder path under +# development/examples/, e.g: +# node-a: calc-mcp +# node-b: code-reviewer-pool/reviewer +# All nodes ship bare by default — assign services to suit what you're testing. node-a: node-b: -node-c: -node-d: -node-e: ``` -- The key is the node's name. The cluster currently ships with a control plane and router plus these **five** agent nodes, all bare by default; each is pinned to a matching worker via the `sam-role` labels in `kind-config.yaml`. +- The key is the node's name. The cluster currently ships with a control plane and router plus these **two** agent nodes, both bare by default; each is pinned to a matching worker via the `sam-role` labels in `kind-config.yaml`. - A **blank** value is a bare node — a `sam-node` with no local service, useful as a caller/consumer. - A **non-blank** value is a folder name under `development/examples/`. That service is built and deployed as a **sidecar** next to the node, and the node is configured to advertise it to the mesh. @@ -75,12 +81,12 @@ A service is any backend a node advertises to the mesh. Its kind is set by the ` ``` The sidecar and `sam-node` share the pod's network, so `target_url` is always `127.0.0.1:`, where `` matches the port your service listens on. -2. **Assign it to a node** in `mesh-config.yaml` — set the value on any free node slot (`node-a` through `node-e`): +2. **Assign it to a node** in `mesh-config.yaml` — set the value on a free node slot (`node-a` or `node-b`): ```yaml node-a: my-mcp ``` > [!NOTE] - > There are five node slots because `kind-config.yaml` defines five workers labeled `sam-role: node-a|node-b|node-c|node-d|node-e`. To host more than five services at once, add a matching labeled worker there too. + > There are two node slots because `kind-config.yaml` defines two workers labeled `sam-role: node-a|node-b`. To host more than two services at once, add a matching labeled worker there too. 3. **Recreate the cluster** so the new service is built and deployed: ```bash @@ -104,7 +110,7 @@ make build # produce ./bin/sam-node make kind-local-node ``` -This mints a ServiceAccount token and runs `./bin/sam-node` against the control plane at `127.0.0.1:9090`, exposing its MCP API on `127.0.0.1:9099` with the API token `devtoken`. Extra flags pass through via `ARGS`, e.g. to host an example service: +This mints a bootstrap token through the control plane's `/admin` API and runs `./bin/sam-node` against the control plane's gateway address — the same credential and path a real external node uses — exposing its MCP API on `127.0.0.1:9099` with the API token `devtoken`. Extra flags pass through via `ARGS`, e.g. to host an example service: ```bash make kind-local-node ARGS="--config development/examples/calc-mcp/sam-node-config.yaml" diff --git a/site/content/docs/use-cases/gemini-buddy.md b/site/content/docs/use-cases/gemini-buddy.md index dbb8abed..b309328b 100644 --- a/site/content/docs/use-cases/gemini-buddy.md +++ b/site/content/docs/use-cases/gemini-buddy.md @@ -86,9 +86,6 @@ Host the buddy on one node in `development/kind/mesh-config.yaml`: ```yaml node-a: # bare node (orchestrator entry) node-b: gemini-buddy-mcp # the buddy -node-c: -node-d: -node-e: ``` ### 3. Bring the mesh up and start a local orchestrator node diff --git a/site/content/docs/use-cases/warm-agent-pool.md b/site/content/docs/use-cases/warm-agent-pool.md index 0f9461e0..f191bd7e 100644 --- a/site/content/docs/use-cases/warm-agent-pool.md +++ b/site/content/docs/use-cases/warm-agent-pool.md @@ -108,7 +108,23 @@ before building (a free key is fine for the demo). ### 2. Mesh layout -The layout ships in `development/kind/mesh-config.yaml`: +This use case needs five node slots, and the cluster ships with two (`node-a` +and `node-b`). Add the three missing workers to +`development/kind/kind-config.yaml` first: + +```yaml + - role: worker + labels: + sam-role: node-c + - role: worker + labels: + sam-role: node-d + - role: worker + labels: + sam-role: node-e +``` + +Then write this layout into `development/kind/mesh-config.yaml`: ```yaml node-a: # bare node (orchestrator entry) From 5163c850c61b4fc92e8366839858bd2eef1dfc30 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 17 Aug 2026 09:42:38 +0000 Subject: [PATCH 08/26] development/examples: pin greeter-mcp to mcp 1.x The unbounded mcp>=1.0.0 pulled mcp 2.0.0, which dropped mcp.server.fastmcp, so the container crashlooped on import. calc-mcp already pins <2.0.0. --- development/examples/greeter-mcp/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/development/examples/greeter-mcp/requirements.txt b/development/examples/greeter-mcp/requirements.txt index c413b608..7ba9cdc1 100644 --- a/development/examples/greeter-mcp/requirements.txt +++ b/development/examples/greeter-mcp/requirements.txt @@ -1 +1 @@ -mcp>=1.0.0 +mcp[cli]>=1.0.0,<2.0.0 From ca94afaa8dac2ff4668968e4fa1406e998aee4e0 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 24 Aug 2026 11:19:18 +0000 Subject: [PATCH 09/26] charts: run the sam pods under the restricted Pod Security Standard runAsNonRoot + RuntimeDefault seccomp at pod level, no privilege escalation and all capabilities dropped per container. The curl init/bootstrap containers pin runAsUser 100 because the image's USER is non-numeric, which runAsNonRoot can't verify. Postgres is left alone: its entrypoint still runs as root and steps down itself. --- charts/sam-mesh/templates/bootstrap-job.yaml | 10 ++++++ .../templates/console-deployment.yaml | 14 ++++++++ .../templates/control-plane-deployment.yaml | 8 +++++ .../templates/router-statefulset.yaml | 10 ++++++ charts/sam-mesh/values.yaml | 32 +++++++++++++++++++ 5 files changed, 74 insertions(+) diff --git a/charts/sam-mesh/templates/bootstrap-job.yaml b/charts/sam-mesh/templates/bootstrap-job.yaml index e93e06ad..00a9da80 100644 --- a/charts/sam-mesh/templates/bootstrap-job.yaml +++ b/charts/sam-mesh/templates/bootstrap-job.yaml @@ -49,10 +49,20 @@ spec: spec: serviceAccountName: {{ include "sam-mesh.fullname" . }}-bootstrap-sa restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + # curlimages/curl's USER is non-numeric; runAsNonRoot needs an explicit uid. + runAsUser: 100 + seccompProfile: + type: RuntimeDefault containers: - name: bootstrap image: curlimages/curl:8.6.0 imagePullPolicy: {{ .Values.global.imagePullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] env: - name: NAMESPACE value: {{ .Release.Namespace | quote }} diff --git a/charts/sam-mesh/templates/console-deployment.yaml b/charts/sam-mesh/templates/console-deployment.yaml index c3f22ce3..c65abf42 100644 --- a/charts/sam-mesh/templates/console-deployment.yaml +++ b/charts/sam-mesh/templates/console-deployment.yaml @@ -15,10 +15,20 @@ spec: labels: app: {{ include "sam-mesh.fullname" . }}-console spec: + {{- with .Values.console.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} initContainers: - name: wait-control-plane image: curlimages/curl:8.6.0 imagePullPolicy: {{ .Values.global.imagePullPolicy }} + securityContext: + # curlimages/curl's USER is non-numeric; runAsNonRoot needs an explicit uid. + runAsUser: 100 + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] command: ["/bin/sh", "-c"] args: - | @@ -32,6 +42,10 @@ spec: - name: sam-console image: "{{ .Values.console.image.repository }}:{{ .Values.global.imageTag }}" imagePullPolicy: {{ .Values.global.imagePullPolicy }} + {{- with .Values.console.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} env: - name: SAM_ADMIN_TOKEN valueFrom: diff --git a/charts/sam-mesh/templates/control-plane-deployment.yaml b/charts/sam-mesh/templates/control-plane-deployment.yaml index 9897cc55..8c22c40a 100644 --- a/charts/sam-mesh/templates/control-plane-deployment.yaml +++ b/charts/sam-mesh/templates/control-plane-deployment.yaml @@ -14,10 +14,18 @@ spec: labels: app: {{ include "sam-mesh.fullname" . }}-control-plane spec: + {{- with .Values.controlPlane.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: sam-control-plane image: "{{ .Values.controlPlane.image.repository }}:{{ .Values.global.imageTag }}" imagePullPolicy: {{ .Values.global.imagePullPolicy }} + {{- with .Values.controlPlane.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} env: - name: DB_PASSWORD valueFrom: diff --git a/charts/sam-mesh/templates/router-statefulset.yaml b/charts/sam-mesh/templates/router-statefulset.yaml index f07b0fd4..c32c9ee1 100644 --- a/charts/sam-mesh/templates/router-statefulset.yaml +++ b/charts/sam-mesh/templates/router-statefulset.yaml @@ -36,6 +36,12 @@ spec: - name: wait-control-plane image: curlimages/curl:8.6.0 imagePullPolicy: {{ .Values.global.imagePullPolicy }} + securityContext: + # curlimages/curl's USER is non-numeric; runAsNonRoot needs an explicit uid. + runAsUser: 100 + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] command: ["/bin/sh", "-c"] args: - | @@ -49,6 +55,10 @@ spec: - name: sam-router image: "{{ .Values.router.image.repository }}:{{ .Values.global.imageTag }}" imagePullPolicy: {{ .Values.global.imagePullPolicy }} + {{- with .Values.router.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} ports: - containerPort: 4501 name: p2p-tcp diff --git a/charts/sam-mesh/values.yaml b/charts/sam-mesh/values.yaml index 33e6217e..4b3a2db1 100644 --- a/charts/sam-mesh/values.yaml +++ b/charts/sam-mesh/values.yaml @@ -45,6 +45,17 @@ controlPlane: # (comma-separated for more than one); the control plane refuses to start without one. oidcIssuer: "" allowedAudiences: "sam-mesh-audience,sam-control-plane-audience" + podSecurityContext: + runAsNonRoot: true + # Dockerfiles say "USER nonroot" by name; runAsNonRoot needs the numeric uid. + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] resources: requests: cpu: 100m @@ -84,8 +95,18 @@ router: enabled: true replicaCount: 1 podSecurityContext: + runAsNonRoot: true + # Dockerfiles say "USER nonroot" by name; runAsNonRoot needs the numeric uid. + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault fsGroup: 65532 fsGroupChangePolicy: "OnRootMismatch" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] image: repository: sam-router useOidcToken: false @@ -113,6 +134,17 @@ console: replicaCount: 1 image: repository: sam-console + podSecurityContext: + runAsNonRoot: true + # Dockerfiles say "USER nonroot" by name; runAsNonRoot needs the numeric uid. + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] resources: requests: cpu: 50m From 713727f2ae5f59d27d0d2341df77d79fcb5c3788 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 24 Aug 2026 11:20:27 +0000 Subject: [PATCH 10/26] charts: allow overriding the image tag per image Each sam image gains image.tag, falling back to global.imageTag. --- charts/sam-mesh/templates/console-deployment.yaml | 2 +- charts/sam-mesh/templates/control-plane-deployment.yaml | 2 +- charts/sam-mesh/templates/router-statefulset.yaml | 2 +- charts/sam-mesh/values.yaml | 6 ++++++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/charts/sam-mesh/templates/console-deployment.yaml b/charts/sam-mesh/templates/console-deployment.yaml index c65abf42..a7126fb0 100644 --- a/charts/sam-mesh/templates/console-deployment.yaml +++ b/charts/sam-mesh/templates/console-deployment.yaml @@ -40,7 +40,7 @@ spec: done containers: - name: sam-console - image: "{{ .Values.console.image.repository }}:{{ .Values.global.imageTag }}" + image: "{{ .Values.console.image.repository }}:{{ .Values.console.image.tag | default .Values.global.imageTag }}" imagePullPolicy: {{ .Values.global.imagePullPolicy }} {{- with .Values.console.securityContext }} securityContext: diff --git a/charts/sam-mesh/templates/control-plane-deployment.yaml b/charts/sam-mesh/templates/control-plane-deployment.yaml index 8c22c40a..f8504ec9 100644 --- a/charts/sam-mesh/templates/control-plane-deployment.yaml +++ b/charts/sam-mesh/templates/control-plane-deployment.yaml @@ -20,7 +20,7 @@ spec: {{- end }} containers: - name: sam-control-plane - image: "{{ .Values.controlPlane.image.repository }}:{{ .Values.global.imageTag }}" + image: "{{ .Values.controlPlane.image.repository }}:{{ .Values.controlPlane.image.tag | default .Values.global.imageTag }}" imagePullPolicy: {{ .Values.global.imagePullPolicy }} {{- with .Values.controlPlane.securityContext }} securityContext: diff --git a/charts/sam-mesh/templates/router-statefulset.yaml b/charts/sam-mesh/templates/router-statefulset.yaml index c32c9ee1..895f2a12 100644 --- a/charts/sam-mesh/templates/router-statefulset.yaml +++ b/charts/sam-mesh/templates/router-statefulset.yaml @@ -53,7 +53,7 @@ spec: done containers: - name: sam-router - image: "{{ .Values.router.image.repository }}:{{ .Values.global.imageTag }}" + image: "{{ .Values.router.image.repository }}:{{ .Values.router.image.tag | default .Values.global.imageTag }}" imagePullPolicy: {{ .Values.global.imagePullPolicy }} {{- with .Values.router.securityContext }} securityContext: diff --git a/charts/sam-mesh/values.yaml b/charts/sam-mesh/values.yaml index 4b3a2db1..d855ce1f 100644 --- a/charts/sam-mesh/values.yaml +++ b/charts/sam-mesh/values.yaml @@ -33,6 +33,8 @@ controlPlane: replicaCount: 2 image: repository: sam-control-plane + # Overrides global.imageTag for this image only. + tag: "" logLevel: info # Leave empty to auto-generate a random admin token on first install (kept # stable across upgrades). Set explicitly (e.g. via --set) to pin a known value. @@ -109,6 +111,8 @@ router: drop: ["ALL"] image: repository: sam-router + # Overrides global.imageTag for this image only. + tag: "" useOidcToken: false logLevel: info # Binds 4501 TCP+UDP on the router's own node and announces /ip4//... for both @@ -134,6 +138,8 @@ console: replicaCount: 1 image: repository: sam-console + # Overrides global.imageTag for this image only. + tag: "" podSecurityContext: runAsNonRoot: true # Dockerfiles say "USER nonroot" by name; runAsNonRoot needs the numeric uid. From 362c1220ed33c78f2d8c096e45f19059c8c12f2a Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 24 Aug 2026 11:20:56 +0000 Subject: [PATCH 11/26] charts: roll control plane and console when the shared secret changes Pinning a new controlPlane.adminToken or db password on upgrade only updated the Secret; the pods kept the old env values until a manual restart. A checksum/secrets pod annotation forces the rollout. The db statefulset is left out: postgres keeps the password initdb stored in the PVC, so restarting it on a values change fixes nothing. --- charts/sam-mesh/templates/console-deployment.yaml | 3 +++ charts/sam-mesh/templates/control-plane-deployment.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/charts/sam-mesh/templates/console-deployment.yaml b/charts/sam-mesh/templates/console-deployment.yaml index a7126fb0..a16ef4e9 100644 --- a/charts/sam-mesh/templates/console-deployment.yaml +++ b/charts/sam-mesh/templates/console-deployment.yaml @@ -12,6 +12,9 @@ spec: app: {{ include "sam-mesh.fullname" . }}-console template: metadata: + annotations: + # Secrets land as env vars, which are not live-reloaded; roll the pods on change. + checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} labels: app: {{ include "sam-mesh.fullname" . }}-console spec: diff --git a/charts/sam-mesh/templates/control-plane-deployment.yaml b/charts/sam-mesh/templates/control-plane-deployment.yaml index f8504ec9..622c80e7 100644 --- a/charts/sam-mesh/templates/control-plane-deployment.yaml +++ b/charts/sam-mesh/templates/control-plane-deployment.yaml @@ -11,6 +11,9 @@ spec: app: {{ include "sam-mesh.fullname" . }}-control-plane template: metadata: + annotations: + # Secrets land as env vars, which are not live-reloaded; roll the pods on change. + checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} labels: app: {{ include "sam-mesh.fullname" . }}-control-plane spec: From ff8159cb8e60db2dfd61c9c265f888d82f98cb34 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 24 Aug 2026 11:21:15 +0000 Subject: [PATCH 12/26] charts: stop granting the bootstrap job access to every secret The Role allowed get/create/delete/patch/update on all secrets in the namespace, including the admin token and db password. The job only server-side-applies the router-token secret: get/patch are now pinned to it via resourceNames, create stays unscoped (admission cannot match a name on create), and delete/update are dropped. --- charts/sam-mesh/templates/bootstrap-job.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/charts/sam-mesh/templates/bootstrap-job.yaml b/charts/sam-mesh/templates/bootstrap-job.yaml index 00a9da80..a737538d 100644 --- a/charts/sam-mesh/templates/bootstrap-job.yaml +++ b/charts/sam-mesh/templates/bootstrap-job.yaml @@ -12,9 +12,15 @@ metadata: labels: {{- include "sam-mesh.labels" . | nindent 4 }} rules: +# resourceNames cannot restrict create (the name is unknown at admission), +# so it gets a bare rule; everything else is pinned to the router-token secret. - apiGroups: [""] resources: ["secrets"] - verbs: ["get", "create", "delete", "patch", "update"] + verbs: ["create"] +- apiGroups: [""] + resources: ["secrets"] + resourceNames: ["{{ include "sam-mesh.fullname" . }}-router-token"] + verbs: ["get", "patch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding From 313d3a97f254d251ae44b188907bf842a00419c1 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 24 Aug 2026 11:21:35 +0000 Subject: [PATCH 13/26] charts: warn in NOTES when hostPort caps routers at one per node --- charts/sam-mesh/templates/NOTES.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/charts/sam-mesh/templates/NOTES.txt b/charts/sam-mesh/templates/NOTES.txt index 62b01ef9..545eed5c 100644 --- a/charts/sam-mesh/templates/NOTES.txt +++ b/charts/sam-mesh/templates/NOTES.txt @@ -31,5 +31,13 @@ valid identity is enrolled automatically without manual admin approval. Set it to false for production deployments where you want to review enrollments. {{- end }} +{{- if and .Values.router.hostPort (gt (int .Values.router.replicaCount) 1) }} + +WARNING: router.hostPort={{ .Values.router.hostPort }} with replicaCount={{ .Values.router.replicaCount }}. +Each router binds that port on its node, so at most one router runs per node. +With fewer than {{ .Values.router.replicaCount }} schedulable nodes (after router.nodeSelector), the extra +pods stay Pending with "node(s) didn't have free ports". +{{- end }} + See charts/sam-mesh/README.md for more details on configuring OIDC, TLS, and production values. From 06ec530707862bad74f1397cff899affdf52d1f2 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 24 Aug 2026 11:59:32 +0000 Subject: [PATCH 14/26] charts: persist the router key on a PVC The libp2p peer ID derives from /data/router.key; on emptyDir every reschedule minted a new identity, forcing re-enrollment and leaving the old peer registered until its lease expired. A volumeClaimTemplate keeps it stable, matching .github/k8s/sam-router-template.yaml. volumeClaimTemplates are immutable: existing installs must delete the router StatefulSet before upgrading (the PVC is created fresh either way). --- charts/sam-mesh/templates/router-statefulset.yaml | 12 +++++++++--- charts/sam-mesh/values.yaml | 2 ++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/charts/sam-mesh/templates/router-statefulset.yaml b/charts/sam-mesh/templates/router-statefulset.yaml index 895f2a12..26b7d2f8 100644 --- a/charts/sam-mesh/templates/router-statefulset.yaml +++ b/charts/sam-mesh/templates/router-statefulset.yaml @@ -129,10 +129,8 @@ spec: mountPath: /var/run/secrets/tokens readOnly: true {{- end }} - volumes: - - name: router-data - emptyDir: {} {{- if .Values.router.useOidcToken }} + volumes: - name: sam-token projected: sources: @@ -141,4 +139,12 @@ spec: expirationSeconds: 3600 audience: "sam-control-plane-audience" {{- end }} + volumeClaimTemplates: + - metadata: + name: router-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.router.storageSize | quote }} {{- end }} diff --git a/charts/sam-mesh/values.yaml b/charts/sam-mesh/values.yaml index d855ce1f..b674969a 100644 --- a/charts/sam-mesh/values.yaml +++ b/charts/sam-mesh/values.yaml @@ -122,6 +122,8 @@ router: # Overrides the announced multiaddrs; derived from hostPort when left empty. externalAddrs: [] allowLoopback: false + # PVC for /data/router.key, so the libp2p peer ID survives rescheduling. + storageSize: 1Gi resources: requests: cpu: 100m From 8cf6ff4e5341df9a37ac3340bdb002acd847ecb7 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 24 Aug 2026 11:59:56 +0000 Subject: [PATCH 15/26] charts: add pg_isready probes to the db statefulset Without a readiness probe the db Service routed to postgres while initdb or crash recovery was still running. PGDATA is deliberately not moved to a subdirectory: existing PVCs are initialized at the volume root and a new PGDATA would shadow their data. --- charts/sam-mesh/templates/db-statefulset.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/charts/sam-mesh/templates/db-statefulset.yaml b/charts/sam-mesh/templates/db-statefulset.yaml index 9a585a8e..5a75e891 100644 --- a/charts/sam-mesh/templates/db-statefulset.yaml +++ b/charts/sam-mesh/templates/db-statefulset.yaml @@ -34,6 +34,14 @@ spec: - containerPort: 5432 name: postgres protocol: TCP + readinessProbe: + exec: + command: ["pg_isready", "-U", {{ .Values.database.postgres.user | quote }}, "-d", {{ .Values.database.postgres.database | quote }}] + periodSeconds: 5 + livenessProbe: + exec: + command: ["pg_isready", "-U", {{ .Values.database.postgres.user | quote }}, "-d", {{ .Values.database.postgres.database | quote }}] + periodSeconds: 15 resources: {{- toYaml .Values.database.postgres.resources | nindent 10 }} volumeMounts: From b079565d5c4ee277da62fce7456787d0b222c0ed Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 24 Aug 2026 12:01:01 +0000 Subject: [PATCH 16/26] charts: pass nodeSelector, tolerations and affinity through to every workload The router only exposed nodeSelector; the other workloads had nothing, so the chart could not target dedicated or tainted node pools. --- charts/sam-mesh/templates/console-deployment.yaml | 12 ++++++++++++ .../sam-mesh/templates/control-plane-deployment.yaml | 12 ++++++++++++ charts/sam-mesh/templates/db-statefulset.yaml | 12 ++++++++++++ charts/sam-mesh/templates/router-statefulset.yaml | 8 ++++++++ charts/sam-mesh/values.yaml | 11 +++++++++++ 5 files changed, 55 insertions(+) diff --git a/charts/sam-mesh/templates/console-deployment.yaml b/charts/sam-mesh/templates/console-deployment.yaml index a16ef4e9..1490c4f3 100644 --- a/charts/sam-mesh/templates/console-deployment.yaml +++ b/charts/sam-mesh/templates/console-deployment.yaml @@ -18,6 +18,18 @@ spec: labels: app: {{ include "sam-mesh.fullname" . }}-console spec: + {{- with .Values.console.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.console.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.console.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.console.podSecurityContext }} securityContext: {{- toYaml . | nindent 8 }} diff --git a/charts/sam-mesh/templates/control-plane-deployment.yaml b/charts/sam-mesh/templates/control-plane-deployment.yaml index 622c80e7..2f371037 100644 --- a/charts/sam-mesh/templates/control-plane-deployment.yaml +++ b/charts/sam-mesh/templates/control-plane-deployment.yaml @@ -17,6 +17,18 @@ spec: labels: app: {{ include "sam-mesh.fullname" . }}-control-plane spec: + {{- with .Values.controlPlane.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controlPlane.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controlPlane.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.controlPlane.podSecurityContext }} securityContext: {{- toYaml . | nindent 8 }} diff --git a/charts/sam-mesh/templates/db-statefulset.yaml b/charts/sam-mesh/templates/db-statefulset.yaml index 5a75e891..2e635355 100644 --- a/charts/sam-mesh/templates/db-statefulset.yaml +++ b/charts/sam-mesh/templates/db-statefulset.yaml @@ -16,6 +16,18 @@ spec: labels: app: {{ include "sam-mesh.fullname" . }}-db spec: + {{- with .Values.database.postgres.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.database.postgres.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.database.postgres.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: postgres image: "{{ .Values.database.postgres.image.repository }}:{{ .Values.database.postgres.image.tag }}" diff --git a/charts/sam-mesh/templates/router-statefulset.yaml b/charts/sam-mesh/templates/router-statefulset.yaml index 26b7d2f8..9b1561fe 100644 --- a/charts/sam-mesh/templates/router-statefulset.yaml +++ b/charts/sam-mesh/templates/router-statefulset.yaml @@ -28,6 +28,14 @@ spec: nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.router.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.router.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.router.podSecurityContext }} securityContext: {{- toYaml . | nindent 8 }} diff --git a/charts/sam-mesh/values.yaml b/charts/sam-mesh/values.yaml index b674969a..1a82c2d3 100644 --- a/charts/sam-mesh/values.yaml +++ b/charts/sam-mesh/values.yaml @@ -47,6 +47,9 @@ controlPlane: # (comma-separated for more than one); the control plane refuses to start without one. oidcIssuer: "" allowedAudiences: "sam-mesh-audience,sam-control-plane-audience" + nodeSelector: {} + tolerations: [] + affinity: {} podSecurityContext: runAsNonRoot: true # Dockerfiles say "USER nonroot" by name; runAsNonRoot needs the numeric uid. @@ -85,6 +88,9 @@ database: port: 5432 sslmode: disable storageSize: 1Gi + nodeSelector: {} + tolerations: [] + affinity: {} resources: requests: cpu: 100m @@ -119,6 +125,8 @@ router: # transports. One router per node. hostPort: null nodeSelector: {} + tolerations: [] + affinity: {} # Overrides the announced multiaddrs; derived from hostPort when left empty. externalAddrs: [] allowLoopback: false @@ -142,6 +150,9 @@ console: repository: sam-console # Overrides global.imageTag for this image only. tag: "" + nodeSelector: {} + tolerations: [] + affinity: {} podSecurityContext: runAsNonRoot: true # Dockerfiles say "USER nonroot" by name; runAsNonRoot needs the numeric uid. From 64968a71a9db61cf2e0d026d595a91605d201dbd Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 24 Aug 2026 12:01:30 +0000 Subject: [PATCH 17/26] charts: give the curl init/bootstrap containers requests and limits They were the only containers without resources, which blocks scheduling in namespaces whose ResourceQuota requires them. Hardcoded small values: the containers only poll /info and POST a couple of requests. --- charts/sam-mesh/templates/bootstrap-job.yaml | 7 +++++++ charts/sam-mesh/templates/console-deployment.yaml | 7 +++++++ charts/sam-mesh/templates/router-statefulset.yaml | 7 +++++++ 3 files changed, 21 insertions(+) diff --git a/charts/sam-mesh/templates/bootstrap-job.yaml b/charts/sam-mesh/templates/bootstrap-job.yaml index a737538d..1a2f2f58 100644 --- a/charts/sam-mesh/templates/bootstrap-job.yaml +++ b/charts/sam-mesh/templates/bootstrap-job.yaml @@ -69,6 +69,13 @@ spec: allowPrivilegeEscalation: false capabilities: drop: ["ALL"] + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 100m + memory: 64Mi env: - name: NAMESPACE value: {{ .Release.Namespace | quote }} diff --git a/charts/sam-mesh/templates/console-deployment.yaml b/charts/sam-mesh/templates/console-deployment.yaml index 1490c4f3..8d77457b 100644 --- a/charts/sam-mesh/templates/console-deployment.yaml +++ b/charts/sam-mesh/templates/console-deployment.yaml @@ -44,6 +44,13 @@ spec: allowPrivilegeEscalation: false capabilities: drop: ["ALL"] + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 100m + memory: 64Mi command: ["/bin/sh", "-c"] args: - | diff --git a/charts/sam-mesh/templates/router-statefulset.yaml b/charts/sam-mesh/templates/router-statefulset.yaml index 9b1561fe..3a11b95a 100644 --- a/charts/sam-mesh/templates/router-statefulset.yaml +++ b/charts/sam-mesh/templates/router-statefulset.yaml @@ -50,6 +50,13 @@ spec: allowPrivilegeEscalation: false capabilities: drop: ["ALL"] + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 100m + memory: 64Mi command: ["/bin/sh", "-c"] args: - | From 7fa1d8ab29b4403380982203f09b87f0f0fc3cab Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 24 Aug 2026 15:03:09 +0000 Subject: [PATCH 18/26] charts: add helm-unittest template tests One suite per template covering the load-bearing logic: numeric runAsUser pins, bootstrap RBAC scoping, secret checksum rollout, hostPort/externalAddrs address derivation, gateway routing toggles, the router PVC and OIDC mode, scheduling passthroughs and external postgres. Run with make helm-test; the target installs the helm-unittest plugin on first use. Suites that assert the checksum/secrets annotation also load templates/secrets.yaml, or the include cannot resolve. --- Makefile | 6 + charts/sam-mesh/tests/bootstrap-job_test.yaml | 34 ++++++ .../tests/console-deployment_test.yaml | 35 ++++++ .../tests/control-plane-deployment_test.yaml | 59 ++++++++++ .../sam-mesh/tests/db-statefulset_test.yaml | 37 ++++++ charts/sam-mesh/tests/gateway_test.yaml | 62 ++++++++++ .../tests/router-statefulset_test.yaml | 109 ++++++++++++++++++ 7 files changed, 342 insertions(+) create mode 100644 charts/sam-mesh/tests/bootstrap-job_test.yaml create mode 100644 charts/sam-mesh/tests/console-deployment_test.yaml create mode 100644 charts/sam-mesh/tests/control-plane-deployment_test.yaml create mode 100644 charts/sam-mesh/tests/db-statefulset_test.yaml create mode 100644 charts/sam-mesh/tests/gateway_test.yaml create mode 100644 charts/sam-mesh/tests/router-statefulset_test.yaml diff --git a/Makefile b/Makefile index f5c6a237..aa965981 100644 --- a/Makefile +++ b/Makefile @@ -197,6 +197,12 @@ helm-lint: lint: fmt helm-lint hack/lint.sh +# fast chart template checks; no cluster needed +.PHONY: helm-test +helm-test: + @helm plugin list 2>/dev/null | grep -q '^unittest' || helm plugin install https://github.com/helm-unittest/helm-unittest --verify=false + helm unittest charts/sam-mesh + .PHONY: verify verify: ./hack/verify-generated.sh diff --git a/charts/sam-mesh/tests/bootstrap-job_test.yaml b/charts/sam-mesh/tests/bootstrap-job_test.yaml new file mode 100644 index 00000000..9be914f7 --- /dev/null +++ b/charts/sam-mesh/tests/bootstrap-job_test.yaml @@ -0,0 +1,34 @@ +suite: bootstrap job +templates: + - templates/bootstrap-job.yaml +release: + name: sam-mesh +tests: + - it: role only touches the router-token secret + documentSelector: + path: kind + value: Role + asserts: + - equal: + path: rules[0].verbs + value: [create] + - equal: + path: rules[1].resourceNames + value: [sam-mesh-router-token] + - equal: + path: rules[1].verbs + value: [get, patch] + - lengthEqual: + path: rules + count: 2 + + - it: job pod runs as the numeric curl uid with resources set + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: spec.template.spec.securityContext.runAsUser + value: 100 + - exists: + path: spec.template.spec.containers[0].resources.limits diff --git a/charts/sam-mesh/tests/console-deployment_test.yaml b/charts/sam-mesh/tests/console-deployment_test.yaml new file mode 100644 index 00000000..4b4c7426 --- /dev/null +++ b/charts/sam-mesh/tests/console-deployment_test.yaml @@ -0,0 +1,35 @@ +suite: console deployment +templates: + # secrets.yaml must be loaded for the checksum/secrets include to resolve + - templates/console-deployment.yaml + - templates/secrets.yaml +release: + name: sam-mesh +tests: + - it: pins the numeric uids for the app and the curl init container + template: templates/console-deployment.yaml + asserts: + - equal: + path: spec.template.spec.securityContext.runAsUser + value: 65532 + - equal: + path: spec.template.spec.initContainers[0].securityContext.runAsUser + value: 100 + + - it: rolls the pods when the shared secret changes + template: templates/console-deployment.yaml + set: + controlPlane.adminToken: aaa + database.postgres.password: p + asserts: + - matchRegex: + path: spec.template.metadata.annotations["checksum/secrets"] + pattern: ^[0-9a-f]{64}$ + + - it: console.enabled=false renders nothing + template: templates/console-deployment.yaml + set: + console.enabled: false + asserts: + - hasDocuments: + count: 0 diff --git a/charts/sam-mesh/tests/control-plane-deployment_test.yaml b/charts/sam-mesh/tests/control-plane-deployment_test.yaml new file mode 100644 index 00000000..7b9f34d4 --- /dev/null +++ b/charts/sam-mesh/tests/control-plane-deployment_test.yaml @@ -0,0 +1,59 @@ +suite: control plane deployment +templates: + # secrets.yaml must be loaded for the checksum/secrets include to resolve + - templates/control-plane-deployment.yaml + - templates/secrets.yaml +release: + name: sam-mesh +tests: + - it: pins the numeric distroless uid so runAsNonRoot is verifiable + template: templates/control-plane-deployment.yaml + asserts: + - equal: + path: spec.template.spec.securityContext.runAsNonRoot + value: true + - equal: + path: spec.template.spec.securityContext.runAsUser + value: 65532 + + - it: rolls the pods when the shared secret changes + template: templates/control-plane-deployment.yaml + set: + controlPlane.adminToken: aaa + database.postgres.password: p + asserts: + - matchRegex: + path: spec.template.metadata.annotations["checksum/secrets"] + pattern: ^[0-9a-f]{64}$ + + - it: uses the default image tag when none is pinned + template: templates/control-plane-deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: sam-control-plane:local + + - it: external postgres points the DSN at the configured host + template: templates/control-plane-deployment.yaml + set: + database.postgres.deployInternal: false + database.postgres.host: ext-db + asserts: + - matchRegex: + path: spec.template.spec.containers[0].args[2] + pattern: "@ext-db:5432" + + - it: scheduling knobs pass through to the pod spec + template: templates/control-plane-deployment.yaml + set: + controlPlane.nodeSelector.disk: ssd + controlPlane.tolerations: + - key: dedicated + asserts: + - equal: + path: spec.template.spec.nodeSelector.disk + value: ssd + - contains: + path: spec.template.spec.tolerations + content: + key: dedicated diff --git a/charts/sam-mesh/tests/db-statefulset_test.yaml b/charts/sam-mesh/tests/db-statefulset_test.yaml new file mode 100644 index 00000000..7d3090e7 --- /dev/null +++ b/charts/sam-mesh/tests/db-statefulset_test.yaml @@ -0,0 +1,37 @@ +suite: db statefulset +templates: + - templates/db-statefulset.yaml +release: + name: sam-mesh +tests: + - it: probes postgres with pg_isready + documentSelector: + path: kind + value: StatefulSet + asserts: + - equal: + path: spec.template.spec.containers[0].readinessProbe.exec.command[0] + value: pg_isready + - equal: + path: spec.template.spec.containers[0].livenessProbe.exec.command[0] + value: pg_isready + + - it: scheduling knobs pass through to the pod spec + set: + database.postgres.tolerations: + - key: dedicated + documentSelector: + path: kind + value: StatefulSet + asserts: + - contains: + path: spec.template.spec.tolerations + content: + key: dedicated + + - it: external postgres renders nothing + set: + database.postgres.deployInternal: false + asserts: + - hasDocuments: + count: 0 diff --git a/charts/sam-mesh/tests/gateway_test.yaml b/charts/sam-mesh/tests/gateway_test.yaml new file mode 100644 index 00000000..23b69ced --- /dev/null +++ b/charts/sam-mesh/tests/gateway_test.yaml @@ -0,0 +1,62 @@ +suite: gateway +templates: + - templates/gateway.yaml +release: + name: sam-mesh +tests: + - it: disabled by default renders nothing + asserts: + - hasDocuments: + count: 0 + + - it: enabling it requires gateway.className + set: + gateway.enabled: true + asserts: + - failedTemplate: + errorPattern: gateway.className is required when gateway.enabled is true + + - it: routes enrollment plus the console redirect and rewrite by default + set: + gateway.enabled: true + gateway.className: x + documentSelector: + path: kind + value: HTTPRoute + asserts: + # enrollment surface + consolePath 302 + consolePath URLRewrite + - lengthEqual: + path: spec.rules + count: 3 + + - it: empty consolePath leaves the console unrouted + set: + gateway.enabled: true + gateway.className: x + gateway.consolePath: "" + documentSelector: + path: kind + value: HTTPRoute + asserts: + - lengthEqual: + path: spec.rules + count: 1 + + - it: adminRoute adds the /admin rule + set: + gateway.enabled: true + gateway.className: x + gateway.adminRoute: true + documentSelector: + path: kind + value: HTTPRoute + asserts: + - lengthEqual: + path: spec.rules + count: 4 + - contains: + path: spec.rules[1].matches + content: + path: + type: PathPrefix + value: /admin diff --git a/charts/sam-mesh/tests/router-statefulset_test.yaml b/charts/sam-mesh/tests/router-statefulset_test.yaml new file mode 100644 index 00000000..eb54ee90 --- /dev/null +++ b/charts/sam-mesh/tests/router-statefulset_test.yaml @@ -0,0 +1,109 @@ +suite: router statefulset +templates: + - templates/router-statefulset.yaml +release: + # collapses fullname to "sam-mesh" (see _helpers.tpl) + name: sam-mesh +tests: + - it: pins the numeric distroless uid so runAsNonRoot is verifiable + documentSelector: + path: kind + value: StatefulSet + asserts: + - equal: + path: spec.template.spec.securityContext.runAsNonRoot + value: true + - equal: + path: spec.template.spec.securityContext.runAsUser + value: 65532 + - equal: + path: spec.template.spec.initContainers[0].securityContext.runAsUser + value: 100 + + - it: announces the service DNS address when hostPort is unset + documentSelector: + path: kind + value: StatefulSet + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --external-addr=/dns4/sam-mesh-router/tcp/4501 + - notExists: + path: spec.template.spec.containers[0].ports[0].hostPort + + - it: hostPort binds both protocols and announces the node IP + set: + router.hostPort: 4501 + documentSelector: + path: kind + value: StatefulSet + asserts: + - equal: + path: spec.template.spec.containers[0].ports[0].hostPort + value: 4501 + - equal: + path: spec.template.spec.containers[0].ports[1].hostPort + value: 4501 + - contains: + path: spec.template.spec.containers[0].args + content: --external-addr=/ip4/$(HOST_IP)/tcp/4501 + - contains: + path: spec.template.spec.containers[0].args + content: --external-addr=/ip4/$(HOST_IP)/udp/4501/quic-v1 + + - it: externalAddrs overrides the derived announcement + set: + router.hostPort: 4501 + router.externalAddrs: + - /dns4/x/tcp/1 + documentSelector: + path: kind + value: StatefulSet + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --external-addr=/dns4/x/tcp/1 + - notContains: + path: spec.template.spec.containers[0].args + content: --external-addr=/ip4/$(HOST_IP)/tcp/4501 + + - it: keeps the key on a PVC, not emptyDir + documentSelector: + path: kind + value: StatefulSet + asserts: + - equal: + path: spec.volumeClaimTemplates[0].metadata.name + value: router-data + - equal: + path: spec.volumeClaimTemplates[0].spec.resources.requests.storage + value: 1Gi + - notExists: + path: spec.template.spec.volumes + + - it: useOidcToken swaps the bootstrap token for a projected JWT + set: + router.useOidcToken: true + documentSelector: + path: kind + value: StatefulSet + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --jwt-path=/var/run/secrets/tokens/sam-token + - notContains: + path: spec.template.spec.containers[0].args + content: --bootstrap-token=$(BOOTSTRAP_TOKEN) + - exists: + path: spec.template.spec.volumes[0].projected + + - it: image.tag overrides global.imageTag for the router only + set: + router.image.tag: v9 + documentSelector: + path: kind + value: StatefulSet + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: sam-router:v9 From 291396098f3f8daed4388cdef776b3a4cd5fb817 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 24 Aug 2026 20:01:12 +0000 Subject: [PATCH 19/26] ci: lint and unit-test the chart on chart changes Runs make helm-lint and make helm-test (which installs the helm-unittest plugin on first use); helm ships on the ubuntu runners. Path-filtered to charts/ so it stays out of unrelated PRs. --- .github/workflows/chart-test.yml | 46 ++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/chart-test.yml diff --git a/.github/workflows/chart-test.yml b/.github/workflows/chart-test.yml new file mode 100644 index 00000000..764454e6 --- /dev/null +++ b/.github/workflows/chart-test.yml @@ -0,0 +1,46 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: chart-test + +on: + push: + branches: + - "main" + paths: + - "charts/**" + - ".github/workflows/chart-test.yml" + pull_request: + branches: [main] + paths: + - "charts/**" + - ".github/workflows/chart-test.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + chart_test: + runs-on: ubuntu-22.04 + name: Chart lint and unit tests + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Lint chart + run: make helm-lint + + - name: Run chart unit tests + run: make helm-test From e356126b0cd2131c4c331854ae24c906c64ae80d Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Mon, 24 Aug 2026 21:00:06 +0000 Subject: [PATCH 20/26] Add helm-template make target. Reduce comments --- Makefile | 6 ++++++ charts/sam-mesh/README.md | 18 +++++++++--------- charts/sam-mesh/values.yaml | 11 +++-------- development/kind/run.sh | 2 -- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index aa965981..39afef81 100644 --- a/Makefile +++ b/Makefile @@ -194,6 +194,12 @@ helm-lint: fi; \ $$HELM_BIN lint ./charts/sam-mesh +# render the chart to bin/chart/ for inspection; pass extra flags via ARGS, e.g. ARGS="--set gateway.enabled=true" +.PHONY: helm-template +helm-template: + rm -rf bin/chart + helm template sam-mesh ./charts/sam-mesh --output-dir bin/chart $(ARGS) + lint: fmt helm-lint hack/lint.sh diff --git a/charts/sam-mesh/README.md b/charts/sam-mesh/README.md index 4399b93e..600e7330 100644 --- a/charts/sam-mesh/README.md +++ b/charts/sam-mesh/README.md @@ -2,7 +2,7 @@ Deploys a self-contained SAM mesh (control plane, router, console, and an in-cluster Postgres) for local development, testing, or self-hosting your -own hub. +own mesh. > For large-scale production deployments (GKE/EKS/AKS) using externally > managed Postgres/DNS/OIDC, see the @@ -81,11 +81,11 @@ The route exposes only the control plane's enrollment surface (`/register`, including `/admin` and `/user`, is unrouted. `gateway.adminRoute: true` additionally routes `/admin` — a dev convenience, leave it off in production. -The console rules mirror the cloud deployment exactly: the bare prefix -(`/console`) is answered with a 302 to `/console/`, and a `URLRewrite` filter -strips the prefix before the request reaches the console. `URLRewrite` is -**Extended** (not core) Gateway API conformance, so the provider must support -it. Set `gateway.consolePath: ""` to leave the console unrouted. +For the console, the bare prefix (`/console`) is answered with a 302 to +`/console/`, and a `URLRewrite` filter strips the prefix before the request +reaches the console. `URLRewrite` is **Extended** (not core) Gateway API +conformance, so the provider must support it. Set `gateway.consolePath: ""` +to leave the console unrouted. `listeners`, `hostnames`, `addresses` and `annotations` are passed through to the Gateway API objects verbatim, so anything the spec allows is expressible. @@ -106,14 +106,14 @@ gateway: allowedRoutes: namespaces: from: Same - hostnames: [hub.example.com] + hostnames: [sam.example.com] addresses: - type: NamedAddress - value: sam-hub-ip + value: sam-cp-ip ``` ## OIDC login for the console There is no bundled Dex. Point `controlPlane.oidcIssuer` at your identity -provider and register `https:///auth/callback` as +provider and register `https:///auth/callback` as a redirect URI for the OIDC client the control plane reports. diff --git a/charts/sam-mesh/values.yaml b/charts/sam-mesh/values.yaml index 1a82c2d3..75f11b5e 100644 --- a/charts/sam-mesh/values.yaml +++ b/charts/sam-mesh/values.yaml @@ -3,21 +3,16 @@ global: imagePullPolicy: IfNotPresent # One Gateway fronting the mesh: the control plane's enrollment surface plus the console -# under gateway.consolePath, mirroring the cloud deployment. +# under gateway.consolePath. gateway: enabled: false # Required when enabled. No default: the GatewayClass is provider-specific, e.g. # cloud-provider-kind in kind, gke-l7-global-external-managed on GKE. className: "" - # Routes the control plane's /admin API. Dev only; the cloud leaves it unrouted. + # Routes the control plane's /admin API. Dev only. adminRoute: false - # External prefix the console is served under. The bare prefix 302s to / and a - # URLRewrite filter (Extended conformance) strips it before the console sees the - # request, as on the cloud. Empty leaves the console unrouted. + # External prefix the console is served under. Empty leaves the console unrouted. consolePath: /console - # listeners, hostnames, addresses and annotations pass through to the Gateway API - # objects verbatim, so anything the spec allows is expressible. Plain HTTP on 80 - # matching any host suits a local cluster. listeners: - name: http protocol: HTTP diff --git a/development/kind/run.sh b/development/kind/run.sh index 35b94939..e8f1e178 100755 --- a/development/kind/run.sh +++ b/development/kind/run.sh @@ -15,8 +15,6 @@ HELM="helm" # cloud-provider-kind GatewayClass itself, so the cluster needs no CRD step. CPK_CONTAINER="cloud-provider-kind" CPK_IMAGE="registry.k8s.io/cloud-provider-kind/cloud-controller-manager:v0.11.1" -# The console shares the control plane's gateway under this prefix, as it does on the -# cloud: the route rewrites the prefix away before the console sees the request. CONSOLE_BASE_PATH="/console" From 2ebbe51841d6db436956bdc3e4b772b1f8ff4ab0 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Tue, 25 Aug 2026 06:01:14 +0000 Subject: [PATCH 21/26] tests/e2e: recover the reused cluster from immutable StatefulSet changes The lane reuses the sam-wi-test kind cluster across runs, so upgrading over a release from before the router volumeClaimTemplates change fails with 'updates to statefulset spec ... are forbidden'. On upgrade failure delete the StatefulSets (PVCs survive) and retry. Fresh clusters, as on hosted CI, never take the retry path. --- tests/e2e/lib/container_mesh.bash | 36 ++++++++++++++++++------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/tests/e2e/lib/container_mesh.bash b/tests/e2e/lib/container_mesh.bash index 22fb3bf3..baf633fb 100644 --- a/tests/e2e/lib/container_mesh.bash +++ b/tests/e2e/lib/container_mesh.bash @@ -367,21 +367,27 @@ if [[ -z "${MESH_HELPERS_LOADED:-}" ]]; then # announce that name. Announcing a node IP instead is unroutable from an isolated test network. # bootstrap.nodeServices: policy.bats asserts the control plane denies an ungranted service, so # this lane pins the grants it tests rather than inheriting the chart default. - "${helm_bin}" --kube-context="${KUBECONTEXT}" upgrade --install sam ./charts/sam-mesh \ - --namespace default \ - --set fullnameOverride="sam" \ - --set global.imageTag="local" \ - --set controlPlane.oidcIssuer="${ISSUERS//,/\\,}" \ - --set controlPlane.allowedAudiences="sam-mesh-audience\,sam-control-plane-audience" \ - --set controlPlane.insecureSkipTlsVerify=true \ - --set controlPlane.adminToken="super-secret-admin-token" \ - --set controlPlane.replicaCount=2 \ - --set controlPlane.hostPort=8080 \ - --set router.useOidcToken=false \ - --set router.hostPort=4501 \ - --set console.enabled=false \ - --set 'router.externalAddrs={/dns4/sam-router/tcp/4501}' \ - --set 'bootstrap.nodeServices={mcp://calculator,mcp://db-agent,mcp://http-tool,mcp://stdio-tool,system://sam.catalog}' + local helm_args=(--kube-context="${KUBECONTEXT}" upgrade --install sam ./charts/sam-mesh + --namespace default + --set fullnameOverride="sam" + --set global.imageTag="local" + --set controlPlane.oidcIssuer="${ISSUERS//,/\\,}" + --set controlPlane.allowedAudiences="sam-mesh-audience\,sam-control-plane-audience" + --set controlPlane.insecureSkipTlsVerify=true + --set controlPlane.adminToken="super-secret-admin-token" + --set controlPlane.replicaCount=2 + --set controlPlane.hostPort=8080 + --set router.useOidcToken=false + --set router.hostPort=4501 + --set console.enabled=false + --set 'router.externalAddrs={/dns4/sam-router/tcp/4501}' + --set 'bootstrap.nodeServices={mcp://calculator,mcp://db-agent,mcp://http-tool,mcp://stdio-tool,system://sam.catalog}') + if ! "${helm_bin}" "${helm_args[@]}"; then + # The reused cluster may hold StatefulSets whose immutable spec (e.g. + # volumeClaimTemplates) changed; drop them (PVCs survive) and retry. + kubectl --context="${KUBECONTEXT}" delete statefulset sam-router sam-db --ignore-not-found + "${helm_bin}" "${helm_args[@]}" + fi mesh_wait_for_rollout statefulset/sam-db mesh_wait_for_rollout deployment/sam-control-plane From 54618a8fbbd21fa56f34a8db5daf29df9ce91ec4 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Tue, 25 Aug 2026 11:24:57 +0000 Subject: [PATCH 22/26] charts: derive announced router port from service.port; optional PVC storageClass --- .../templates/router-statefulset.yaml | 5 +++- .../tests/router-statefulset_test.yaml | 24 +++++++++++++++++++ charts/sam-mesh/values.yaml | 2 ++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/charts/sam-mesh/templates/router-statefulset.yaml b/charts/sam-mesh/templates/router-statefulset.yaml index 3a11b95a..b4c51f39 100644 --- a/charts/sam-mesh/templates/router-statefulset.yaml +++ b/charts/sam-mesh/templates/router-statefulset.yaml @@ -124,7 +124,7 @@ spec: - "--external-addr=/ip4/$(HOST_IP)/tcp/{{ .Values.router.hostPort }}" - "--external-addr=/ip4/$(HOST_IP)/udp/{{ .Values.router.hostPort }}/quic-v1" {{- else }} - - "--external-addr=/dns4/{{ include "sam-mesh.fullname" . }}-router/tcp/4501" + - "--external-addr=/dns4/{{ include "sam-mesh.fullname" . }}-router/tcp/{{ .Values.router.service.port }}" {{- end }} - "--keys-path=/data/router.key" {{- if .Values.router.allowLoopback }} @@ -159,6 +159,9 @@ spec: name: router-data spec: accessModes: [ "ReadWriteOnce" ] + {{- if .Values.router.storageClass }} + storageClassName: {{ .Values.router.storageClass | quote }} + {{- end }} resources: requests: storage: {{ .Values.router.storageSize | quote }} diff --git a/charts/sam-mesh/tests/router-statefulset_test.yaml b/charts/sam-mesh/tests/router-statefulset_test.yaml index eb54ee90..a29bc394 100644 --- a/charts/sam-mesh/tests/router-statefulset_test.yaml +++ b/charts/sam-mesh/tests/router-statefulset_test.yaml @@ -31,6 +31,17 @@ tests: - notExists: path: spec.template.spec.containers[0].ports[0].hostPort + - it: announced DNS address follows router.service.port + set: + router.service.port: 9999 + documentSelector: + path: kind + value: StatefulSet + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --external-addr=/dns4/sam-mesh-router/tcp/9999 + - it: hostPort binds both protocols and announces the node IP set: router.hostPort: 4501 @@ -78,9 +89,22 @@ tests: - equal: path: spec.volumeClaimTemplates[0].spec.resources.requests.storage value: 1Gi + - notExists: + path: spec.volumeClaimTemplates[0].spec.storageClassName - notExists: path: spec.template.spec.volumes + - it: storageClass sets the PVC storageClassName + set: + router.storageClass: fast-ssd + documentSelector: + path: kind + value: StatefulSet + asserts: + - equal: + path: spec.volumeClaimTemplates[0].spec.storageClassName + value: fast-ssd + - it: useOidcToken swaps the bootstrap token for a projected JWT set: router.useOidcToken: true diff --git a/charts/sam-mesh/values.yaml b/charts/sam-mesh/values.yaml index 75f11b5e..4ae7afc2 100644 --- a/charts/sam-mesh/values.yaml +++ b/charts/sam-mesh/values.yaml @@ -127,6 +127,8 @@ router: allowLoopback: false # PVC for /data/router.key, so the libp2p peer ID survives rescheduling. storageSize: 1Gi + # PVC StorageClass; cluster default when null. + storageClass: null resources: requests: cpu: 100m From 678ea5f854d26042e365de91629d604b12bbecbd Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Tue, 25 Aug 2026 11:46:48 +0000 Subject: [PATCH 23/26] charts: disable surge when controlPlane.hostPort is set The default RollingUpdate surges a replacement pod before freeing the old pod's hostPort, so on nodes where every schedulable node already runs a replica the rollout deadlocks (FailedScheduling: no free ports). Seen on the reused e2e kind cluster: 2 replicas, 2 workers, hostPort 8080. --- .../templates/control-plane-deployment.yaml | 8 ++++++++ .../tests/control-plane-deployment_test.yaml | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/charts/sam-mesh/templates/control-plane-deployment.yaml b/charts/sam-mesh/templates/control-plane-deployment.yaml index 2f371037..96d76d8d 100644 --- a/charts/sam-mesh/templates/control-plane-deployment.yaml +++ b/charts/sam-mesh/templates/control-plane-deployment.yaml @@ -6,6 +6,14 @@ metadata: {{- include "sam-mesh.labels" . | nindent 4 }} spec: replicas: {{ .Values.controlPlane.replicaCount }} + {{- if .Values.controlPlane.hostPort }} + # hostPort: a surged pod can't bind a port the old pod still holds, so the + # default maxSurge=1 deadlocks the rollout on port-saturated nodes. + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + {{- end }} selector: matchLabels: app: {{ include "sam-mesh.fullname" . }}-control-plane diff --git a/charts/sam-mesh/tests/control-plane-deployment_test.yaml b/charts/sam-mesh/tests/control-plane-deployment_test.yaml index 7b9f34d4..238f010b 100644 --- a/charts/sam-mesh/tests/control-plane-deployment_test.yaml +++ b/charts/sam-mesh/tests/control-plane-deployment_test.yaml @@ -26,6 +26,24 @@ tests: path: spec.template.metadata.annotations["checksum/secrets"] pattern: ^[0-9a-f]{64}$ + - it: hostPort disables surge so rollouts can rebind the node port + template: templates/control-plane-deployment.yaml + set: + controlPlane.hostPort: 8080 + asserts: + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 0 + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 1 + + - it: no strategy override without hostPort + template: templates/control-plane-deployment.yaml + asserts: + - notExists: + path: spec.strategy + - it: uses the default image tag when none is pinned template: templates/control-plane-deployment.yaml asserts: From e4116b19e8ffe2ab9aa4c2de2f64a1b5ff59aee3 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Tue, 25 Aug 2026 12:36:01 +0000 Subject: [PATCH 24/26] Makefile: drop --verify=false from helm plugin install; helm 3 has no such flag --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 39afef81..f2e4ca78 100644 --- a/Makefile +++ b/Makefile @@ -206,7 +206,7 @@ lint: fmt helm-lint # fast chart template checks; no cluster needed .PHONY: helm-test helm-test: - @helm plugin list 2>/dev/null | grep -q '^unittest' || helm plugin install https://github.com/helm-unittest/helm-unittest --verify=false + @helm plugin list 2>/dev/null | grep -q '^unittest' || helm plugin install https://github.com/helm-unittest/helm-unittest helm unittest charts/sam-mesh .PHONY: verify From d7a70667dfbb2d114cb78ebc4a15c300f778ae51 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Tue, 25 Aug 2026 14:59:07 +0000 Subject: [PATCH 25/26] ci: drop helm-lint from chart-test, test.yaml already runs it --- .github/workflows/chart-test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/chart-test.yml b/.github/workflows/chart-test.yml index 764454e6..64825839 100644 --- a/.github/workflows/chart-test.yml +++ b/.github/workflows/chart-test.yml @@ -39,8 +39,6 @@ jobs: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Lint chart - run: make helm-lint - + # helm-lint already runs in test.yaml - name: Run chart unit tests run: make helm-test From 7b3c17288a9142f5658bb0ee46cef8ce773918ca Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Tue, 25 Aug 2026 14:59:07 +0000 Subject: [PATCH 26/26] charts: let the bootstrap job update the router-token secret Server-side apply falls back to update when the object exists, so patch alone is not always enough; still pinned to the router-token secret via resourceNames. --- charts/sam-mesh/templates/bootstrap-job.yaml | 2 +- charts/sam-mesh/tests/bootstrap-job_test.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/sam-mesh/templates/bootstrap-job.yaml b/charts/sam-mesh/templates/bootstrap-job.yaml index 1a2f2f58..1365c176 100644 --- a/charts/sam-mesh/templates/bootstrap-job.yaml +++ b/charts/sam-mesh/templates/bootstrap-job.yaml @@ -20,7 +20,7 @@ rules: - apiGroups: [""] resources: ["secrets"] resourceNames: ["{{ include "sam-mesh.fullname" . }}-router-token"] - verbs: ["get", "patch"] + verbs: ["get", "patch", "update"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/charts/sam-mesh/tests/bootstrap-job_test.yaml b/charts/sam-mesh/tests/bootstrap-job_test.yaml index 9be914f7..621e0dc7 100644 --- a/charts/sam-mesh/tests/bootstrap-job_test.yaml +++ b/charts/sam-mesh/tests/bootstrap-job_test.yaml @@ -17,7 +17,7 @@ tests: value: [sam-mesh-router-token] - equal: path: rules[1].verbs - value: [get, patch] + value: [get, patch, update] - lengthEqual: path: rules count: 2