feat(kargo): add a progressive delivery pipeline with a verification gate

.bootstrap/kargo/manifests/git-credentials.template.yaml

@@ -0,0 +1,22 @@
+# Rendered by up.sh, which substitutes <placeholder> with the base64 of
+# GITHUB_TOKEN from .env.
+#
+# Kept outside any Argo CD source path -- it holds a real token, and the
+# backstage-secrets template is in the repo for exactly the same reason.
+#
+# Kargo finds credentials by label, not by name: kargo.akuity.io/cred-type=git
+# marks this Secret as Git credentials, and Kargo matches it to a repository by
+# the repoURL key. HTTPS is used rather than SSH because a token is what .env
+# already provides.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: repo-credentials
+  namespace: microservice-delivery
+  labels:
+    kargo.akuity.io/cred-type: git
+type: Opaque
+data:
+  repoURL: <repo-url-placeholder>
+  username: <username-placeholder>
+  password: <placeholder>

.bootstrap/kargo/up.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Change to platform cluster
+if [[ "$(kubectl config current-context)" != "kind-platform" ]]; then
+    kubectl config use-context kind-platform || {
+        echo "Failed to switch context to kind-platform"
+        exit 1
+    }
+fi
+
+NS=kargo-system
+PROJECT_NS=microservice-delivery
+BASE_DIR="$(dirname "$0")"
+KARGO_VERSION=1.11.2
+ROLLOUTS_VERSION=v1.7.2
+PORT=3002
+REPO_URL="https://github.com/koorikla/platform-engineering-backstack.git"
+GIT_USERNAME=koorikla
+
+# Kargo delegates verification to Argo Rollouts' analysis engine: the
+# AnalysisTemplate/AnalysisRun kinds live in argoproj.io, not kargo.akuity.io.
+# Kargo's chart checks for those CRDs at startup and silently disables the
+# integration when they are missing, so the quality gate would never run and
+# nothing would say why. Only the controller and CRDs are needed -- no Rollout
+# resources are used.
+echo "Installing Argo Rollouts (provides the AnalysisTemplate CRD Kargo verifies with)..."
+kubectl create namespace argo-rollouts --dry-run=client -o yaml | kubectl apply -f -
+kubectl apply -n argo-rollouts -f \
+    "https://github.com/argoproj/argo-rollouts/releases/download/${ROLLOUTS_VERSION}/install.yaml" >/dev/null
+
+echo "Waiting for the AnalysisTemplate CRD to be established..."
+kubectl wait --for=condition=established --timeout=120s \
+    crd/analysistemplates.argoproj.io crd/analysisruns.argoproj.io || {
+    echo "❌ Argo Rollouts CRDs did not become established"
+    exit 1
+}
+
+echo "Installing or upgrading Kargo..."
+helm upgrade --install kargo \
+    oci://ghcr.io/akuity/kargo-charts/kargo \
+    --version "$KARGO_VERSION" \
+    --namespace "$NS" \
+    --create-namespace \
+    --set api.service.type=ClusterIP \
+    --set api.adminAccount.passwordHash='$2a$10$Zrhhie4vLz5ygtVSaif6o.qN36jgs6vjtHbdWoYjX4uMe3Q8hnfsy' \
+    --set api.adminAccount.tokenSigningKey=kargo-local-dev-signing-key \
+    --set api.rollouts.integrationEnabled=true \
+    --wait --timeout 5m
+
+echo "Waiting for Kargo to be ready..."
+kubectl wait --for=condition=available --timeout=180s -n "$NS" deployment --all || {
+    echo "❌ Kargo deployments are not ready"
+    exit 1
+}
+
+# The Project creates its own namespace, so it must exist before the Secret and
+# the Warehouse/Stages that live in it.
+echo "Applying the Kargo project..."
+kubectl apply -f ./kargo/project.yaml
+for _ in $(seq 1 30); do
+    kubectl get namespace "$PROJECT_NS" >/dev/null 2>&1 && break
+    sleep 2
+done
+
+# Same .env handling as the Backstage bootstrap: sourcing under `set -a` rather
+# than `export $(cat .env | xargs)`, which breaks on a trailing comment.
+if [[ ! -f .env ]]; then
+    echo "❌ .env not found in the repo root. It must define GITHUB_TOKEN."
+    exit 1
+fi
+set -a
+# shellcheck source=/dev/null
+source ./.env
+set +a
+if [[ -z "${GITHUB_TOKEN:-}" ]]; then
+    echo "❌ GITHUB_TOKEN is not set in .env"
+    exit 1
+fi
+
+echo "Creating Kargo Git credentials..."
+b64() { printf '%s' "$1" | base64 | tr -d '\n'; }
+sed -e "s|<repo-url-placeholder>|$(b64 "$REPO_URL")|" \
+    -e "s|<username-placeholder>|$(b64 "$GIT_USERNAME")|" \
+    -e "s|<placeholder>|$(b64 "$GITHUB_TOKEN")|" \
+    "$BASE_DIR/manifests/git-credentials.template.yaml" |
+    kubectl apply -f -
+
+echo "Applying the analysis template, warehouse and stages..."
+kubectl apply -f ./kargo/analysis-templates
+kubectl apply -f ./kargo/warehouse.yaml
+kubectl apply -f ./kargo/stages
+
+# Probe rather than trust lsof: a port-forward whose pod has gone still owns the
+# socket briefly, which would make a port check report a healthy forward.
+if ! curl -s --max-time 3 "http://localhost:$PORT" >/dev/null 2>&1; then
+    pkill -f "port-forward svc/kargo-api" 2>/dev/null || true
+    echo "Starting port-forward for the Kargo UI on port $PORT..."
+    nohup kubectl --namespace "$NS" port-forward svc/kargo-api "$PORT":80 >/dev/null 2>&1 &
+    for _ in $(seq 1 30); do
+        curl -s --max-time 2 "http://localhost:$PORT" >/dev/null 2>&1 && break
+        sleep 1
+    done
+else
+    echo "Port-forward on $PORT is already serving."
+fi
+
+echo "✅ Kargo setup completed successfully!"

Makefile

@@ -35,6 +35,7 @@ up: check_bins
 	@./.bootstrap/crossplane/up.sh
 	@./.bootstrap/crossview/up.sh
 	@./.bootstrap/kyverno/up.sh
+	@./.bootstrap/kargo/up.sh
 
 	@make setup-local-config
 
@@ -43,6 +44,7 @@ up: check_bins
 	@echo "Backstage is accessible at http://localhost:3000"
 	@echo "Argo CD is accessible at http://localhost:8080"
 	@echo "Crossview is accessible at http://localhost:3001"
+	@echo "Kargo is accessible at http://localhost:3002 (admin / admin)"
 	@echo "LocalStack is accessible at http://localhost:4566 (Manage through the platform at: https://app.localstack.cloud/instances)"
 
 down: check_bins

crossplane/namespaces/stages.yaml

@@ -0,0 +1,28 @@
+# Delivery stages for the Kargo pipeline.
+#
+# Separate from tenants.yaml on purpose: team-a/team-b are tenants, these are
+# environments. They are different axes, and a tenant's queues should not be
+# disturbed by a promotion.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: dev1-0
+  labels: { platform.hooli.tech/stage: dev1-0, platform.hooli.tech/stage-class: dev }
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: dev1-1
+  labels: { platform.hooli.tech/stage: dev1-1, platform.hooli.tech/stage-class: dev }
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: tst
+  labels: { platform.hooli.tech/stage: tst, platform.hooli.tech/stage-class: test }
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: prd
+  labels: { platform.hooli.tech/stage: prd, platform.hooli.tech/stage-class: prod }

crossplane/xrs/stages/dev1-0/podinfo.yaml

@@ -0,0 +1,13 @@
+# The artifact Kargo promotes. Its `spec.image` is the single line a promotion
+# rewrites -- Kargo commits the change here and Argo CD syncs it, so the
+# promotion is an ordinary Git commit rather than a direct cluster write.
+apiVersion: sparky.ee/v1alpha1
+kind: XMicroservice
+metadata:
+  name: podinfo
+  namespace: dev1-0
+spec:
+  image: ghcr.io/stefanprodan/podinfo:6.7.0
+  replicas: 1
+  port: 9898
+  serviceType: ClusterIP

crossplane/xrs/stages/dev1-1/podinfo.yaml

@@ -0,0 +1,13 @@
+# The artifact Kargo promotes. Its `spec.image` is the single line a promotion
+# rewrites -- Kargo commits the change here and Argo CD syncs it, so the
+# promotion is an ordinary Git commit rather than a direct cluster write.
+apiVersion: sparky.ee/v1alpha1
+kind: XMicroservice
+metadata:
+  name: podinfo
+  namespace: dev1-1
+spec:
+  image: ghcr.io/stefanprodan/podinfo:6.7.0
+  replicas: 1
+  port: 9898
+  serviceType: ClusterIP

crossplane/xrs/stages/prd/podinfo.yaml

@@ -0,0 +1,13 @@
+# The artifact Kargo promotes. Its `spec.image` is the single line a promotion
+# rewrites -- Kargo commits the change here and Argo CD syncs it, so the
+# promotion is an ordinary Git commit rather than a direct cluster write.
+apiVersion: sparky.ee/v1alpha1
+kind: XMicroservice
+metadata:
+  name: podinfo
+  namespace: prd
+spec:
+  image: ghcr.io/stefanprodan/podinfo:6.7.0
+  replicas: 1
+  port: 9898
+  serviceType: ClusterIP

crossplane/xrs/stages/tst/podinfo.yaml

@@ -0,0 +1,13 @@
+# The artifact Kargo promotes. Its `spec.image` is the single line a promotion
+# rewrites -- Kargo commits the change here and Argo CD syncs it, so the
+# promotion is an ordinary Git commit rather than a direct cluster write.
+apiVersion: sparky.ee/v1alpha1
+kind: XMicroservice
+metadata:
+  name: podinfo
+  namespace: tst
+spec:
+  image: ghcr.io/stefanprodan/podinfo:6.7.0
+  replicas: 1
+  port: 9898
+  serviceType: ClusterIP

kargo/analysis-templates/podinfo-healthy.yaml

@@ -0,0 +1,54 @@
+# The quality gate Kargo runs after each promotion.
+#
+# AnalysisTemplate is an Argo Rollouts kind, not a Kargo one -- Kargo delegates
+# verification to Rollouts' analysis engine, which is why .bootstrap/kargo/up.sh
+# installs the Rollouts CRDs and controller. Kargo's chart self-disables the
+# integration if those CRDs are absent, so the gate would silently never run.
+#
+# The check is deliberately a real one: it calls podinfo's /healthz through the
+# Service that the XMicroservice composition created, in the namespace the
+# promotion just targeted. That exercises the whole chain -- Kargo committed to
+# Git, Argo CD synced, Crossplane composed a Deployment and Service, and pods
+# are actually serving. A bad image fails here and the Freight never becomes
+# eligible for the next stage.
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: podinfo-healthy
+  namespace: microservice-delivery
+spec:
+  args:
+    # Supplied per stage by Stage.spec.verification.args.
+    - name: stage-namespace
+
+  metrics:
+    - name: healthz
+      # Three consecutive successes, so a pod that happens to answer once while
+      # the rollout is still settling is not enough to pass the gate.
+      count: 3
+      interval: 15s
+      # No retries: any failed probe fails the analysis and blocks promotion.
+      failureLimit: 0
+      # Give Argo CD and Crossplane time to reconcile before the first probe.
+      initialDelay: 30s
+      provider:
+        job:
+          spec:
+            backoffLimit: 0
+            template:
+              spec:
+                restartPolicy: Never
+                containers:
+                  - name: probe
+                    image: curlimages/curl:8.11.1
+                    command:
+                      - /bin/sh
+                      - -c
+                    args:
+                      - |
+                        set -eu
+                        URL="http://podinfo.{{args.stage-namespace}}.svc.cluster.local:9898/healthz"
+                        echo "probing $URL"
+                        # --fail makes a non-2xx status a non-zero exit, which is
+                        # what Rollouts reads as a failed metric.
+                        curl --fail --silent --show-error --max-time 10 "$URL"

kargo/project.yaml

@@ -0,0 +1,7 @@
+# A Kargo Project owns a namespace of the same name. Warehouses, Stages and
+# Promotions live here; the workloads they promote live in the stage namespaces
+# (dev1-0, dev1-1, tst, prd).
+apiVersion: kargo.akuity.io/v1alpha1
+kind: Project
+metadata:
+  name: microservice-delivery

kargo/stages/dev1-0.yaml

@@ -0,0 +1,54 @@
+# Stage dev1-0. Accepts straight from the Warehouse -- this is the pipeline's entry point.
+#
+# A promotion is a Git commit, not a cluster write: the steps below clone the
+# repo, rewrite spec.image in this stage's XMicroservice, and push. Argo CD then
+# syncs it, so the cluster state still comes from Git and the promotion is
+# auditable in history.
+apiVersion: kargo.akuity.io/v1alpha1
+kind: Stage
+metadata:
+  name: dev1-0
+  namespace: microservice-delivery
+spec:
+  requestedFreight:
+    - origin:
+        kind: Warehouse
+        name: podinfo
+      sources:
+        direct: true
+
+  promotionTemplate:
+    spec:
+      steps:
+        - uses: git-clone
+          config:
+            repoURL: https://github.com/koorikla/platform-engineering-backstack.git
+            checkout:
+              - branch: main
+                path: ./repo
+
+        - uses: yaml-update
+          config:
+            path: ./repo/crossplane/xrs/stages/dev1-0/podinfo.yaml
+            updates:
+              - key: spec.image
+                value: ${{ imageFrom("ghcr.io/stefanprodan/podinfo").RepoURL }}:${{ imageFrom("ghcr.io/stefanprodan/podinfo").Tag }}
+
+        - uses: git-commit
+          config:
+            path: ./repo
+            message: 'chore(dev1-0): promote podinfo ${{ imageFrom("ghcr.io/stefanprodan/podinfo").Tag }}'
+
+        - uses: git-push
+          config:
+            path: ./repo
+
+  # The quality gate. Runs after the promotion lands and decides whether this
+  # Freight becomes eligible for the next stage -- a failure here stops the
+  # rollout rather than merely reporting it.
+  verification:
+    analysisTemplates:
+      - name: podinfo-healthy
+    args:
+      - name: stage-namespace
+        value: dev1-0