test(golden): harness, three sentinels, semver gate (PR 1/9)

.github/workflows/golden-semver-gate.yml

@@ -0,0 +1,69 @@
+name: Golden Templates Semver Gate
+
+# Run on every PR AND on every merge_group event. The check_semver_bump.py
+# script self-gates: it returns exit 0 when no expected.*.yaml files
+# changed, so an unconditional run is cheap and posts a definitive
+# status on every event.
+#
+# Two pitfalls this trigger configuration avoids — both flavors of the
+# "skipped but required check" problem:
+#
+# 1. on.pull_request.paths: filtering would skip the workflow entirely
+#    when no matching path changed, posting no status, which branch
+#    protection treats as missing/pending. We self-gate in the script
+#    instead so the status always posts.
+#
+# 2. listening on pull_request alone would skip the workflow when a PR
+#    enters the merge queue (which dispatches merge_group, not
+#    pull_request). The merge queue would then treat the required
+#    check as missing/pending and block the merge. We listen on both
+#    events to match build.yml.
+on:
+  pull_request:
+    branches:
+      - develop
+      - "feat/*"
+      - "feat-*"
+  merge_group:
+    types: [checks_requested]
+    branches:
+      - develop
+      - "feat/*"
+      - "feat-*"
+
+permissions:
+  contents: read
+
+jobs:
+  semver-gate:
+    if: github.repository_owner == 'aws'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+      - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
+        with:
+          python-version: "3.11"
+      - name: Resolve base ref
+        id: baseref
+        env:
+          # github.base_ref is set on pull_request events. On merge_group
+          # events it's empty and the target branch lives at
+          # github.event.merge_group.base_ref as a full ref ("refs/heads/develop");
+          # strip the prefix to match the bare branch name pull_request emits.
+          PR_BASE_REF: ${{ github.base_ref }}
+          MG_BASE_REF: ${{ github.event.merge_group.base_ref }}
+        run: |
+          if [ -n "$PR_BASE_REF" ]; then
+            echo "base=$PR_BASE_REF" >> "$GITHUB_OUTPUT"
+          else
+            echo "base=${MG_BASE_REF#refs/heads/}" >> "$GITHUB_OUTPUT"
+          fi
+      - name: Run semver gate
+        env:
+          BASE_REF: ${{ steps.baseref.outputs.base }}
+        run: |
+          python tests/golden/check_semver_bump.py \
+            --base "origin/${BASE_REF}" \
+            --head HEAD

Makefile

@@ -29,6 +29,9 @@ test:
 	# Run unit tests (excluding cfn_language_extensions) and fail if coverage falls below 94%
 	@echo "NOTE: Excluding cfn_language_extensions tests. Use 'make test-all' for full coverage."
 	pytest --cov samcli --cov schema --cov-report term-missing --cov-fail-under 94 tests/unit --ignore=tests/unit/lib/cfn_language_extensions --cov-config=.coveragerc_no_lang_ext
+	# Golden-template suite runs as a separate pytest invocation so it
+	# does not enter the samcli coverage calculation above.
+	pytest tests/golden
 
 test-lang-ext:
 	# Run cfn_language_extensions unit tests with coverage
@@ -37,6 +40,9 @@ test-lang-ext:
 test-all:
 	# Run all unit tests including cfn_language_extensions
 	pytest --cov samcli --cov schema --cov-report term-missing --cov-fail-under 94 tests/unit
+	# Golden-template suite runs as a separate pytest invocation so it
+	# does not enter the samcli coverage calculation above.
+	pytest tests/golden
 
 test-cov-report:
 	# Run all unit tests with html coverage report

tests/golden/README.md

@@ -0,0 +1,65 @@
+# Golden Templates
+
+Pinned outputs of `sam build` and `sam package` for representative templates.
+Catches output-shape regressions in SAM-CLI's local pipeline. See
+[`designs/golden-templates-and-semver-check.md`](../../designs/golden-templates-and-semver-check.md)
+for the full design.
+
+## Layout
+
+```
+tests/golden/
+├── templates/
+│   ├── sam_resources/<case>/         # AWS::Serverless::* coverage
+│   ├── packageable_resources/<case>/ # Raw CFN resource coverage
+│   ├── language_extensions/<case>/   # Fn::ForEach et al
+│   └── cross_cutting/<case>/         # Nested stacks, AWS::Include, etc.
+└── ... (harness code)
+```
+
+Each case directory contains:
+- `template.yaml` — the input template.
+- `metadata.yaml` — `language_extensions: bool`, `description`, `issue_refs`.
+- `src/` — any source files referenced by `CodeUri` etc.
+- `expected.build.yaml` — pinned post-build template (regenerated by harness).
+- `expected.package.yaml` — pinned post-package template.
+- `README.md` — what this case covers and why it's pinned.
+
+## Adding a new case
+
+1. Create the case directory and author `template.yaml`, `metadata.yaml`, `src/`, `README.md`.
+2. Generate the pinned outputs:
+   ```
+   python tests/golden/update_goldens.py --new --filter '<axis>/<case-name>'
+   ```
+3. Eyeball `expected.{build,package}.yaml`. Confirm the shape is what you expect.
+4. Commit everything together.
+
+The semver gate treats new cases as additions and does not require a version bump.
+
+## Re-pinning an existing case
+
+If a SAM-CLI behavior change intentionally alters the expanded output:
+
+```
+python tests/golden/update_goldens.py --filter '<axis>/<case-name>'
+```
+
+Review the diff in your IDE before committing. **Modifying or deleting an
+existing pin requires a major version bump in `samcli/__init__.py` in the
+same PR**, enforced by `.github/workflows/golden-semver-gate.yml`.
+
+## Inspecting differences
+
+```
+python tests/golden/update_goldens.py --diff --filter '<glob>'   # show would-be changes
+python tests/golden/update_goldens.py --check                    # exit 1 if anything would change
+```
+
+## Running the suite
+
+```
+pytest tests/golden                                              # all cases
+pytest tests/golden -k foreach_static                            # one case
+make test-all                                                    # part of make pr
+```

tests/golden/check_semver_bump.py

@@ -0,0 +1,159 @@
+"""Semver gate: require a major version bump when an existing pinned
+golden output is modified or deleted.
+
+Usage (CI):
+    python tests/golden/check_semver_bump.py --base origin/develop --head HEAD
+"""
+
+from __future__ import annotations
+
+# Allow direct script invocation (`python tests/golden/check_semver_bump.py`)
+# in addition to module form (`python -m tests.golden.check_semver_bump`).
+# When run as a script, Python does not auto-add the repo root to sys.path,
+# so absolute imports rooted at `tests.golden...` would fail. This script
+# happens to have no such imports today, but keep the guard symmetrical with
+# update_goldens.py so future imports do not silently regress the script
+# form. mypy treats the __main__ guard as always-False, hence the
+# `type: ignore`.
+if __name__ == "__main__" and __package__ is None:
+    import sys  # type: ignore[unreachable]  # noqa: E402
+    from pathlib import Path  # noqa: E402
+
+    sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+import argparse
+import re
+import subprocess
+import sys
+from dataclasses import dataclass
+from typing import List, Optional, Tuple
+
+EXPECTED_GLOB = re.compile(r"^tests/golden/templates/.+/expected\.(build|package)\.yaml$")
+
+# git diff --name-status emits "R<score>\tOLD\tNEW" for renames or
+# "C<score>\tOLD\tNEW" for copies (3 fields each).
+_RENAME_OR_COPY_PARTS = 3
+
+
+@dataclass(frozen=True)
+class Change:
+    path: str
+    status: str  # "A" added, "M" modified, "D" deleted
+
+
+def _parse_version(s: str) -> Tuple[int, int, int]:
+    m = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)(?:.*)?", s.strip())
+    if not m:
+        raise ValueError(f"unparseable version: {s!r}")
+    return int(m.group(1)), int(m.group(2)), int(m.group(3))
+
+
+def _is_corpus_pin(path: str) -> bool:
+    return bool(EXPECTED_GLOB.match(path))
+
+
+def check(
+    changed: List[Change],
+    base_version: str,
+    head_version: str,
+) -> Tuple[int, str]:
+    """Return (exit_code, message)."""
+    relevant = [c for c in changed if _is_corpus_pin(c.path)]
+
+    if not relevant:
+        return 0, "No corpus pin changes; gate passes."
+
+    modifications = [c for c in relevant if c.status == "M"]
+    deletions = [c for c in relevant if c.status == "D"]
+    additions = [c for c in relevant if c.status == "A"]
+
+    if not (modifications or deletions):
+        # Additions only — no bump required.
+        return 0, f"{len(additions)} new pin(s); no version bump required."
+
+    base_major, _, _ = _parse_version(base_version)
+    head_major, _, _ = _parse_version(head_version)
+
+    if head_major > base_major:
+        return 0, "Major version bumped; gate passes."
+
+    suggested = f"{base_major + 1}.0.0"
+    summary_lines = [
+        "Semver gate FAILED.",
+        f"  base version: {base_version}",
+        f"  head version: {head_version}",
+        f"  required:     major bump (suggested {suggested})",
+        "",
+        "Reason: an existing pinned golden output was modified or deleted.",
+        "  Modifications:",
+        *[f"    M  {c.path}" for c in modifications],
+        "  Deletions:",
+        *[f"    D  {c.path}" for c in deletions],
+        "",
+        f'To fix: edit samcli/__init__.py and set __version__ = "{suggested}".',
+        "If the change is intentional, the major bump signals that to consumers.",
+        "If unintentional, run python tests/golden/update_goldens.py --diff to inspect.",
+    ]
+    return 1, "\n".join(summary_lines)
+
+
+def _read_version_at_ref(ref: str) -> str:
+    """Read __version__ from samcli/__init__.py at a git ref."""
+    out = subprocess.check_output(["git", "show", f"{ref}:samcli/__init__.py"], text=True)
+    m = re.search(r'__version__\s*=\s*"([^"]+)"', out)
+    if not m:
+        raise RuntimeError(f"cannot find __version__ at {ref}")
+    return m.group(1)
+
+
+def _git_changed_files(base: str, head: str) -> List[Change]:
+    """Return all files changed between base and head with their git status."""
+    out = subprocess.check_output(["git", "diff", "--name-status", f"{base}...{head}"], text=True)
+    changes: List[Change] = []
+    for line in out.splitlines():
+        if not line.strip():
+            continue
+        parts = line.split("\t")
+        status, path = parts[0], parts[-1]
+        # Renames present as "R<score>\tOLD\tNEW" and copies as
+        # "C<score>\tOLD\tNEW". Note the asymmetry:
+        #   - Rename: source path goes away (D) and target appears (A).
+        #   - Copy:   source persists, only the target is new (A only).
+        # Without this branch, statuses like "R100" / "C100" fall through
+        # to the else and silently bypass the gate's A/M/D filters.
+        if status.startswith(("R", "C")) and len(parts) == _RENAME_OR_COPY_PARTS:
+            old, new = parts[1], parts[2]
+            if status.startswith("R"):
+                # Pure rename: source goes away.
+                changes.append(Change(old, "D"))
+            # Both rename and copy produce a new path classified as A.
+            changes.append(Change(new, "A"))
+        else:
+            changes.append(Change(path, status[0]))
+    return changes
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base", required=True, help="git ref of merge base / target branch")
+    parser.add_argument("--head", required=True, help="git ref of PR head")
+    args = parser.parse_args(argv)
+
+    changed = _git_changed_files(args.base, args.head)
+    # Self-gate: short-circuit before reading versions when no corpus pin
+    # changed. _read_version_at_ref raises if the regex misses or if
+    # samcli/__init__.py is absent at the ref, so on the common-case PR
+    # (no corpus pin churn) we skip those reads entirely. The workflow
+    # comment claims "self-gates" — make that literally true here.
+    if not any(_is_corpus_pin(c.path) for c in changed):
+        print("No corpus pin changes; gate passes.")
+        return 0
+    base_version = _read_version_at_ref(args.base)
+    head_version = _read_version_at_ref(args.head)
+    rc, msg = check(changed, base_version, head_version)
+    print(msg)
+    return rc
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/golden/conftest.py

@@ -0,0 +1,19 @@
+"""Pytest fixtures for the golden-template corpus.
+
+Each case directory under tests/golden/templates/ becomes one parametrized
+test ID (relative path), enabling `pytest -k <name>` selection.
+"""
+
+from pathlib import Path
+
+TEMPLATES_ROOT = Path(__file__).parent / "templates"
+
+
+def pytest_generate_tests(metafunc):
+    if "golden_case" in metafunc.fixturenames:
+        cases = sorted(p.parent for p in TEMPLATES_ROOT.rglob("template.yaml"))
+        metafunc.parametrize(
+            "golden_case",
+            cases,
+            ids=lambda p: str(p.relative_to(TEMPLATES_ROOT)),
+        )