diff --git a/.github/workflows/golden-semver-gate.yml b/.github/workflows/golden-semver-gate.yml new file mode 100644 index 0000000000..7ea633b534 --- /dev/null +++ b/.github/workflows/golden-semver-gate.yml @@ -0,0 +1,69 @@ +name: Golden Templates Semver Gate + +# Run on every PR AND on every merge_group event. The check_semver_bump.py +# script self-gates: it returns exit 0 when no expected.*.yaml files +# changed, so an unconditional run is cheap and posts a definitive +# status on every event. +# +# Two pitfalls this trigger configuration avoids — both flavors of the +# "skipped but required check" problem: +# +# 1. on.pull_request.paths: filtering would skip the workflow entirely +# when no matching path changed, posting no status, which branch +# protection treats as missing/pending. We self-gate in the script +# instead so the status always posts. +# +# 2. listening on pull_request alone would skip the workflow when a PR +# enters the merge queue (which dispatches merge_group, not +# pull_request). The merge queue would then treat the required +# check as missing/pending and block the merge. We listen on both +# events to match build.yml. +on: + pull_request: + branches: + - develop + - "feat/*" + - "feat-*" + merge_group: + types: [checks_requested] + branches: + - develop + - "feat/*" + - "feat-*" + +permissions: + contents: read + +jobs: + semver-gate: + if: github.repository_owner == 'aws' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + python-version: "3.11" + - name: Resolve base ref + id: baseref + env: + # github.base_ref is set on pull_request events. On merge_group + # events it's empty and the target branch lives at + # github.event.merge_group.base_ref as a full ref ("refs/heads/develop"); + # strip the prefix to match the bare branch name pull_request emits. + PR_BASE_REF: ${{ github.base_ref }} + MG_BASE_REF: ${{ github.event.merge_group.base_ref }} + run: | + if [ -n "$PR_BASE_REF" ]; then + echo "base=$PR_BASE_REF" >> "$GITHUB_OUTPUT" + else + echo "base=${MG_BASE_REF#refs/heads/}" >> "$GITHUB_OUTPUT" + fi + - name: Run semver gate + env: + BASE_REF: ${{ steps.baseref.outputs.base }} + run: | + python tests/golden/check_semver_bump.py \ + --base "origin/${BASE_REF}" \ + --head HEAD diff --git a/Makefile b/Makefile index 88ab0294b3..39ee5b7be1 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,9 @@ test: # Run unit tests (excluding cfn_language_extensions) and fail if coverage falls below 94% @echo "NOTE: Excluding cfn_language_extensions tests. Use 'make test-all' for full coverage." pytest --cov samcli --cov schema --cov-report term-missing --cov-fail-under 94 tests/unit --ignore=tests/unit/lib/cfn_language_extensions --cov-config=.coveragerc_no_lang_ext + # Golden-template suite runs as a separate pytest invocation so it + # does not enter the samcli coverage calculation above. + pytest tests/golden test-lang-ext: # Run cfn_language_extensions unit tests with coverage @@ -37,6 +40,9 @@ test-lang-ext: test-all: # Run all unit tests including cfn_language_extensions pytest --cov samcli --cov schema --cov-report term-missing --cov-fail-under 94 tests/unit + # Golden-template suite runs as a separate pytest invocation so it + # does not enter the samcli coverage calculation above. + pytest tests/golden test-cov-report: # Run all unit tests with html coverage report diff --git a/tests/golden/README.md b/tests/golden/README.md new file mode 100644 index 0000000000..047a6c5804 --- /dev/null +++ b/tests/golden/README.md @@ -0,0 +1,65 @@ +# Golden Templates + +Pinned outputs of `sam build` and `sam package` for representative templates. +Catches output-shape regressions in SAM-CLI's local pipeline. See +[`designs/golden-templates-and-semver-check.md`](../../designs/golden-templates-and-semver-check.md) +for the full design. + +## Layout + +``` +tests/golden/ +├── templates/ +│ ├── sam_resources// # AWS::Serverless::* coverage +│ ├── packageable_resources// # Raw CFN resource coverage +│ ├── language_extensions// # Fn::ForEach et al +│ └── cross_cutting// # Nested stacks, AWS::Include, etc. +└── ... (harness code) +``` + +Each case directory contains: +- `template.yaml` — the input template. +- `metadata.yaml` — `language_extensions: bool`, `description`, `issue_refs`. +- `src/` — any source files referenced by `CodeUri` etc. +- `expected.build.yaml` — pinned post-build template (regenerated by harness). +- `expected.package.yaml` — pinned post-package template. +- `README.md` — what this case covers and why it's pinned. + +## Adding a new case + +1. Create the case directory and author `template.yaml`, `metadata.yaml`, `src/`, `README.md`. +2. Generate the pinned outputs: + ``` + python tests/golden/update_goldens.py --new --filter '/' + ``` +3. Eyeball `expected.{build,package}.yaml`. Confirm the shape is what you expect. +4. Commit everything together. + +The semver gate treats new cases as additions and does not require a version bump. + +## Re-pinning an existing case + +If a SAM-CLI behavior change intentionally alters the expanded output: + +``` +python tests/golden/update_goldens.py --filter '/' +``` + +Review the diff in your IDE before committing. **Modifying or deleting an +existing pin requires a major version bump in `samcli/__init__.py` in the +same PR**, enforced by `.github/workflows/golden-semver-gate.yml`. + +## Inspecting differences + +``` +python tests/golden/update_goldens.py --diff --filter '' # show would-be changes +python tests/golden/update_goldens.py --check # exit 1 if anything would change +``` + +## Running the suite + +``` +pytest tests/golden # all cases +pytest tests/golden -k foreach_static # one case +make test-all # part of make pr +``` diff --git a/tests/golden/__init__.py b/tests/golden/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/golden/check_semver_bump.py b/tests/golden/check_semver_bump.py new file mode 100644 index 0000000000..6dddc1c249 --- /dev/null +++ b/tests/golden/check_semver_bump.py @@ -0,0 +1,159 @@ +"""Semver gate: require a major version bump when an existing pinned +golden output is modified or deleted. + +Usage (CI): + python tests/golden/check_semver_bump.py --base origin/develop --head HEAD +""" + +from __future__ import annotations + +# Allow direct script invocation (`python tests/golden/check_semver_bump.py`) +# in addition to module form (`python -m tests.golden.check_semver_bump`). +# When run as a script, Python does not auto-add the repo root to sys.path, +# so absolute imports rooted at `tests.golden...` would fail. This script +# happens to have no such imports today, but keep the guard symmetrical with +# update_goldens.py so future imports do not silently regress the script +# form. mypy treats the __main__ guard as always-False, hence the +# `type: ignore`. +if __name__ == "__main__" and __package__ is None: + import sys # type: ignore[unreachable] # noqa: E402 + from pathlib import Path # noqa: E402 + + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +import argparse +import re +import subprocess +import sys +from dataclasses import dataclass +from typing import List, Optional, Tuple + +EXPECTED_GLOB = re.compile(r"^tests/golden/templates/.+/expected\.(build|package)\.yaml$") + +# git diff --name-status emits "R\tOLD\tNEW" for renames or +# "C\tOLD\tNEW" for copies (3 fields each). +_RENAME_OR_COPY_PARTS = 3 + + +@dataclass(frozen=True) +class Change: + path: str + status: str # "A" added, "M" modified, "D" deleted + + +def _parse_version(s: str) -> Tuple[int, int, int]: + m = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)(?:.*)?", s.strip()) + if not m: + raise ValueError(f"unparseable version: {s!r}") + return int(m.group(1)), int(m.group(2)), int(m.group(3)) + + +def _is_corpus_pin(path: str) -> bool: + return bool(EXPECTED_GLOB.match(path)) + + +def check( + changed: List[Change], + base_version: str, + head_version: str, +) -> Tuple[int, str]: + """Return (exit_code, message).""" + relevant = [c for c in changed if _is_corpus_pin(c.path)] + + if not relevant: + return 0, "No corpus pin changes; gate passes." + + modifications = [c for c in relevant if c.status == "M"] + deletions = [c for c in relevant if c.status == "D"] + additions = [c for c in relevant if c.status == "A"] + + if not (modifications or deletions): + # Additions only — no bump required. + return 0, f"{len(additions)} new pin(s); no version bump required." + + base_major, _, _ = _parse_version(base_version) + head_major, _, _ = _parse_version(head_version) + + if head_major > base_major: + return 0, "Major version bumped; gate passes." + + suggested = f"{base_major + 1}.0.0" + summary_lines = [ + "Semver gate FAILED.", + f" base version: {base_version}", + f" head version: {head_version}", + f" required: major bump (suggested {suggested})", + "", + "Reason: an existing pinned golden output was modified or deleted.", + " Modifications:", + *[f" M {c.path}" for c in modifications], + " Deletions:", + *[f" D {c.path}" for c in deletions], + "", + f'To fix: edit samcli/__init__.py and set __version__ = "{suggested}".', + "If the change is intentional, the major bump signals that to consumers.", + "If unintentional, run python tests/golden/update_goldens.py --diff to inspect.", + ] + return 1, "\n".join(summary_lines) + + +def _read_version_at_ref(ref: str) -> str: + """Read __version__ from samcli/__init__.py at a git ref.""" + out = subprocess.check_output(["git", "show", f"{ref}:samcli/__init__.py"], text=True) + m = re.search(r'__version__\s*=\s*"([^"]+)"', out) + if not m: + raise RuntimeError(f"cannot find __version__ at {ref}") + return m.group(1) + + +def _git_changed_files(base: str, head: str) -> List[Change]: + """Return all files changed between base and head with their git status.""" + out = subprocess.check_output(["git", "diff", "--name-status", f"{base}...{head}"], text=True) + changes: List[Change] = [] + for line in out.splitlines(): + if not line.strip(): + continue + parts = line.split("\t") + status, path = parts[0], parts[-1] + # Renames present as "R\tOLD\tNEW" and copies as + # "C\tOLD\tNEW". Note the asymmetry: + # - Rename: source path goes away (D) and target appears (A). + # - Copy: source persists, only the target is new (A only). + # Without this branch, statuses like "R100" / "C100" fall through + # to the else and silently bypass the gate's A/M/D filters. + if status.startswith(("R", "C")) and len(parts) == _RENAME_OR_COPY_PARTS: + old, new = parts[1], parts[2] + if status.startswith("R"): + # Pure rename: source goes away. + changes.append(Change(old, "D")) + # Both rename and copy produce a new path classified as A. + changes.append(Change(new, "A")) + else: + changes.append(Change(path, status[0])) + return changes + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--base", required=True, help="git ref of merge base / target branch") + parser.add_argument("--head", required=True, help="git ref of PR head") + args = parser.parse_args(argv) + + changed = _git_changed_files(args.base, args.head) + # Self-gate: short-circuit before reading versions when no corpus pin + # changed. _read_version_at_ref raises if the regex misses or if + # samcli/__init__.py is absent at the ref, so on the common-case PR + # (no corpus pin churn) we skip those reads entirely. The workflow + # comment claims "self-gates" — make that literally true here. + if not any(_is_corpus_pin(c.path) for c in changed): + print("No corpus pin changes; gate passes.") + return 0 + base_version = _read_version_at_ref(args.base) + head_version = _read_version_at_ref(args.head) + rc, msg = check(changed, base_version, head_version) + print(msg) + return rc + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/golden/conftest.py b/tests/golden/conftest.py new file mode 100644 index 0000000000..3143806c98 --- /dev/null +++ b/tests/golden/conftest.py @@ -0,0 +1,19 @@ +"""Pytest fixtures for the golden-template corpus. + +Each case directory under tests/golden/templates/ becomes one parametrized +test ID (relative path), enabling `pytest -k ` selection. +""" + +from pathlib import Path + +TEMPLATES_ROOT = Path(__file__).parent / "templates" + + +def pytest_generate_tests(metafunc): + if "golden_case" in metafunc.fixturenames: + cases = sorted(p.parent for p in TEMPLATES_ROOT.rglob("template.yaml")) + metafunc.parametrize( + "golden_case", + cases, + ids=lambda p: str(p.relative_to(TEMPLATES_ROOT)), + ) diff --git a/tests/golden/harness.py b/tests/golden/harness.py new file mode 100644 index 0000000000..ae61728d58 --- /dev/null +++ b/tests/golden/harness.py @@ -0,0 +1,363 @@ +"""In-process harness for golden-template testing. + +Two pipelines, each implemented as a single function: +- run_build_pipeline: drives the real :class:`BuildContext`. +- run_package_pipeline: drives the real :class:`PackageContext`. + +The harness drives the same ``BuildContext`` and ``PackageContext`` that +``sam build`` and ``sam package`` instantiate; it does not reimplement the +build / package logic itself. Two narrow stubs replace the network / +compute boundaries: + +- :class:`samcli.lib.build.app_builder.ApplicationBuilder` ``.build()`` is + replaced with a stub that returns an :class:`ApplicationBuildResult` with + one entry per buildable function / layer pointing at a per-resource + artifact directory under ``/``. The + directory contains a single file whose content is the resource's full + path, which makes the downstream content-addressed S3 URI deterministic + per resource. +- :meth:`samcli.lib.package.s3_uploader.S3Uploader.upload` and + ``upload_with_dedup`` are replaced with deterministic stubs returning + ``s3://golden-bucket/`` where the digest is taken over the file + content (zip-aware: zip envelopes are hashed by sorted member names plus + per-member content hashes so the digest is OS-deterministic). + +The case directory (``template.yaml`` + ``src/``) is staged into a fresh +temp dir before driving ``BuildContext``. Real ``sam build`` writes +``CodeUri`` as ``os.path.relpath(absolute_artifact, template_dir)``; with +the case staged into a temp tree, ``original_dir`` is the staged template's +parent and the relpath is the deterministic ``.aws-sam/build/`` +that real ``sam build`` produces. No post-build path rewriting is needed. + +This keeps the harness narrow and lets every other concern (LE expansion, +ForEach merge, Mappings generation, AWS::Include export, etc.) flow through +the real CLI code paths. +""" + +from __future__ import annotations + +import hashlib +import os +import shutil +import zipfile +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, Dict, Optional, Tuple +from unittest.mock import patch + +from samcli.commands.build.build_context import BuildContext +from samcli.commands.package.package_context import PackageContext +from samcli.lib.build.app_builder import ApplicationBuilder, ApplicationBuildResult +from samcli.lib.build.build_graph import BuildGraph +from samcli.lib.package.s3_uploader import S3Uploader +from samcli.lib.providers.provider import get_full_path +from samcli.yamlhelper import yaml_parse + + +def _to_plain_dict(value: Any) -> Any: + """Recursively convert OrderedDict (and friends) to plain dict / list / scalar. + + ``yamlhelper.yaml_parse`` returns ``OrderedDict`` instances; downstream + YAML serialization with ``yaml.safe_dump`` (used in + :mod:`tests.golden.normalize`) does not know how to represent + ``OrderedDict``. Flatten before returning so the test corpus and pin + regenerator see plain dicts. + """ + if isinstance(value, dict): + return {k: _to_plain_dict(v) for k, v in value.items()} + if isinstance(value, list): + return [_to_plain_dict(v) for v in value] + return value + + +def _read_template_as_plain_dict(path: Path) -> Dict[str, Any]: + """Read a YAML/JSON template from disk and return a plain dict. + + Centralizes the ``yaml_parse`` -> ``_to_plain_dict`` -> ``cast`` chain + so the type signature stays clean for callers (mypy infers Any from + yaml_parse otherwise). + """ + with open(path, "r", encoding="utf-8") as f: + parsed = _to_plain_dict(yaml_parse(f.read())) + if not isinstance(parsed, dict): + return {} + return parsed + + +GOLDEN_BUCKET = "golden-bucket" + + +def _stub_application_builder_build(build_dir: str) -> Any: + """Build a stand-in for ``ApplicationBuilder.build``. + + The returned function captures ``build_dir`` and writes one artifact + directory per buildable function / layer at the canonical location + real ``sam build`` uses: + + //__golden_placeholder__ + + The placeholder file's content is the resource's full path, which makes + the package phase's content-addressed S3 URI distinct per resource even + though no real source code was compiled. ``ApplicationBuilder.update_template`` + rewrites these absolute paths to ``relpath(artifact, template_dir)``; + when the case is staged under the temp dir before driving + ``BuildContext``, the relpath becomes the deterministic + ``.aws-sam/build/`` string real ``sam build`` produces. + + The returned ``ApplicationBuildResult.artifacts`` map is keyed on + ``get_full_path(stack_path, function_id)`` exactly as real SAM CLI does; + ``BuildContext._handle_build_post_processing`` reads this map. + """ + + def _fake_build(self: ApplicationBuilder) -> ApplicationBuildResult: + artifacts: Dict[str, str] = {} + + for function in self._resources_to_build.functions: # type: ignore[attr-defined] + full_path = get_full_path(function.stack_path, function.function_id) + artifact_dir = Path(build_dir) / full_path.replace("/", os.sep) + artifact_dir.mkdir(parents=True, exist_ok=True) + (artifact_dir / "__golden_placeholder__").write_text(full_path, encoding="utf-8") + artifacts[full_path] = str(artifact_dir) + + for layer in self._resources_to_build.layers: # type: ignore[attr-defined] + full_path = layer.full_path + artifact_dir = Path(build_dir) / full_path.replace("/", os.sep) + artifact_dir.mkdir(parents=True, exist_ok=True) + (artifact_dir / "__golden_placeholder__").write_text(full_path, encoding="utf-8") + artifacts[full_path] = str(artifact_dir) + + return ApplicationBuildResult(BuildGraph(self._build_dir), artifacts) # type: ignore[attr-defined] + + return _fake_build + + +def _fake_s3_upload(self: S3Uploader, file_name: str, key: Optional[str] = None) -> str: + """Deterministic ``S3Uploader.upload`` replacement. + + Hashes the file content so two different inputs produce two different + URIs, but identical content always produces the same URI. The bucket + name is fixed to :data:`GOLDEN_BUCKET` regardless of the uploader's own + bucket attribute so pinned outputs do not depend on local config. + + Zip files (the common artifact shape `sam package` uploads) are hashed + over a normalized representation — sorted member names plus each + member's content sha256 — instead of the raw zip bytes. Raw zip bytes + embed OS-specific metadata (DOS timestamps, Unix mode bits, sometimes + path separators) which would otherwise produce different digests on + Windows vs. POSIX runners. + """ + digest = _hash_zip_normalized(file_name) if _looks_like_zip(file_name) else _hash_raw_bytes(file_name) + return f"s3://{GOLDEN_BUCKET}/{digest}" + + +def _looks_like_zip(file_name: str) -> bool: + """True if ``file_name`` looks like a zip-format artifact.""" + try: + return zipfile.is_zipfile(file_name) + except OSError: + return False + + +def _hash_raw_bytes(file_name: str) -> str: + with open(file_name, "rb") as f: + return hashlib.sha256(f.read()).hexdigest() + + +def _hash_zip_normalized(file_name: str) -> str: + """Hash a zip file by its sorted member names + per-member content hashes. + + This skips the zip envelope metadata (DOS timestamps, Unix permission + bits, central directory ordering) that varies by OS and makes raw + byte-level hashes non-deterministic across runners. + """ + hasher = hashlib.sha256() + with zipfile.ZipFile(file_name, "r") as zf: + for name in sorted(zf.namelist()): + hasher.update(name.encode("utf-8")) + hasher.update(b"\x00") + with zf.open(name) as member: + hasher.update(hashlib.sha256(member.read()).digest()) + return hasher.hexdigest() + + +def _fake_s3_upload_with_dedup( + self: S3Uploader, + file_name: str, + extension: Optional[str] = None, + precomputed_md5: Optional[str] = None, +) -> str: + return _fake_s3_upload(self, file_name) + + +_STAGED_CASE_DIRNAME = "case" + + +def _stage_case_dir(case_dir: Path, tmp_root: Path) -> Path: + """Copy the case directory (template + src/) into ``tmp_root/case``. + + Real ``sam build`` rewrites artifact paths to + ``relpath(absolute_artifact, template_dir)``. Staging the case dir into + a known location under the temp root makes that relpath the canonical + deterministic ``.aws-sam/build/`` string regardless of where + the host's tempdir lives. + """ + staged = tmp_root / _STAGED_CASE_DIRNAME + shutil.copytree(case_dir, staged) + return staged + + +def run_build_pipeline_to_dir( + staged_template_path: Path, + language_extensions: bool, + build_dir: Path, + cache_dir: Path, +) -> Dict[str, Any]: + """Drive ``BuildContext`` to produce a built template under ``build_dir``. + + The caller is responsible for staging the case dir under a temp root + (see :func:`_stage_case_dir`) and pointing ``staged_template_path`` at + the staged copy. ``build_dir`` should sit inside the same staged tree + so ``ApplicationBuilder.update_template`` produces the canonical + ``.aws-sam/build/`` relative paths. + + Returns the parsed dict read back from ``/template.yaml``. + The on-disk file is left in place so callers (notably + :func:`run_package_pipeline`) can hand the path to ``PackageContext`` + directly. + """ + bctx = BuildContext( + resource_identifier=None, + template_file=str(staged_template_path), + base_dir=str(staged_template_path.parent), + build_dir=str(build_dir), + cache_dir=str(cache_dir), + cached=False, + parallel=False, + mode=None, + aws_region="us-east-1", + print_success_message=False, + language_extensions=language_extensions, + ) + + fake_build = _stub_application_builder_build(str(build_dir)) + with bctx: + with patch.object(ApplicationBuilder, "build", fake_build): + bctx.run() + + return _read_template_as_plain_dict(build_dir / "template.yaml") + + +def run_build_pipeline(template_path: Path, language_extensions: bool) -> Dict[str, Any]: + """In-process equivalent of ``sam build`` for a single template. + + Stages the case dir (``template.yaml`` + ``src/``) into a fresh temp + tree, drives :class:`BuildContext` end-to-end, and returns the parsed + built template dict. ``CodeUri`` etc. emerge as the canonical + ``.aws-sam/build/`` relative path that real ``sam build`` + writes — no post-processing needed. + """ + with TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + staged = _stage_case_dir(template_path.parent, tmp_path) + build_dir = staged / ".aws-sam" / "build" + cache_dir = staged / ".aws-sam" / "cache" + return run_build_pipeline_to_dir(staged / template_path.name, language_extensions, build_dir, cache_dir) + + +def run_package_pipeline(template_path: Path, language_extensions: bool) -> Dict[str, Any]: + """In-process equivalent of ``sam package``. + + Runs the build pipeline first (against a staged copy of the case dir) + to produce a real on-disk ``.aws-sam/build/template.yaml``, then drives + :class:`PackageContext` against that template. The S3 uploader is + stubbed for determinism; everything else (LE expansion, ForEach merge, + Mappings generation, AWS::Include export) runs through real code. + """ + with TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + staged = _stage_case_dir(template_path.parent, tmp_path) + build_dir = staged / ".aws-sam" / "build" + cache_dir = staged / ".aws-sam" / "cache" + + # Re-run the build phase so we have a freshly built template on + # disk that PackageContext can read. We can't reuse a previously + # produced dict because PackageContext re-loads the template from + # the path itself. + run_build_pipeline_to_dir(staged / template_path.name, language_extensions, build_dir, cache_dir) + + built_template_path = build_dir / "template.yaml" + output_template_path = tmp_path / "packaged" / "template.yaml" + output_template_path.parent.mkdir(parents=True, exist_ok=True) + + pctx = PackageContext( + template_file=str(built_template_path), + s3_bucket=GOLDEN_BUCKET, + image_repository=None, + image_repositories=None, + s3_prefix=None, + kms_key_id=None, + output_template_file=str(output_template_path), + use_json=False, + force_upload=False, + no_progressbar=True, + metadata=None, + region="us-east-1", + profile=None, + language_extensions=language_extensions, + ) + + with ( + patch.object(S3Uploader, "upload", _fake_s3_upload), + patch.object(S3Uploader, "upload_with_dedup", _fake_s3_upload_with_dedup), + ): + pctx.run() + + return _read_template_as_plain_dict(output_template_path) + + +# --- compatibility helpers used by callers ---------------------------------- + + +def run_build_and_package(template_path: Path, language_extensions: bool) -> Tuple[Dict[str, Any], Dict[str, Any]]: + """Convenience: produce both pins from a single build. + + Saves work for callers (``update_goldens`` and ``test_corpus``) that + need both pins for the same template; PackageContext relies on a real + on-disk built template, so we reuse the one from the build phase. + """ + with TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + staged = _stage_case_dir(template_path.parent, tmp_path) + build_dir = staged / ".aws-sam" / "build" + cache_dir = staged / ".aws-sam" / "cache" + build_dict = run_build_pipeline_to_dir(staged / template_path.name, language_extensions, build_dir, cache_dir) + + built_template_path = build_dir / "template.yaml" + output_template_path = tmp_path / "packaged" / "template.yaml" + output_template_path.parent.mkdir(parents=True, exist_ok=True) + + pctx = PackageContext( + template_file=str(built_template_path), + s3_bucket=GOLDEN_BUCKET, + image_repository=None, + image_repositories=None, + s3_prefix=None, + kms_key_id=None, + output_template_file=str(output_template_path), + use_json=False, + force_upload=False, + no_progressbar=True, + metadata=None, + region="us-east-1", + profile=None, + language_extensions=language_extensions, + ) + + with ( + patch.object(S3Uploader, "upload", _fake_s3_upload), + patch.object(S3Uploader, "upload_with_dedup", _fake_s3_upload_with_dedup), + ): + pctx.run() + + pkg_dict = _read_template_as_plain_dict(output_template_path) + return build_dict, pkg_dict diff --git a/tests/golden/normalize.py b/tests/golden/normalize.py new file mode 100644 index 0000000000..f3cd06d828 --- /dev/null +++ b/tests/golden/normalize.py @@ -0,0 +1,69 @@ +"""Deterministic YAML rendering for golden-template comparison. + +The harness and update_goldens.py both call normalize() so the bytes +written to disk match the bytes the test compares. Determinism rules: + +- Resources / Mappings keys sorted alphabetically. +- Metadata.SamTransformMetrics dropped (varies by run). +- Empty Metadata block dropped. +- yaml.safe_dump with sort_keys=True, default_flow_style=False. +- Single trailing newline. + +OrderedDict handling: ``samcli.yamlhelper.yaml_parse`` registers a +``DEFAULT_MAPPING_TAG`` constructor that emits ``OrderedDict`` instead of +``dict`` on the global ``yaml.SafeLoader``. Once any caller has imported +yamlhelper, every subsequent ``yaml.safe_load`` returns OrderedDict. The +default ``yaml.safe_dump`` representer rejects OrderedDict, so we +recursively coerce the input to plain ``dict`` before serializing. +""" + +from __future__ import annotations + +from typing import Any, Dict + +import yaml + +_VOLATILE_METADATA_KEYS = frozenset({"SamTransformMetrics"}) + + +def _to_plain(value: Any) -> Any: + """Recursively coerce OrderedDict (and anything dict-like) to plain dict. + + Necessary because ``samcli.yamlhelper.yaml_parse`` mutates the global + ``yaml.SafeLoader`` to emit OrderedDict, and ``yaml.safe_dump`` does + not know how to represent OrderedDict. + """ + if isinstance(value, dict): + return {k: _to_plain(v) for k, v in value.items()} + if isinstance(value, list): + return [_to_plain(v) for v in value] + return value + + +def _filter_metadata(template: Dict[str, Any]) -> None: + metadata = template.get("Metadata") + if not isinstance(metadata, dict): + return + for key in _VOLATILE_METADATA_KEYS: + metadata.pop(key, None) + if not metadata: + template.pop("Metadata", None) + + +def normalize(template: Dict[str, Any]) -> str: + """Render template to deterministic YAML string.""" + # Coerce to plain dict so yaml.safe_dump can represent it; also gives + # us a deep copy so we don't surprise the caller. + template = _to_plain(template) + _filter_metadata(template) + + rendered = yaml.safe_dump( + template, + sort_keys=True, + default_flow_style=False, + width=10**9, # don't wrap long strings + allow_unicode=True, + ) + if not rendered.endswith("\n"): + rendered += "\n" + return rendered diff --git a/tests/golden/templates/language_extensions/foreach_static_zip/README.md b/tests/golden/templates/language_extensions/foreach_static_zip/README.md new file mode 100644 index 0000000000..8c6bec56cc --- /dev/null +++ b/tests/golden/templates/language_extensions/foreach_static_zip/README.md @@ -0,0 +1,7 @@ +# foreach_static_zip + +Sentinel LE case. `Fn::ForEach` over a static collection with all expanded +functions sharing the same `CodeUri`. Locks the expanded-template shape +(`AlphaFunction`, `BetaFunction` inlined as siblings, ForEach key gone) +and the post-package shape (each function's `CodeUri` rewritten to the +same `s3://golden-bucket/`). diff --git a/tests/golden/templates/language_extensions/foreach_static_zip/expected.build.yaml b/tests/golden/templates/language_extensions/foreach_static_zip/expected.build.yaml new file mode 100644 index 0000000000..2a6f500baa --- /dev/null +++ b/tests/golden/templates/language_extensions/foreach_static_zip/expected.build.yaml @@ -0,0 +1,15 @@ +AWSTemplateFormatVersion: '2010-09-09' +Resources: + Fn::ForEach::Workers: + - WorkerName + - - Alpha + - Beta + - ${WorkerName}Function: + Properties: + CodeUri: AlphaFunction + Handler: app.handler + Runtime: python3.11 + Type: AWS::Serverless::Function +Transform: +- AWS::LanguageExtensions +- AWS::Serverless-2016-10-31 diff --git a/tests/golden/templates/language_extensions/foreach_static_zip/expected.package.yaml b/tests/golden/templates/language_extensions/foreach_static_zip/expected.package.yaml new file mode 100644 index 0000000000..872449f50d --- /dev/null +++ b/tests/golden/templates/language_extensions/foreach_static_zip/expected.package.yaml @@ -0,0 +1,15 @@ +AWSTemplateFormatVersion: '2010-09-09' +Resources: + Fn::ForEach::Workers: + - WorkerName + - - Alpha + - Beta + - ${WorkerName}Function: + Properties: + CodeUri: s3://golden-bucket/b38f16354b937b1d3c3d74e0c9d24ab809eee8b0622ec5b6408fee535b822d42 + Handler: app.handler + Runtime: python3.11 + Type: AWS::Serverless::Function +Transform: +- AWS::LanguageExtensions +- AWS::Serverless-2016-10-31 diff --git a/tests/golden/templates/language_extensions/foreach_static_zip/metadata.yaml b/tests/golden/templates/language_extensions/foreach_static_zip/metadata.yaml new file mode 100644 index 0000000000..e2113d53a4 --- /dev/null +++ b/tests/golden/templates/language_extensions/foreach_static_zip/metadata.yaml @@ -0,0 +1,3 @@ +language_extensions: true +description: "Sentinel: Fn::ForEach over static collection, shared CodeUri." +issue_refs: [] diff --git a/tests/golden/templates/language_extensions/foreach_static_zip/src/app.py b/tests/golden/templates/language_extensions/foreach_static_zip/src/app.py new file mode 100644 index 0000000000..29193508dc --- /dev/null +++ b/tests/golden/templates/language_extensions/foreach_static_zip/src/app.py @@ -0,0 +1,2 @@ +def handler(event, context): + return {"ok": True} diff --git a/tests/golden/templates/language_extensions/foreach_static_zip/template.yaml b/tests/golden/templates/language_extensions/foreach_static_zip/template.yaml new file mode 100644 index 0000000000..9b4969402b --- /dev/null +++ b/tests/golden/templates/language_extensions/foreach_static_zip/template.yaml @@ -0,0 +1,16 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: + - AWS::LanguageExtensions + - AWS::Serverless-2016-10-31 + +Resources: + Fn::ForEach::Workers: + - WorkerName + - - Alpha + - Beta + - ${WorkerName}Function: + Type: AWS::Serverless::Function + Properties: + Handler: app.handler + Runtime: python3.11 + CodeUri: ./src/ diff --git a/tests/golden/templates/packageable_resources/lambda_function_zip/README.md b/tests/golden/templates/packageable_resources/lambda_function_zip/README.md new file mode 100644 index 0000000000..c71fa401dd --- /dev/null +++ b/tests/golden/templates/packageable_resources/lambda_function_zip/README.md @@ -0,0 +1,5 @@ +# lambda_function_zip + +Sentinel non-SAM case. Raw `AWS::Lambda::Function` with `Code: ./src/`. Locks +the post-package output shape after the CFN-resource-list packageable walker +rewrites `Code` to `s3://golden-bucket/`. diff --git a/tests/golden/templates/packageable_resources/lambda_function_zip/expected.build.yaml b/tests/golden/templates/packageable_resources/lambda_function_zip/expected.build.yaml new file mode 100644 index 0000000000..7844827503 --- /dev/null +++ b/tests/golden/templates/packageable_resources/lambda_function_zip/expected.build.yaml @@ -0,0 +1,13 @@ +AWSTemplateFormatVersion: '2010-09-09' +Resources: + HelloFunction: + Metadata: + SamResourceId: HelloFunction + Properties: + Code: HelloFunction + FunctionName: hello + Handler: app.handler + MemorySize: 128 + Role: arn:aws:iam::123456789012:role/lambda-role + Runtime: python3.11 + Type: AWS::Lambda::Function diff --git a/tests/golden/templates/packageable_resources/lambda_function_zip/expected.package.yaml b/tests/golden/templates/packageable_resources/lambda_function_zip/expected.package.yaml new file mode 100644 index 0000000000..3a07f5dff4 --- /dev/null +++ b/tests/golden/templates/packageable_resources/lambda_function_zip/expected.package.yaml @@ -0,0 +1,15 @@ +AWSTemplateFormatVersion: '2010-09-09' +Resources: + HelloFunction: + Metadata: + SamResourceId: HelloFunction + Properties: + Code: + S3Bucket: golden-bucket + S3Key: 22186578b8d9d69e7157330653df80d6801d2e1bc7af9c2a7622640de339ae63 + FunctionName: hello + Handler: app.handler + MemorySize: 128 + Role: arn:aws:iam::123456789012:role/lambda-role + Runtime: python3.11 + Type: AWS::Lambda::Function diff --git a/tests/golden/templates/packageable_resources/lambda_function_zip/metadata.yaml b/tests/golden/templates/packageable_resources/lambda_function_zip/metadata.yaml new file mode 100644 index 0000000000..c9bdf896ef --- /dev/null +++ b/tests/golden/templates/packageable_resources/lambda_function_zip/metadata.yaml @@ -0,0 +1,3 @@ +language_extensions: false +description: "Sentinel: raw AWS::Lambda::Function with local Code path." +issue_refs: [] diff --git a/tests/golden/templates/packageable_resources/lambda_function_zip/src/app.py b/tests/golden/templates/packageable_resources/lambda_function_zip/src/app.py new file mode 100644 index 0000000000..29193508dc --- /dev/null +++ b/tests/golden/templates/packageable_resources/lambda_function_zip/src/app.py @@ -0,0 +1,2 @@ +def handler(event, context): + return {"ok": True} diff --git a/tests/golden/templates/packageable_resources/lambda_function_zip/template.yaml b/tests/golden/templates/packageable_resources/lambda_function_zip/template.yaml new file mode 100644 index 0000000000..e91999c61e --- /dev/null +++ b/tests/golden/templates/packageable_resources/lambda_function_zip/template.yaml @@ -0,0 +1,12 @@ +AWSTemplateFormatVersion: '2010-09-09' + +Resources: + HelloFunction: + Type: AWS::Lambda::Function + Properties: + FunctionName: hello + Runtime: python3.11 + Handler: app.handler + Role: arn:aws:iam::123456789012:role/lambda-role + Code: ./src/ + MemorySize: 128 diff --git a/tests/golden/templates/sam_resources/serverless_function_zip/README.md b/tests/golden/templates/sam_resources/serverless_function_zip/README.md new file mode 100644 index 0000000000..f4882c56ed --- /dev/null +++ b/tests/golden/templates/sam_resources/serverless_function_zip/README.md @@ -0,0 +1,7 @@ +# serverless_function_zip + +Sentinel SAM case. Vanilla `AWS::Serverless::Function` with local `CodeUri` +pointing at a ZIP-packaged source dir. Locks the post-build `template.yaml` +shape (`CodeUri` rewritten to the resource-id-relative path real `sam build` +produces) and post-package shape (CodeUri rewritten to +`s3://golden-bucket/`). diff --git a/tests/golden/templates/sam_resources/serverless_function_zip/expected.build.yaml b/tests/golden/templates/sam_resources/serverless_function_zip/expected.build.yaml new file mode 100644 index 0000000000..efe1826616 --- /dev/null +++ b/tests/golden/templates/sam_resources/serverless_function_zip/expected.build.yaml @@ -0,0 +1,12 @@ +AWSTemplateFormatVersion: '2010-09-09' +Resources: + HelloFunction: + Metadata: + SamResourceId: HelloFunction + Properties: + CodeUri: HelloFunction + Handler: app.handler + MemorySize: 128 + Runtime: python3.11 + Type: AWS::Serverless::Function +Transform: AWS::Serverless-2016-10-31 diff --git a/tests/golden/templates/sam_resources/serverless_function_zip/expected.package.yaml b/tests/golden/templates/sam_resources/serverless_function_zip/expected.package.yaml new file mode 100644 index 0000000000..6c13e577a8 --- /dev/null +++ b/tests/golden/templates/sam_resources/serverless_function_zip/expected.package.yaml @@ -0,0 +1,12 @@ +AWSTemplateFormatVersion: '2010-09-09' +Resources: + HelloFunction: + Metadata: + SamResourceId: HelloFunction + Properties: + CodeUri: s3://golden-bucket/22186578b8d9d69e7157330653df80d6801d2e1bc7af9c2a7622640de339ae63 + Handler: app.handler + MemorySize: 128 + Runtime: python3.11 + Type: AWS::Serverless::Function +Transform: AWS::Serverless-2016-10-31 diff --git a/tests/golden/templates/sam_resources/serverless_function_zip/metadata.yaml b/tests/golden/templates/sam_resources/serverless_function_zip/metadata.yaml new file mode 100644 index 0000000000..7b7f0e7497 --- /dev/null +++ b/tests/golden/templates/sam_resources/serverless_function_zip/metadata.yaml @@ -0,0 +1,3 @@ +language_extensions: false +description: "Sentinel: AWS::Serverless::Function with local CodeUri (ZIP packaging)." +issue_refs: [] diff --git a/tests/golden/templates/sam_resources/serverless_function_zip/src/app.py b/tests/golden/templates/sam_resources/serverless_function_zip/src/app.py new file mode 100644 index 0000000000..29193508dc --- /dev/null +++ b/tests/golden/templates/sam_resources/serverless_function_zip/src/app.py @@ -0,0 +1,2 @@ +def handler(event, context): + return {"ok": True} diff --git a/tests/golden/templates/sam_resources/serverless_function_zip/template.yaml b/tests/golden/templates/sam_resources/serverless_function_zip/template.yaml new file mode 100644 index 0000000000..47f228ea94 --- /dev/null +++ b/tests/golden/templates/sam_resources/serverless_function_zip/template.yaml @@ -0,0 +1,11 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 + +Resources: + HelloFunction: + Type: AWS::Serverless::Function + Properties: + Handler: app.handler + Runtime: python3.11 + CodeUri: ./src/ + MemorySize: 128 diff --git a/tests/golden/test_check_semver_bump.py b/tests/golden/test_check_semver_bump.py new file mode 100644 index 0000000000..58e9342214 --- /dev/null +++ b/tests/golden/test_check_semver_bump.py @@ -0,0 +1,187 @@ +"""Unit tests for check_semver_bump.py — the semver-gate logic. + +Avoids actual git invocation by passing changed-files lists directly to +the lower-level entrypoint. +""" + +from tests.golden import check_semver_bump as csb + +# A rename emits one D + one A change. +_RENAME_CHANGE_COUNT = 2 + + +def test_no_changes_passes(): + rc, msg = csb.check( + changed=[], + base_version="1.161.1", + head_version="1.161.1", + ) + assert rc == 0 + + +def test_addition_only_does_not_require_bump(): + rc, msg = csb.check( + changed=[ + csb.Change("tests/golden/templates/x/case_a/expected.build.yaml", "A"), + ], + base_version="1.161.1", + head_version="1.161.1", + ) + assert rc == 0 + + +def test_modification_without_major_bump_fails(): + rc, msg = csb.check( + changed=[ + csb.Change("tests/golden/templates/x/case_a/expected.build.yaml", "M"), + ], + base_version="1.161.1", + head_version="1.161.2", + ) + assert rc != 0 + assert "major" in msg.lower() + assert "2.0.0" in msg # suggested next version + + +def test_modification_with_major_bump_passes(): + rc, msg = csb.check( + changed=[ + csb.Change("tests/golden/templates/x/case_a/expected.build.yaml", "M"), + ], + base_version="1.161.1", + head_version="2.0.0", + ) + assert rc == 0 + + +def test_deletion_requires_major_bump(): + rc, msg = csb.check( + changed=[ + csb.Change("tests/golden/templates/x/case_a/expected.build.yaml", "D"), + ], + base_version="1.161.1", + head_version="1.162.0", + ) + assert rc != 0 + + +def test_rename_treated_as_delete_plus_add(): + rc, msg = csb.check( + changed=[ + csb.Change("tests/golden/templates/x/old/expected.build.yaml", "D"), + csb.Change("tests/golden/templates/x/new/expected.build.yaml", "A"), + ], + base_version="1.161.1", + head_version="1.161.2", + ) + # The deletion alone forces major; rename is conservative. + assert rc != 0 + + +def test_parser_rename_emits_delete_plus_add(monkeypatch): + """A `R\\tOLD\\tNEW` line from `git diff --name-status` must be + split into a deletion of OLD and an addition of NEW so the gate sees + the source go away.""" + fake_diff = ( + "R100\ttests/golden/templates/x/old/expected.build.yaml" "\ttests/golden/templates/x/new/expected.build.yaml\n" + ) + monkeypatch.setattr(csb.subprocess, "check_output", lambda *a, **kw: fake_diff) + changes = csb._git_changed_files("base", "head") + assert csb.Change("tests/golden/templates/x/old/expected.build.yaml", "D") in changes + assert csb.Change("tests/golden/templates/x/new/expected.build.yaml", "A") in changes + assert len(changes) == _RENAME_CHANGE_COUNT + + +def test_parser_copy_treated_as_addition_only(monkeypatch): + """A `C\\tOLD\\tNEW` line means the source still exists, only + the new target is added. The gate must classify this as A only — not + let it silently bypass via an unrecognized "C" status.""" + fake_diff = ( + "C100\ttests/golden/templates/x/old/expected.build.yaml" "\ttests/golden/templates/x/new/expected.build.yaml\n" + ) + monkeypatch.setattr(csb.subprocess, "check_output", lambda *a, **kw: fake_diff) + changes = csb._git_changed_files("base", "head") + # Only the new path is reported, classified as A. + assert changes == [ + csb.Change("tests/golden/templates/x/new/expected.build.yaml", "A"), + ] + + +def test_copy_of_existing_pin_does_not_bypass_gate(monkeypatch): + """End-to-end: even if a copy lands an existing pin's content at a + new path (status `C100`), the resulting Change list must contain an + addition — feeding only A's through `check()` correctly returns 0, + but the path must NOT be silently dropped.""" + fake_diff = ( + "C100\ttests/golden/templates/x/old/expected.build.yaml" "\ttests/golden/templates/x/new/expected.build.yaml\n" + ) + monkeypatch.setattr(csb.subprocess, "check_output", lambda *a, **kw: fake_diff) + changes = csb._git_changed_files("base", "head") + rc, msg = csb.check(changes, base_version="1.161.1", head_version="1.161.1") + # Addition-only is allowed without a major bump; the key thing is + # that the change was observed (not silently elided as status "C"). + assert rc == 0 + assert "1 new pin" in msg + + +def test_changes_outside_corpus_ignored(): + rc, msg = csb.check( + changed=[ + csb.Change("samcli/lib/foo.py", "M"), + csb.Change("tests/golden/harness.py", "M"), + ], + base_version="1.161.1", + head_version="1.161.1", + ) + assert rc == 0 + + +def test_template_yaml_changes_ignored_only_expected_yaml_gates(): + rc, msg = csb.check( + changed=[ + csb.Change("tests/golden/templates/x/case_a/template.yaml", "M"), + ], + base_version="1.161.1", + head_version="1.161.1", + ) + assert rc == 0 + + +def test_main_short_circuits_before_reading_versions(monkeypatch, capsys): + """When no corpus pin changed, main() must NOT call + _read_version_at_ref. That fn raises on missing samcli/__init__.py + or unparseable __version__, so on the common-case unrelated PR it + could fail spuriously. The workflow comment claims 'self-gates' — + this test asserts main() actually short-circuits.""" + # Diff contains only non-corpus changes. + fake_diff = "M\tsamcli/lib/foo.py\nM\ttests/golden/harness.py\n" + monkeypatch.setattr(csb.subprocess, "check_output", lambda *a, **kw: fake_diff) + + def boom(_ref): + raise AssertionError("_read_version_at_ref must not be called when no corpus pin changed") + + monkeypatch.setattr(csb, "_read_version_at_ref", boom) + rc = csb.main(["--base", "origin/develop", "--head", "HEAD"]) + assert rc == 0 + out = capsys.readouterr().out + assert "No corpus pin changes" in out + + +def test_main_reads_versions_when_corpus_pin_changes(monkeypatch): + """Counterpart: when a corpus pin DID change, main() must read + versions and run the full check — i.e. the short-circuit doesn't + over-fire.""" + fake_diff = "M\ttests/golden/templates/x/case_a/expected.build.yaml\n" + monkeypatch.setattr(csb.subprocess, "check_output", lambda *a, **kw: fake_diff) + + calls = [] + + def fake_read(ref): + calls.append(ref) + return "1.161.1" if ref == "base" else "2.0.0" + + monkeypatch.setattr(csb, "_read_version_at_ref", fake_read) + rc = csb.main(["--base", "base", "--head", "head"]) + # Major bump satisfies the gate. + assert rc == 0 + assert calls == ["base", "head"] diff --git a/tests/golden/test_corpus.py b/tests/golden/test_corpus.py new file mode 100644 index 0000000000..06c7403038 --- /dev/null +++ b/tests/golden/test_corpus.py @@ -0,0 +1,43 @@ +"""Golden corpus: byte-exact diff for build + package output per case.""" + +from pathlib import Path + +from tests.golden.harness import run_build_and_package +from tests.golden.normalize import normalize +from tests.golden.update_goldens import _read_metadata, _resolve_le_default + +TEMPLATES_ROOT = Path(__file__).parent / "templates" + + +def _hint(case_dir: Path) -> str: + rel = case_dir.relative_to(TEMPLATES_ROOT) + return ( + f"\nGolden mismatch in {rel}.\n" + f"To inspect: python tests/golden/update_goldens.py --diff --filter '{rel}'\n" + f"To re-pin: python tests/golden/update_goldens.py --filter '{rel}'\n" + f"If intentional:\n" + f" - new case (added expected.*.yaml) -> no version bump at PR time\n" + f" - modified/deleted existing expected.* -> bump major in samcli/__init__.py\n" + ) + + +def test_build_output_matches_golden(golden_case): + meta = _read_metadata(golden_case) + le_enabled = _resolve_le_default(golden_case, meta) + build_dict, _ = run_build_and_package(golden_case / "template.yaml", language_extensions=le_enabled) + actual = normalize(build_dict) + expected_path = golden_case / "expected.build.yaml" + assert expected_path.exists(), f"missing {expected_path}; run update_goldens.py --new" + expected = expected_path.read_text(encoding="utf-8") + assert actual == expected, _hint(golden_case) + + +def test_package_output_matches_golden(golden_case): + meta = _read_metadata(golden_case) + le_enabled = _resolve_le_default(golden_case, meta) + _, pkg_dict = run_build_and_package(golden_case / "template.yaml", language_extensions=le_enabled) + actual = normalize(pkg_dict) + expected_path = golden_case / "expected.package.yaml" + assert expected_path.exists(), f"missing {expected_path}; run update_goldens.py --new" + expected = expected_path.read_text(encoding="utf-8") + assert actual == expected, _hint(golden_case) diff --git a/tests/golden/test_harness_build.py b/tests/golden/test_harness_build.py new file mode 100644 index 0000000000..0b6c0ffecf --- /dev/null +++ b/tests/golden/test_harness_build.py @@ -0,0 +1,74 @@ +"""Unit tests for run_build_pipeline. + +The pipeline drives the real :class:`BuildContext`; these tests assert that +the in-process flow matches what real ``sam build`` writes to +``.aws-sam/build/template.yaml``: + +- SAM templates: ``AWS::Serverless::Function`` is preserved, ``CodeUri`` + is the resource's logical id (the canonical relative path real + ``sam build`` writes — relative to the output template's directory). +- Raw CFN ``AWS::Lambda::Function``: ``Code`` is the same logical-id + relative path; the resource type is preserved. +- LE ``Fn::ForEach`` templates: the ForEach key is preserved at the top + of ``Resources``; the body's artifact property is the logical-id path. +""" + +from pathlib import Path + +from tests.golden.harness import run_build_pipeline + +CASES_ROOT = Path(__file__).parent / "templates" + + +def test_build_sam_case_preserves_serverless_function(): + case = CASES_ROOT / "sam_resources" / "serverless_function_zip" + result = run_build_pipeline(case / "template.yaml", language_extensions=False) + func = result["Resources"]["HelloFunction"] + # SAM transform is server-side; build output keeps AWS::Serverless::Function. + assert func["Type"] == "AWS::Serverless::Function" + # CodeUri is the artifact's path relative to the output template's dir + # (.aws-sam/build/template.yaml -> .aws-sam/build/HelloFunction/). + assert func["Properties"]["CodeUri"] == "HelloFunction" + # No auto-generated IAM Role at build time. + assert "HelloFunctionRole" not in result["Resources"] + + +def test_build_le_case_preserves_foreach(): + """LE templates must keep Fn::ForEach::* in the output. + + Real ``sam build`` produces a template that preserves the original + ``Fn::ForEach`` structure so CloudFormation can re-expand it server-side + at deploy time. Inside the body, the artifact property is the artifact + directory's relative path (one path per expanded resource). The body + resource type stays ``AWS::Serverless::Function`` because the SAM + transform also runs server-side, not at build time. See PR #8637. + """ + case = CASES_ROOT / "language_extensions" / "foreach_static_zip" + result = run_build_pipeline(case / "template.yaml", language_extensions=True) + + # ForEach key preserved at the top of Resources. + foreach_keys = [k for k in result["Resources"] if k.startswith("Fn::ForEach")] + assert foreach_keys, "Fn::ForEach::* must survive the build pipeline for LE templates" + + # The expanded resource ids must NOT appear at the top level — that would + # mean the ForEach got expanded away, which is exactly the bug PR #8637 + # fixed. The harness must mirror real-world output. + assert "AlphaFunction" not in result["Resources"] + assert "BetaFunction" not in result["Resources"] + + # Body is still a Serverless::Function (SAM transform runs server-side), + # and its CodeUri is set to a non-empty relative path string. + foreach_value = result["Resources"][foreach_keys[0]] + body = foreach_value[2] + body_resource = next(iter(body.values())) + assert body_resource["Type"] == "AWS::Serverless::Function" + code_uri = body_resource["Properties"]["CodeUri"] + assert isinstance(code_uri, str) and code_uri, "CodeUri must be a non-empty deterministic path" + + +def test_build_cfn_case_keeps_lambda_function(): + case = CASES_ROOT / "packageable_resources" / "lambda_function_zip" + result = run_build_pipeline(case / "template.yaml", language_extensions=False) + func = result["Resources"]["HelloFunction"] + assert func["Type"] == "AWS::Lambda::Function" + assert func["Properties"]["Code"] == "HelloFunction" diff --git a/tests/golden/test_harness_package.py b/tests/golden/test_harness_package.py new file mode 100644 index 0000000000..783bce751d --- /dev/null +++ b/tests/golden/test_harness_package.py @@ -0,0 +1,74 @@ +"""Unit tests for run_package_pipeline. + +The pipeline drives the real :class:`PackageContext` (after a real build +phase); these tests assert that the on-disk packaged template matches +what real ``sam package`` writes: + +- SAM templates: ``AWS::Serverless::Function`` is preserved (no SAM + transform); ``CodeUri`` becomes an ``s3://golden-bucket/`` URI. +- Raw CFN ``AWS::Lambda::Function``: ``Code`` becomes + ``{S3Bucket: golden-bucket, S3Key: }``. +- LE templates: ``Fn::ForEach::*`` is preserved; the body's artifact + property is rewritten to a single ``s3://`` URI. +""" + +from pathlib import Path + +from tests.golden.harness import run_package_pipeline + +CASES_ROOT = Path(__file__).parent / "templates" + + +def test_package_sam_case_rewrites_codeuri_to_s3(): + case = CASES_ROOT / "sam_resources" / "serverless_function_zip" + pkg_out = run_package_pipeline(case / "template.yaml", language_extensions=False) + func = pkg_out["Resources"]["HelloFunction"] + assert func["Type"] == "AWS::Serverless::Function" + code_uri = func["Properties"]["CodeUri"] + assert isinstance(code_uri, str) + assert code_uri.startswith("s3://golden-bucket/") + + +def test_package_le_case_preserves_foreach_with_s3_uri(): + """LE templates must keep Fn::ForEach::* through the package pipeline. + + The body's artifact property (CodeUri here) should be rewritten to an + S3 URI string so that, at deploy time, CloudFormation can re-expand the + ForEach and each iteration receives the same packaged artifact. See + ``merge_language_extensions_s3_uris``. + """ + case = CASES_ROOT / "language_extensions" / "foreach_static_zip" + pkg_out = run_package_pipeline(case / "template.yaml", language_extensions=True) + + foreach_keys = [k for k in pkg_out["Resources"] if k.startswith("Fn::ForEach")] + assert foreach_keys, "Fn::ForEach::* must survive the package pipeline for LE templates" + # No expanded ids at the top level. + assert "AlphaFunction" not in pkg_out["Resources"] + assert "BetaFunction" not in pkg_out["Resources"] + + foreach_value = pkg_out["Resources"][foreach_keys[0]] + body = foreach_value[2] + body_resource = next(iter(body.values())) + code_uri = body_resource["Properties"]["CodeUri"] + # Single shared S3 URI for the whole loop (static collection, no + # per-iteration Mappings needed). + assert isinstance(code_uri, str) + assert code_uri.startswith("s3://golden-bucket/") + + +def test_package_cfn_case_rewrites_raw_code_to_s3_dict(): + case = CASES_ROOT / "packageable_resources" / "lambda_function_zip" + pkg_out = run_package_pipeline(case / "template.yaml", language_extensions=False) + code = pkg_out["Resources"]["HelloFunction"]["Properties"]["Code"] + # Raw Lambda::Function packaging emits {S3Bucket, S3Key}. + assert isinstance(code, dict) + assert code.get("S3Bucket") == "golden-bucket" + assert "S3Key" in code + + +def test_package_is_deterministic(): + """Same input produces same output across runs.""" + case = CASES_ROOT / "sam_resources" / "serverless_function_zip" + a = run_package_pipeline(case / "template.yaml", language_extensions=False) + b = run_package_pipeline(case / "template.yaml", language_extensions=False) + assert a == b diff --git a/tests/golden/test_normalize.py b/tests/golden/test_normalize.py new file mode 100644 index 0000000000..c0a91acd00 --- /dev/null +++ b/tests/golden/test_normalize.py @@ -0,0 +1,86 @@ +"""Unit tests for normalize() — deterministic YAML serialization.""" + +from tests.golden.normalize import normalize + + +def test_normalize_produces_trailing_newline(): + out = normalize({"Resources": {}}) + assert out.endswith("\n") + assert not out.endswith("\n\n") + + +def test_normalize_sorts_resources_keys(): + template = { + "Resources": { + "Zeta": {"Type": "AWS::Lambda::Function"}, + "Alpha": {"Type": "AWS::Lambda::Function"}, + }, + } + out = normalize(template) + assert out.index("Alpha") < out.index("Zeta") + + +def test_normalize_sorts_mappings_keys(): + template = { + "Mappings": { + "Z": {"k": {"v": "1"}}, + "A": {"k": {"v": "1"}}, + }, + } + out = normalize(template) + assert out.index("A:") < out.index("Z:") + + +def test_normalize_drops_sam_transform_metrics(): + template = { + "Metadata": { + "SamTransformMetrics": {"foo": "bar"}, + "Other": "keep", + }, + "Resources": {}, + } + out = normalize(template) + assert "SamTransformMetrics" not in out + assert "Other" in out + + +def test_normalize_drops_metadata_block_if_empty_after_filter(): + template = { + "Metadata": {"SamTransformMetrics": {"foo": "bar"}}, + "Resources": {}, + } + out = normalize(template) + assert "Metadata" not in out + + +def test_normalize_is_idempotent(): + template = {"Resources": {"A": {"Type": "T"}}} + once = normalize(template) + import yaml + + twice = normalize(yaml.safe_load(once)) + assert once == twice + + +def test_normalize_preserves_intrinsic_function_dicts(): + template = { + "Resources": { + "F": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": {"ZipFile": {"Fn::Sub": "x-${AWS::Region}"}}, + }, + } + } + } + out = normalize(template) + assert "Fn::Sub" in out + assert "x-${AWS::Region}" in out + + +def test_normalize_uses_block_style_not_flow_style(): + template = {"Resources": {"A": {"Type": "T", "Properties": {"k": "v"}}}} + out = normalize(template) + # block style uses indentation, not braces + assert "{" not in out + assert "}" not in out diff --git a/tests/golden/test_update_goldens.py b/tests/golden/test_update_goldens.py new file mode 100644 index 0000000000..84a60d9b9c --- /dev/null +++ b/tests/golden/test_update_goldens.py @@ -0,0 +1,118 @@ +"""Unit tests for update_goldens.py CLI behavior.""" + +import shutil +from pathlib import Path + +import pytest + +from tests.golden import update_goldens + +REPO_CASES = Path(__file__).parent / "templates" + +# argparse's parser.error() exits with status 2 (Unix convention for usage error). +_ARGPARSE_USAGE_ERROR = 2 + + +@pytest.fixture +def isolated_corpus(tmp_path, monkeypatch): + """Copy the real corpus to tmp_path and point the CLI at it.""" + dest = tmp_path / "templates" + shutil.copytree(REPO_CASES, dest) + monkeypatch.setattr(update_goldens, "TEMPLATES_ROOT", dest) + return dest + + +def test_default_regenerates_all_cases(isolated_corpus): + # Wipe pinned files first, then regenerate + for p in isolated_corpus.rglob("expected.*.yaml"): + p.unlink() + rc = update_goldens.main([]) + assert rc == 0 + # Every case dir now has both expected files + for case_dir in (p.parent for p in isolated_corpus.rglob("template.yaml")): + assert (case_dir / "expected.build.yaml").exists() + assert (case_dir / "expected.package.yaml").exists() + + +def test_filter_only_regenerates_matching(isolated_corpus): + # Pre-populate by running a full regen + update_goldens.main([]) + sam_case = isolated_corpus / "sam_resources" / "serverless_function_zip" + le_case = isolated_corpus / "language_extensions" / "foreach_static_zip" + (sam_case / "expected.build.yaml").write_text("STALE\n") + rc = update_goldens.main(["--filter", "language_extensions/*"]) + assert rc == 0 + # SAM case stays stale + assert (sam_case / "expected.build.yaml").read_text() == "STALE\n" + # LE case regenerated + assert (le_case / "expected.build.yaml").read_text() != "STALE\n" + + +def test_check_returns_nonzero_when_diff(isolated_corpus): + update_goldens.main([]) # pin everything fresh + sam_case = isolated_corpus / "sam_resources" / "serverless_function_zip" + (sam_case / "expected.build.yaml").write_text("STALE\n") + rc = update_goldens.main(["--check"]) + assert rc != 0 + # File NOT modified by --check + assert (sam_case / "expected.build.yaml").read_text() == "STALE\n" + + +def test_check_returns_zero_when_clean(isolated_corpus): + update_goldens.main([]) + rc = update_goldens.main(["--check"]) + assert rc == 0 + + +def test_diff_prints_unified_diff(isolated_corpus, capsys): + update_goldens.main([]) + sam_case = isolated_corpus / "sam_resources" / "serverless_function_zip" + (sam_case / "expected.build.yaml").write_text("STALE\n") + rc = update_goldens.main(["--diff"]) + assert rc != 0 + out = capsys.readouterr().out + assert "STALE" in out + assert "---" in out and "+++" in out + + +def test_new_only_writes_missing_pins(isolated_corpus): + sam_case = isolated_corpus / "sam_resources" / "serverless_function_zip" + # Pre-pin everything + update_goldens.main([]) + # Wipe just one file + (sam_case / "expected.build.yaml").unlink() + # Add an unrelated stale modification to a *different* case + le_case = isolated_corpus / "language_extensions" / "foreach_static_zip" + le_pin = le_case / "expected.build.yaml" + le_pin.write_text("STALE\n") + rc = update_goldens.main(["--new"]) + assert rc == 0 + # SAM case now has the file back + assert (sam_case / "expected.build.yaml").exists() + # LE case stays stale (--new does not touch existing pins) + assert le_pin.read_text() == "STALE\n" + + +def test_new_short_circuits_when_all_pins_exist(isolated_corpus, monkeypatch): + """When every case is fully pinned, --new must not invoke _generate.""" + # Pre-pin everything (uses real _generate). + update_goldens.main([]) + + # Now poison _generate so a single call would blow up. + def boom(*_args, **_kwargs): + raise AssertionError("_generate must not be called when all pins exist under --new") + + monkeypatch.setattr(update_goldens, "_generate", boom) + rc = update_goldens.main(["--new"]) + assert rc == 0 + + +def test_check_and_diff_are_mutually_exclusive(capsys): + """--check and --diff are documented as alternatives. Reject the + combination at the parser, symmetric with the existing --new mutex + checks.""" + with pytest.raises(SystemExit) as excinfo: + update_goldens.main(["--check", "--diff"]) + assert excinfo.value.code == _ARGPARSE_USAGE_ERROR + err = capsys.readouterr().err + assert "--check and --diff are mutually exclusive" in err diff --git a/tests/golden/update_goldens.py b/tests/golden/update_goldens.py new file mode 100644 index 0000000000..75db958eb8 --- /dev/null +++ b/tests/golden/update_goldens.py @@ -0,0 +1,157 @@ +"""CLI for regenerating, checking, or diffing pinned golden outputs. + +Usage: + python tests/golden/update_goldens.py # regenerate all + python tests/golden/update_goldens.py --filter 'sam_*/*' # subset + python tests/golden/update_goldens.py --check # dry-run, exit 1 on diff + python tests/golden/update_goldens.py --diff # like --check + show unified diff + python tests/golden/update_goldens.py --new # only write missing pins +""" + +from __future__ import annotations + +# Allow direct script invocation (`python tests/golden/update_goldens.py`) +# in addition to module form (`python -m tests.golden.update_goldens`). +# When run as a script, Python does not auto-add the repo root to sys.path, +# so the absolute `from tests.golden...` imports below would fail. mypy +# treats the __main__ guard as always-False, hence the `type: ignore`. +if __name__ == "__main__" and __package__ is None: + import sys # type: ignore[unreachable] # noqa: E402 + from pathlib import Path # noqa: E402 + + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +import argparse +import difflib +import fnmatch +import sys +from pathlib import Path +from typing import Iterable, List, Optional, Tuple + +import yaml + +from tests.golden.harness import run_build_and_package +from tests.golden.normalize import normalize + +TEMPLATES_ROOT = Path(__file__).parent / "templates" + + +def _read_metadata(case_dir: Path) -> dict: + md = case_dir / "metadata.yaml" + if not md.exists(): + return {} + with open(md, "r", encoding="utf-8") as f: + return yaml.safe_load(f) or {} + + +def _resolve_le_default(case_dir: Path, meta: dict) -> bool: + if "language_extensions" in meta: + return bool(meta["language_extensions"]) + rel = case_dir.relative_to(TEMPLATES_ROOT) + return rel.parts[0] == "language_extensions" + + +def _generate(case_dir: Path) -> Tuple[str, str]: + """Return (build_yaml, package_yaml) strings for one case.""" + meta = _read_metadata(case_dir) + le_enabled = _resolve_le_default(case_dir, meta) + build_dict, pkg_dict = run_build_and_package(case_dir / "template.yaml", language_extensions=le_enabled) + return normalize(build_dict), normalize(pkg_dict) + + +def _iter_cases(filter_glob: Optional[str]) -> Iterable[Path]: + cases = sorted(p.parent for p in TEMPLATES_ROOT.rglob("template.yaml")) + if not filter_glob: + yield from cases + return + for c in cases: + rel = str(c.relative_to(TEMPLATES_ROOT)) + if fnmatch.fnmatch(rel, filter_glob): + yield c + + +def _existing(case_dir: Path, kind: str) -> Optional[str]: + p = case_dir / f"expected.{kind}.yaml" + return p.read_text(encoding="utf-8") if p.exists() else None + + +def _print_diff(case_rel: str, kind: str, current: Optional[str], generated: str) -> None: + cur_lines = (current or "").splitlines(keepends=True) + gen_lines = generated.splitlines(keepends=True) + diff = difflib.unified_diff( + cur_lines, + gen_lines, + fromfile=f"a/{case_rel}/expected.{kind}.yaml", + tofile=f"b/{case_rel}/expected.{kind}.yaml", + lineterm="", + ) + for line in diff: + print(line, end="" if line.endswith("\n") else "\n") + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--filter", help="glob applied to case path relative to templates/") + parser.add_argument("--check", action="store_true", help="dry-run; exit 1 on any diff") + parser.add_argument("--diff", action="store_true", help="like --check + print unified diff") + parser.add_argument( + "--new", + action="store_true", + help="only write missing expected.*.yaml; never overwrite existing", + ) + args = parser.parse_args(argv) + + if args.check and args.new: + parser.error("--check and --new are mutually exclusive") + if args.diff and args.new: + parser.error("--diff and --new are mutually exclusive") + if args.check and args.diff: + # The implementation tolerates this (both imply dry_run, --diff + # wins on output), but the docstring describes them as + # alternatives. Reject at the parser, symmetric with the --new + # checks above. + parser.error("--check and --diff are mutually exclusive") + + dry_run = args.check or args.diff + any_diff = False + + for case_dir in _iter_cases(args.filter): + case_rel = str(case_dir.relative_to(TEMPLATES_ROOT)) + + # --new fast path: if both pins already exist, skip the build+package + # pipeline entirely. Only invoke _generate when at least one pin is + # missing, then write only the missing files. + if args.new: + build_existing = _existing(case_dir, "build") + pkg_existing = _existing(case_dir, "package") + if build_existing is not None and pkg_existing is not None: + continue + build_yaml, pkg_yaml = _generate(case_dir) + if build_existing is None: + (case_dir / "expected.build.yaml").write_text(build_yaml, encoding="utf-8") + if pkg_existing is None: + (case_dir / "expected.package.yaml").write_text(pkg_yaml, encoding="utf-8") + continue + + build_yaml, pkg_yaml = _generate(case_dir) + + for kind, generated in (("build", build_yaml), ("package", pkg_yaml)): + existing = _existing(case_dir, kind) + if dry_run: + if existing != generated: + any_diff = True + if args.diff: + _print_diff(case_rel, kind, existing, generated) + else: + print(f"WOULD CHANGE: {case_rel}/expected.{kind}.yaml") + continue + if existing != generated: + (case_dir / f"expected.{kind}.yaml").write_text(generated, encoding="utf-8") + + if dry_run and any_diff: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main())