diff --git a/.github/actions/free-space/action.yml b/.github/actions/free-space/action.yml new file mode 100644 index 00000000..99ec9ecb --- /dev/null +++ b/.github/actions/free-space/action.yml @@ -0,0 +1,73 @@ +# Reusable free-disk-space step for the persistent self-hosted runners (#567 / +# REQ-271). Rust + Bazel + Docker layer caches accumulate across runs on a +# long-lived runner until the disk fills and every compile gate fails; the +# old post-job cleanup hook freed nothing. This prunes the space hogs and +# reports before/after free bytes so the effect is auditable — never a silent +# no-op. Idempotent and safe to call at the top of any compile-heavy job. +name: Free disk space +description: > + Prune cargo / Bazel / Docker caches and /tmp on a persistent runner, and + report free space before and after so the cleanup is auditable (REQ-271, #567). + +inputs: + cargo-target-cap-gb: + description: > + Delete cargo target directories under the runner work tree when the + total free space is below this many GiB. 0 disables the cargo sweep. + required: false + default: "0" + +runs: + using: composite + steps: + - name: Free disk space (before/after audit) + shell: bash + run: | + set -uo pipefail + avail_kb() { df -Pk / | awk 'NR==2 {print $4}'; } + before=$(avail_kb) + echo "::group::disk before cleanup" + df -h / || true + echo "::endgroup::" + + # Docker: dangling images + build cache are the biggest recurring hog + # on a self-hosted runner that also builds containers. + if command -v docker >/dev/null 2>&1; then + echo "pruning docker system (images + build cache)..." + docker system prune -af --volumes >/dev/null 2>&1 || true + docker builder prune -af >/dev/null 2>&1 || true + fi + + # Bazel disk cache: the Rocq/Bazel jobs leave a large external+disk + # cache. Clear the shared disk cache dir if present. + for d in "$HOME/.cache/bazel" "$HOME/.bazel" /tmp/bazel-disk-cache; do + [ -d "$d" ] && { echo "removing bazel cache $d..."; rm -rf "$d" || true; } + done + + # Cargo registry cache grows unbounded across runs; the registry + # sources/cache are re-fetched on demand, so they are safe to drop. + for d in "$HOME/.cargo/registry/cache" "$HOME/.cargo/registry/src" "$HOME/.cargo/git/checkouts"; do + [ -d "$d" ] && { echo "removing cargo cache $d..."; rm -rf "$d" || true; } + done + + # Optional: sweep stale cargo target dirs under the work tree when we + # are genuinely low on space (guards against nuking an in-use target). + cap="${{ inputs.cargo-target-cap-gb }}" + if [ "$cap" != "0" ]; then + free_gb=$(( $(avail_kb) / 1024 / 1024 )) + if [ "$free_gb" -lt "$cap" ]; then + echo "free ${free_gb}GiB < ${cap}GiB cap; sweeping stale target/ dirs..." + find "${RUNNER_WORKSPACE:-$HOME}" -maxdepth 4 -type d -name target -prune \ + -exec rm -rf {} + 2>/dev/null || true + fi + fi + + # Ephemeral scratch. + rm -rf /tmp/* 2>/dev/null || true + + after=$(avail_kb) + echo "::group::disk after cleanup" + df -h / || true + echo "::endgroup::" + freed_mb=$(( (after - before) / 1024 )) + echo "free-space: ${freed_mb} MiB reclaimed (before=$((before/1024)) MiB, after=$((after/1024)) MiB free)" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a075ffd..b816cbed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,7 @@ jobs: runs-on: [self-hosted, linux, x64, light] outputs: rust: ${{ steps.filter.outputs.rust }} + wasm: ${{ steps.filter.outputs.wasm }} steps: - uses: actions/checkout@v6 with: @@ -48,6 +49,7 @@ jobs: run: | if [ "${{ github.event_name }}" != "pull_request" ]; then echo "rust=true" >> "$GITHUB_OUTPUT" + echo "wasm=true" >> "$GITHUB_OUTPUT" echo "non-PR event (${{ github.event_name }}) → full matrix" exit 0 fi @@ -61,6 +63,19 @@ jobs: echo "rust=false" >> "$GITHUB_OUTPUT" echo "→ no Rust/Cargo/workflow changes; skipping compile-heavy jobs" fi + # REQ-268: the wasm SEAM (compose-witness component + WIT, the host + # `wasm` feature / wasm_runtime, the composition core it wraps, and + # the spar-wasm build assets) is consumed downstream (melded → + # loomed → synth'd), so a change that rots it is a downstream break. + # It is built only in release.yml today, so gate a cheap per-PR + # host-side build on these paths. + if echo "$changed" | grep -qE '(^compose-witness/)|(wasm_runtime\.rs$)|(feature_model\.rs$)|(\.wit$)|(scripts/build-wasm\.sh$)|(serve/js\.rs$)|(assets/wasm/)|(^\.github/workflows/)'; then + echo "wasm=true" >> "$GITHUB_OUTPUT" + echo "→ wasm-seam paths changed; running the wasm-seam build gate" + else + echo "wasm=false" >> "$GITHUB_OUTPUT" + echo "→ no wasm-seam changes; skipping the wasm-seam gate" + fi # ── Fast checks ─────────────────────────────────────────────────────── fmt: @@ -93,6 +108,36 @@ jobs: - name: Minimal build (--no-default-features) stays clean run: cargo clippy -p rivet-cli --no-default-features -- -D warnings + # REQ-268: build the wasm SEAM per-PR (not only in release.yml) so a change + # to the composition core / wasm_runtime / compose-witness / WIT / `wasm` + # feature can't rot the component consumed downstream (melded → loomed → + # synth'd) undetected until release. The host-side `--features wasm` build + # exercises wasm_runtime.rs + the wasmtime integration on the normal + # toolchain (no wasi-sdk / cargo-component needed), so it is cheap and + # reliable — unlike the full wasm32-wasip2 spar build, which is left to + # release.yml. Gated on the wasm-seam paths (see the `changes` filter). + wasm-seam: + name: WASM seam build + needs: changes + if: needs.changes.outputs.wasm == 'true' + runs-on: [self-hosted, linux, x64, rust-cpu] + steps: + - uses: actions/checkout@v6 + # REQ-271 / #567: reclaim disk before a compile-heavy job on the + # persistent runner. + - uses: ./.github/actions/free-space + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Build host wasm seam (rivet-cli --features wasm) + run: cargo build -p rivet-cli --features wasm + # NOTE: the compose-witness component's `src/bindings.rs` is generated by + # cargo-component / wit-bindgen (gitignored), so a plain `cargo check` + # can't compile it — it needs `cargo component build`, which requires the + # wasi-sdk toolchain. That full component build stays in release.yml; this + # per-PR gate is deliberately the cheap, reliable host-side `--features + # wasm` build (REQ-268 "at minimum"), which still exercises wasm_runtime.rs + # + the wasmtime integration + the composition core the component wraps. + yaml-lint: name: YAML Lint runs-on: [self-hosted, linux, x64, light] @@ -219,6 +264,28 @@ jobs: exit 1 fi + # REQ-272 / #509: single-point-of-failure mitigation. Every gate above runs + # on the self-hosted pool; when it goes offline, ALL of main's verification + # queues forever (slice 1 shipped the liveness alert). This job mirrors the + # single most load-bearing gate — `rivet validate` (the trace graph must + # carry no errors) — on GitHub-hosted `ubuntu-latest`, so a self-hosted + # outage cannot leave a main commit with zero traceability verification. + # `push`-only (not every PR) so it does not burn hosted minutes on the PR + # path, where the self-hosted gate already covers it. Compile-heavy gates + # stay self-hosted; this is a floor, not a full mirror. + traceability-hosted-fallback: + name: Traceability (hosted fallback) + if: github.event_name == 'push' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: rivet validate (fail on errors) — hosted floor + run: cargo run --release -p rivet-cli -- validate + # ── Tests ───────────────────────────────────────────────────────────── test: name: Test diff --git a/artifacts/requirements.yaml b/artifacts/requirements.yaml index 60635466..0bdd3851 100644 --- a/artifacts/requirements.yaml +++ b/artifacts/requirements.yaml @@ -7865,7 +7865,7 @@ artifacts: - id: REQ-268 type: requirement title: "per-PR CI gate builds the wasm seam (compose-witness component + wasm feature)" - status: proposed + status: implemented description: "v0.30 (CI resilience). rivet's composition core is exposed as a Wasm component — compose-witness/ (cdylib + WIT in compose-witness/wit/, built via `cargo component build` or Bazel) — plus the host `wasm` feature (rivet-core/src/wasm_runtime.rs, wasmtime) and the spar-wasm browser assets. That component is the SEAM consumed downstream (melded → loomed → synth'd), so a broken/buggy wasm build is a downstream breakage. Today the wasm seam is built ONLY in release.yml, not per-PR ci.yml, so a change can rot the seam and nothing catches it until release. Add a per-PR CI job that at minimum runs `cargo build -p rivet-cli --features wasm` (cheap, host-side) and ideally `cargo component build` on compose-witness (needs cargo-component + wasi-sdk on the runner, mirror the release.yml wasm setup). Gate it behind the existing changes/paths-filter so it only runs when the composition core / wasm_runtime / compose-witness / WIT / wasm feature / spar-wasm assets change." release: v0.30.0 provenance: @@ -7898,7 +7898,7 @@ artifacts: - id: REQ-271 type: requirement title: "self-hosted CI runner disk cleanup that actually frees space (#567)" - status: proposed + status: implemented description: "v0.30 (CI resilience). #567: the self-hosted runner pool exhausts disk and every compile gate fails; the existing post-job cleanup hook frees nothing. Rust + Bazel + Docker layer caches accumulate across runs on a persistent runner. Add a reliable free-space step (a reusable composite action or a leading ci.yml step) that prunes the space hogs — `cargo` target/registry caches beyond a cap, Bazel disk cache, docker/buildx dangling images+build cache, and /tmp — and reports before/after free bytes so the effect is auditable (not a silent no-op like the current hook). Runs early enough to unblock the compile, and idempotent." release: v0.30.0 provenance: @@ -7909,7 +7909,7 @@ artifacts: - id: REQ-272 type: requirement title: "CI single-point-of-failure mitigation: hosted fallback + surfaced alert (#509 slice 2)" - status: proposed + status: implemented description: "v0.30 (CI resilience). #509: every gating job is `runs-on: [self-hosted, ...]`, so when the pool goes offline all gates queue forever with no fallback (slice 1 shipped the liveness-alert workflow; this is slice 2). Reduce the SPOF: make at least the light, portable gates (e.g. Traceability `rivet validate` + docs check, YAML lint) able to run on GitHub-hosted `ubuntu-latest` as a fallback so a self-hosted outage does not zero out ALL verification on main — via a runner-label group or a hosted mirror job gated to run only when self-hosted is starved. Keep compile-heavy gates self-hosted. Document the SPOF + the liveness signal in AGENTS.md/ci so an outage is diagnosable, not a mystery." release: v0.30.0 provenance: