diff --git a/.github/workflows/perf-numbers.yml b/.github/workflows/perf-numbers.yml index 8f3b8a05..90cacc4f 100644 --- a/.github/workflows/perf-numbers.yml +++ b/.github/workflows/perf-numbers.yml @@ -495,7 +495,19 @@ jobs: timeout-minutes: 2 uses: Ambiguous-Interactive/ambiguous-organization-build-lock/.github/actions/require-confirmed-unity-cleanup@ce12fde93e67a27062350601fb97e4a2f546d405 with: - acquired: ${{ steps.acquire_lock.outputs.acquired }} + # A leg that aborts before `Acquire organization Unity lock` runs never + # took a seat, and this gate's own contract treats `acquired: false` as + # proof that licensed cleanup was not required. Feeding it the skipped + # step's EMPTY output instead made it fail closed on work that never + # started, so a leg with one honest failure reported two (#327). + # + # This does not weaken the gate. It reads `outcome`, not `acquired`, and + # only the two values that prove the step never executed map to 'false'. + # An acquire that RAN and failed part-way reports outcome=failure while + # `acquired` reads empty, so the empty value still flows through and the + # gate still fails -- that ambiguous shape is the seat leak the bare + # `if: always()` above exists to catch, and it stays caught. + acquired: ${{ (steps.acquire_lock.outcome == 'skipped' || steps.acquire_lock.outcome == '') && 'false' || steps.acquire_lock.outputs.acquired }} classification-complete: ${{ steps.cleanup_classification.outputs.classification-complete }} cleanup-status: ${{ steps.cleanup_classification.outputs.resource-cleanup-status }} cleanup-health: ${{ steps.cleanup_classification.outputs.resource-health }} diff --git a/.github/workflows/post-merge-maintenance.yml b/.github/workflows/post-merge-maintenance.yml new file mode 100644 index 00000000..a979e907 --- /dev/null +++ b/.github/workflows/post-merge-maintenance.yml @@ -0,0 +1,354 @@ +--- +name: Post-Merge Maintenance + +# One flow for every generated file the default branch self-heals, so a merge +# produces at most ONE follow-up commit instead of one per generator (#330). +# It replaces update-llms-txt.yml and update-issue-template-versions.yml, which +# were the same 180-line workflow twice: same App token, same branch refresh, +# same three-attempt push loop, different `npm run` line. Two of them could land +# two bot commits on master for a single merge, and each of those commits +# re-triggered the full push-side workflow set. +# +# What it regenerates, and why each is self-healing rather than only gated: +# * llms.txt + the README skill-count claim (`check:llms-txt` gates PRs). +# * The bug-report package-version dropdown (`check:issue-template-versions` +# gates PRs; release-prepare.yml also regenerates it proactively so the +# release PR passes its own gate). +# The `--check` gates still block a stale hand-edit on a pull request. This is +# the backstop for every path that reaches master without running them. +# +# Deliberately NOT folded in: the perf-numbers.yml doc + baseline auto-commit. +# It runs on a self-hosted Windows runner behind the organization Unity build +# lock and can take tens of minutes, so collapsing these two-minute generators +# behind it would delay them by the benchmark's whole queue for no benefit. +# +# The built-in GITHUB_TOKEN cannot push to a protected branch +# (github-actions[bot] is a system account, not a selectable branch-protection +# bypass actor), so the push is authenticated by a GitHub App installation +# token (secrets.AUTO_COMMIT_APP_ID + secrets.AUTO_COMMIT_APP_PRIVATE_KEY) +# whose App is allowed to bypass branch protection. If those secrets are absent +# the commit degrades to a logged no-op (regeneration + validation still run). +# See docs/runbooks/perf-numbers-auto-commit.md. +on: + schedule: + # Weekly on Monday at 9:00 AM UTC. One slot now covers both generators; the + # 10:00 slot the issue-template workflow used to occupy is retired. + - cron: "0 9 * * 1" + workflow_dispatch: + push: + branches: + - main + - master + # The union of both generators' inputs, plus the one output the retired + # issue-template workflow already watched so a hand-edited dropdown is + # corrected without waiting for the weekly cron. + # + # llms.txt is deliberately NOT listed even though it is the other output. + # This job's own commit is pushed with an App token, which (unlike the + # built-in GITHUB_TOKEN) re-triggers workflows, so listing it would make + # every real regeneration schedule a second, guaranteed no-op run -- the + # churn this workflow exists to reduce. A hand-edit to llms.txt is already + # blocked on the pull request by `check:llms-txt`. + paths: + - package.json + - CHANGELOG.md + - .llm/skills/** + - .github/ISSUE_TEMPLATE/bug_report.yml + - scripts/update-llms-txt.js + - scripts/generate-issue-template-versions.js + - .github/workflows/post-merge-maintenance.yml + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# The push is authenticated by the GitHub App installation token, not the job's +# GITHUB_TOKEN, so contents:read is all this job's own token needs (checkout). +permissions: + contents: read + +jobs: + regenerate: + name: Regenerate committed artifacts + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + # Single source for what each generator owns. Every stage -- the two + # regeneration steps, the change probe, the staging step, and the push + # loop -- reads these, so a new generator is added in one place and + # cannot be half-wired. They are kept SEPARATE, not merged into one list, + # because each generator reverts only its own files when it fails. + LLMS_PATHS: "llms.txt README.md" + ISSUE_TEMPLATE_PATHS: ".github/ISSUE_TEMPLATE/bug_report.yml" + + steps: + - name: Check for the auto-commit GitHub App credentials + id: check-autocommit-app + # Degrade to a logged no-op when the App is not provisioned, instead of a + # red build. See docs/runbooks/perf-numbers-auto-commit.md. + env: + AUTO_COMMIT_APP_ID: ${{ secrets.AUTO_COMMIT_APP_ID }} + AUTO_COMMIT_APP_PRIVATE_KEY: ${{ secrets.AUTO_COMMIT_APP_PRIVATE_KEY }} + run: | + set -euo pipefail + if [ -n "${AUTO_COMMIT_APP_ID:-}" ] && \ + [ -n "${AUTO_COMMIT_APP_PRIVATE_KEY:-}" ]; then + echo "has-app=true" >> "${GITHUB_OUTPUT}" + else + echo "::warning::AUTO_COMMIT_APP_* secrets are not set; skipping the post-merge auto-commit. See docs/runbooks/perf-numbers-auto-commit.md." + echo "has-app=false" >> "${GITHUB_OUTPUT}" + fi + + - name: Generate auto-commit GitHub App token + id: app-token + if: steps.check-autocommit-app.outputs.has-app == 'true' + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 + with: + app-id: ${{ secrets.AUTO_COMMIT_APP_ID }} + private-key: ${{ secrets.AUTO_COMMIT_APP_PRIVATE_KEY }} + + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + token: ${{ steps.app-token.outputs.token || github.token }} + persist-credentials: false + + - name: Refresh branch before generating + if: steps.check-autocommit-app.outputs.has-app == 'true' + env: + GH_FETCH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -euo pipefail + auth_header="$(printf 'x-access-token:%s' "${GH_FETCH_TOKEN}" | base64 | tr -d '\n')" + git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic ${auth_header}" \ + fetch --no-tags --depth=1 origin \ + "+refs/heads/${GITHUB_REF_NAME}:refs/remotes/origin/${GITHUB_REF_NAME}" + git checkout -B "${GITHUB_REF_NAME}" "refs/remotes/origin/${GITHUB_REF_NAME}" + echo "Refreshed ${GITHUB_REF_NAME} to $(git rev-parse HEAD) before regenerating." + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22.18.0" + cache: npm + cache-dependency-path: package.json + + - name: Install dependencies + run: | + if [ -f package-lock.json ]; then + npm ci + else + npm i --no-audit --no-fund + fi + + # One step per generator, and the second runs even when the first failed. + # Merging these into a single `set -e` block would re-couple what two + # independent workflows used to keep apart: a `bug_report.yml` the + # dropdown generator cannot process would abort the step before the + # llms.txt drift on master ever got corrected, and vice versa. + # + # Each generator runs its own `--check` immediately after, and REVERTS + # its own files if that check rejects them. That is what makes it safe + # for the commit step to run after a failure: the tree can only ever + # carry output a checker accepted, never a half-written file. The job + # still goes red at the end, so nothing is swallowed. + - name: Regenerate llms.txt + id: gen_llms + run: | + set -uo pipefail + if npm run update:llms-txt && npm run check:llms-txt; then + exit 0 + fi + echo "::error::llms.txt generation did not converge; discarding its output." + # shellcheck disable=SC2086 # LLMS_PATHS is an intentional word list. + git checkout -- ${LLMS_PATHS} + exit 1 + + - name: Regenerate the issue-template version dropdown + id: gen_issue_template + # `!cancelled()`, NOT `always()`. Both run after a failed sibling, which + # is the isolation this step exists for -- but `always()` also runs after + # a CANCEL, and `cancel-in-progress: true` above makes that routine: a + # second push kills the first mid-flight. A generator killed mid-write + # never reaches its own revert, so continuing past a cancel is how a + # half-written file becomes a commit on the default branch. (Bugbot.) + if: ${{ !cancelled() }} + run: | + set -uo pipefail + if npm run update:issue-template-versions && npm run check:issue-template-versions; then + exit 0 + fi + echo "::error::issue-template version generation did not converge; discarding its output." + # shellcheck disable=SC2086 # ISSUE_TEMPLATE_PATHS is an intentional word list. + git checkout -- ${ISSUE_TEMPLATE_PATHS} + exit 1 + + - name: Check for changes + id: git-check + if: ${{ !cancelled() }} + run: | + set -euo pipefail + # `--exit-code` returns 1 for "there are differences" but 128/129 for + # "not a git repository" / bad usage, and a bare `|| echo changed=true` + # swallows all of them alike. That was harmless while a failed + # `Checkout` skipped this step; `always()` made it reachable, and + # fail-OPEN is the wrong direction for a probe that decides whether to + # commit. Only exit 1 means "changed". + set +e + # shellcheck disable=SC2086 # both are intentional word lists. + git diff --quiet -- ${LLMS_PATHS} ${ISSUE_TEMPLATE_PATHS} + diff_status=$? + set -e + case "${diff_status}" in + 0) echo "No generated file changed." ;; + 1) echo "changed=true" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::git could not compare the generated files (exit ${diff_status}); refusing to guess whether they changed." + exit "${diff_status}" + ;; + esac + + # Runs after a FAILED generator (the revert above guarantees the tree + # carries only checker-accepted output, so the generator that DID converge + # still self-heals) but NOT after a cancel, where a killed generator never + # reached its revert and the tree may be half-written. + - name: Commit and push changes + if: >- + ${{ !cancelled() && steps.git-check.outputs.changed == 'true' && + steps.check-autocommit-app.outputs.has-app == 'true' }} + env: + GH_PUSH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -euo pipefail + auth_header="$(printf 'x-access-token:%s' "${GH_PUSH_TOKEN}" | base64 | tr -d '\n')" + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + # shellcheck disable=SC2206 # both are intentional word lists. + generated=(${LLMS_PATHS} ${ISSUE_TEMPLATE_PATHS}) + + # Same isolation as the two steps above: on the stale-head path a + # generator that cannot converge reverts its own files rather than + # aborting the push loop and stranding the other generator's output. + # + # It must also RECORD that it failed. An earlier version ended each + # branch in `|| git checkout`, which succeeds, so the function always + # returned 0: a generator that broke only on the refreshed head + # reverted silently, the loop pushed the other generator's output, and + # the job went green with stale content and no annotation. The two + # step outcomes the terminal gate reads were both `success`, because + # the failure happened here rather than there. + regenerate_failed=0 + regenerate() { + # Reset per call: the flag must describe the regeneration whose output + # actually gets pushed, not every attempt ever made. Leaving it sticky + # meant a transient failure on an intermediate head reddened the job + # even after a later retry converged and pushed good content -- + # trading the old always-green bug for an always-red one. + regenerate_failed=0 + # shellcheck disable=SC2086 # intentional word lists. + if ! (npm run update:llms-txt && npm run check:llms-txt); then + echo "::error::llms.txt generation did not converge on the refreshed head; discarding its output." + # shellcheck disable=SC2086 + git checkout -- ${LLMS_PATHS} + regenerate_failed=1 + fi + if ! (npm run update:issue-template-versions && npm run check:issue-template-versions); then + echo "::error::issue-template version generation did not converge on the refreshed head; discarding its output." + # shellcheck disable=SC2086 + git checkout -- ${ISSUE_TEMPLATE_PATHS} + regenerate_failed=1 + fi + } + + fetch_head() { + git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic ${auth_header}" \ + fetch --no-tags --depth=1 origin \ + "+refs/heads/${GITHUB_REF_NAME}:refs/remotes/origin/${GITHUB_REF_NAME}" + } + + for attempt in 1 2 3; do + fetch_head + latest_sha="$(git rev-parse "refs/remotes/origin/${GITHUB_REF_NAME}")" + current_sha="$(git rev-parse HEAD)" + if [ "${latest_sha}" != "${current_sha}" ]; then + message="Default branch ${GITHUB_REF_NAME} advanced from ${current_sha} " + message+="to ${latest_sha}; regenerating on the latest head before pushing." + echo "::warning::${message}" + git checkout -f -B "${GITHUB_REF_NAME}" "refs/remotes/origin/${GITHUB_REF_NAME}" + regenerate + fi + + if git diff --quiet -- "${generated[@]}"; then + echo "Generated artifacts are already current; nothing to commit." + exit "${regenerate_failed}" + fi + + base_sha="$(git rev-parse HEAD)" + git add -- "${generated[@]}" + # Belt-and-braces after the unstaged check above. Deliberately NOT + # pinned by a test: `git add` makes the staged and unstaged views + # agree, so reaching here needs them to disagree, and every fixture + # that tries ends up exercising the branch above instead. A case for + # a branch that cannot be reached would only look like coverage. + if git diff --cached --quiet -- "${generated[@]}"; then + echo "No staged changes remain; nothing to commit." + exit "${regenerate_failed}" + fi + git commit -m "chore: refresh generated repository metadata" + + set +e + git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic ${auth_header}" \ + push origin "HEAD:${GITHUB_REF_NAME}" + push_status=$? + set -e + if [ "${push_status}" -eq 0 ]; then + exit "${regenerate_failed}" + fi + + fetch_head + latest_after_push="$(git rev-parse "refs/remotes/origin/${GITHUB_REF_NAME}")" + if [ "${latest_after_push}" = "${base_sha}" ]; then + echo "::error::post-merge auto-commit push failed while ${GITHUB_REF_NAME} still pointed at ${base_sha}." + exit "${push_status}" + fi + + if [ "${attempt}" -eq 3 ]; then + message="Default branch ${GITHUB_REF_NAME} kept advancing during the " + message+="post-merge auto-commit; skipping this stale attempt. " + message+="A newer run will regenerate from the latest head." + echo "::warning::${message}" + exit "${regenerate_failed}" + fi + + message="Default branch ${GITHUB_REF_NAME} advanced during the post-merge " + message+="push; retrying from ${latest_after_push}." + echo "::warning::${message}" + git checkout -f -B "${GITHUB_REF_NAME}" "refs/remotes/origin/${GITHUB_REF_NAME}" + regenerate + done + + # The commit step is allowed to run after a failed generator, so the job's + # own verdict has to come from the generators themselves. Without this a + # generator that never converged would be reported green. + - name: Require every generator to have converged + # A cancelled run is already cancelled; adding a gate failure on top only + # obscures why it stopped. + if: ${{ !cancelled() }} + run: | + set -euo pipefail + failed=0 + for outcome in "${LLMS_OUTCOME}" "${ISSUE_TEMPLATE_OUTCOME}"; do + case "${outcome}" in + success|skipped) ;; + *) failed=1 ;; + esac + done + if [ "${failed}" -ne 0 ]; then + echo "::error::a generator did not converge (llms.txt=${LLMS_OUTCOME}, issue-template=${ISSUE_TEMPLATE_OUTCOME}); see the annotations above." + exit 1 + fi + echo "Every generator converged." + env: + LLMS_OUTCOME: ${{ steps.gen_llms.outcome }} + ISSUE_TEMPLATE_OUTCOME: ${{ steps.gen_issue_template.outcome }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ca94749d..563c6dad 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -459,7 +459,19 @@ jobs: timeout-minutes: 2 uses: Ambiguous-Interactive/ambiguous-organization-build-lock/.github/actions/require-confirmed-unity-cleanup@ce12fde93e67a27062350601fb97e4a2f546d405 with: - acquired: ${{ steps.acquire_lock.outputs.acquired }} + # A leg that aborts before `Acquire organization Unity lock` runs never + # took a seat, and this gate's own contract treats `acquired: false` as + # proof that licensed cleanup was not required. Feeding it the skipped + # step's EMPTY output instead made it fail closed on work that never + # started, so a leg with one honest failure reported two (#327). + # + # This does not weaken the gate. It reads `outcome`, not `acquired`, and + # only the two values that prove the step never executed map to 'false'. + # An acquire that RAN and failed part-way reports outcome=failure while + # `acquired` reads empty, so the empty value still flows through and the + # gate still fails -- that ambiguous shape is the seat leak the bare + # `if: always()` above exists to catch, and it stays caught. + acquired: ${{ (steps.acquire_lock.outcome == 'skipped' || steps.acquire_lock.outcome == '') && 'false' || steps.acquire_lock.outputs.acquired }} classification-complete: ${{ steps.cleanup_classification.outputs.classification-complete }} cleanup-status: ${{ steps.cleanup_classification.outputs.resource-cleanup-status }} cleanup-health: ${{ steps.cleanup_classification.outputs.resource-health }} @@ -732,7 +744,19 @@ jobs: timeout-minutes: 2 uses: Ambiguous-Interactive/ambiguous-organization-build-lock/.github/actions/require-confirmed-unity-cleanup@ce12fde93e67a27062350601fb97e4a2f546d405 with: - acquired: ${{ steps.acquire_lock.outputs.acquired }} + # A leg that aborts before `Acquire organization Unity lock` runs never + # took a seat, and this gate's own contract treats `acquired: false` as + # proof that licensed cleanup was not required. Feeding it the skipped + # step's EMPTY output instead made it fail closed on work that never + # started, so a leg with one honest failure reported two (#327). + # + # This does not weaken the gate. It reads `outcome`, not `acquired`, and + # only the two values that prove the step never executed map to 'false'. + # An acquire that RAN and failed part-way reports outcome=failure while + # `acquired` reads empty, so the empty value still flows through and the + # gate still fails -- that ambiguous shape is the seat leak the bare + # `if: always()` above exists to catch, and it stays caught. + acquired: ${{ (steps.acquire_lock.outcome == 'skipped' || steps.acquire_lock.outcome == '') && 'false' || steps.acquire_lock.outputs.acquired }} classification-complete: ${{ steps.cleanup_classification.outputs.classification-complete }} cleanup-status: ${{ steps.cleanup_classification.outputs.resource-cleanup-status }} cleanup-health: ${{ steps.cleanup_classification.outputs.resource-health }} diff --git a/.github/workflows/stuck-job-watchdog.yml b/.github/workflows/stuck-job-watchdog.yml index f27606a2..19c8ca8b 100644 --- a/.github/workflows/stuck-job-watchdog.yml +++ b/.github/workflows/stuck-job-watchdog.yml @@ -6,6 +6,35 @@ name: Stuck Job Watchdog # 7+ minutes, so a queued job whose label set is satisfied by an idle runner # nonetheless waits indefinitely until manually re-run. # +# FAIL CLOSED (#328). A green run means the audit ran to completion. Every +# input the verdict depends on -- the queued-run listing, the organization +# runner inventory, a per-run job listing, and the cancel-cap state branch -- +# fails the job when it cannot be read, instead of logging and exiting 0. +# Before this rule the inventory read was unauthorized on both endpoints the +# watchdog tried and the job still reported success, so a `Performance Numbers` +# run sat queued for ten hours while the automation meant to notice it reported +# green every five minutes. +# +# Runner inventory is ORGANIZATION-scoped and is read with the build-lock +# reader App (`organization_self_hosted_runners: read`), the same credential +# `check-unity-runner-availability` already uses. The job's own GITHUB_TOKEN is +# repository-scoped and can never reach `/orgs/{org}/actions/runner-groups`. +# The repository-level fallback this workflow used to attempt was worse than +# no fallback: this repository registers zero repository-level runners, so the +# call would succeed and report an empty inventory, which reads identically to +# "no runner matches" and still issues no action. +# +# The inventory walk asks for runner groups `visible_to_repository` this +# repository, the same call shape `check-unity-runner-availability` makes in +# production. Note what that does and does not buy today: the organization +# currently registers a single group ("Default"), so visible-to-us and +# all-org-runners are the same set, and the parameter is insurance for the day a +# restricted second group is added rather than a guarantee being relied on now. +# The counted group names are logged for exactly that reason -- if a second group +# ever appears in the summary, the filter's behavior is worth confirming before +# trusting it, because over-counting capacity is what could turn a legitimately +# queued run into a wrongful cancel. +# # Detection requires ALL of the following to be true: # * The workflow run is `status: queued` AND older than MIN_QUEUE_AGE_SECONDS # (default 300s / 5 min; round-2 false-positive guards below -- zero @@ -16,13 +45,20 @@ name: Stuck Job Watchdog # dispatcher-stuck. This is what prevents false positives for matrix cells # or jobs waiting while another job from the same run is actively running. # * At least one job in the run is `status: queued`. -# * At least one idle runner's labels satisfy a queued job's label -# requirements (superset match). +# * At least one ONLINE, NOT-BUSY runner's labels satisfy a queued job's +# label requirements (superset match, case-insensitive on both sides). # * The run's workflow file is NOT in the exclusion list (`release.yml` # is hard-excluded by default; additional entries may be added via the # `WATCHDOG_EXCLUDED_WORKFLOWS` repo variable, whitespace-separated). # * The run id is NOT the watchdog's own run id. # +# A queued run that no ONLINE runner can satisfy is NOT dispatcher-stuck -- +# cancelling it would destroy work the fleet will pick up as soon as the +# machine reconnects. It is reported in its own step-summary section and as a +# `::warning::` annotation, so an offline runner starving the queue is visible +# on the run page without opening a job log. That is the shape #328 hit: a +# registered-but-offline `fast` runner, invisible because the audit was blind. +# # Recovery action (per cli/cli#9221 and the gh-run-rerun manual, # `gh run rerun --failed` cannot be used on a `status: queued` run because # the run never reached `failed`; the documented workaround is cancel + @@ -38,8 +74,12 @@ name: Stuck Job Watchdog # the PR, or escalate any other automatic action. # # Cancel attempts are capped at 2 per run-id per 24h via a small state file -# on the `watchdog-state` orphan branch. State is pushed immediately after each -# successful cancel, with a single rebase+retry on push failure. +# on the `watchdog-state` orphan branch. The state branch is materialized only +# after a run is classified dispatcher-stuck, so a cycle that finds nothing to +# cancel -- the overwhelming majority -- performs no clone and cannot be failed +# by a state-branch problem it never needed to solve. State is pushed +# immediately after each successful cancel, with a single rebase+retry on push +# failure. # # Workflow-level concurrency guarantees only one watchdog instance ever runs; # newer schedules must not cancel an in-flight audit after it has cancelled a @@ -64,10 +104,27 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: + # Fail closed by construction: this action errors when either secret is + # empty, so a missing credential aborts the audit before it can report a + # verdict it had no way to reach. + - name: Mint the runner-inventory reader token + id: reader_token + timeout-minutes: 2 + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 + with: + app-id: ${{ secrets.BUILD_LOCK_READER_APP_ID }} + private-key: ${{ secrets.BUILD_LOCK_READER_APP_PRIVATE_KEY }} + # Organization-scoped (no `repositories:`) because the runner-group + # endpoints are organization endpoints, and narrowed to the single + # permission this audit needs so the minted token can do nothing else. + owner: ${{ github.repository_owner }} + permission-organization-self-hosted-runners: read + - name: Audit + cancel-and-redispatch shell: bash env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUNNER_INVENTORY_TOKEN: ${{ steps.reader_token.outputs.token }} REPO: ${{ github.repository }} OWNER: ${{ github.repository_owner }} SELF_RUN_ID: ${{ github.run_id }} @@ -86,22 +143,83 @@ jobs: printf '%s\n' "$1" | tee -a "${summary_file}" } - flush_summary_and_exit() { - local code="${1:-0}" - { - echo "## Watchdog summary" - cat "${summary_file}" - } >> "${GITHUB_STEP_SUMMARY}" - exit "${code}" - } - # Category buckets for the final step-summary table. healthy_runs_file="$(mktemp)" - : > "${healthy_runs_file}" stuck_runs_file="$(mktemp)" - : > "${stuck_runs_file}" excluded_runs_file="$(mktemp)" + starved_runs_file="$(mktemp)" + : > "${healthy_runs_file}" + : > "${stuck_runs_file}" : > "${excluded_runs_file}" + : > "${starved_runs_file}" + + summary_emitted=0 + deliberate_exit=0 + + emit_summary() { + if (( summary_emitted == 1 )); then + return 0 + fi + summary_emitted=1 + # `|| true`: this runs from an EXIT trap, so a failure here would + # replace the status the job is actually exiting with. + { + echo "## Watchdog summary" + cat "${summary_file}" + echo "" + echo "### Healthy queued (waiting on concurrency / matrix slot)" + if [[ -s "${healthy_runs_file}" ]]; then cat "${healthy_runs_file}"; else echo "_(none)_"; fi + echo "" + echo "### Stuck (auto-cancelled)" + if [[ -s "${stuck_runs_file}" ]]; then cat "${stuck_runs_file}"; else echo "_(none)_"; fi + echo "" + echo "### Starved (no online runner carries the required labels)" + if [[ -s "${starved_runs_file}" ]]; then cat "${starved_runs_file}"; else echo "_(none)_"; fi + echo "" + echo "### Stuck but excluded (operator action needed)" + if [[ -s "${excluded_runs_file}" ]]; then cat "${excluded_runs_file}"; else echo "_(none)_"; fi + } >> "${GITHUB_STEP_SUMMARY}" || true + } + + finish() { + deliberate_exit=1 + emit_summary + exit "${1:-0}" + } + + # Without this, ANY abort `set -e` catches outside the enumerated + # paths -- and this is 500 lines of bash running 288 times a day -- + # produces a red run with a completely empty step summary and no + # annotation, which is the worst signal an operator can be handed. + # The trap makes every exit emit the buckets, and labels the ones + # that did not come from `finish` so an unexpected abort is not + # mistaken for a considered verdict. + # Invoked indirectly by the EXIT trap below, so static analysis cannot + # see the call. Both codes are needed because the version numbering + # differs: 0.9 flags the body as unreachable (SC2317), while newer + # builds -- including the one inside the actionlint container CI runs + # -- flag the function as never invoked (SC2329). Note that a comment + # line may not BEGIN with the analyzer's own name, or it is parsed as + # a malformed directive and the whole script stops being checked. + # shellcheck disable=SC2317,SC2329 + on_exit() { + local code=$? + if (( code != 0 )) && (( deliberate_exit == 0 )); then + log_summary "ERROR: the audit aborted unexpectedly (exit ${code}); the queue was not fully evaluated." + echo "::error::stuck-job watchdog aborted unexpectedly (exit ${code}); see the job log." + fi + emit_summary + } + trap on_exit EXIT + + # The audit could not answer. Never exit 0 from here: a green run has + # to mean the queue was evaluated, not that evaluation was skipped. + fail_closed() { + log_summary "ERROR: ${1}" + log_summary "The audit could not evaluate the queue, so this run fails rather than reporting a verdict it did not reach." + echo "::error::stuck-job watchdog could not complete its audit: ${1}" + finish 1 + } log_summary "## Stuck-job watchdog audit ($(date -u +'%Y-%m-%dT%H:%M:%SZ'))" log_summary "Repo: ${REPO}" @@ -114,6 +232,12 @@ jobs: [[ -z "${wf}" ]] && continue # Normalize: strip any leading .github/workflows/ if present. base="${wf##*/}" + # An empty subscript is a hard bash error, and the strip above turns + # any operator entry ending in `/` into one -- which the very next + # comment invites by talking about directory prefixes. Guarding only + # the read site left the write site able to kill the audit outright, + # every five minutes, on a repo-variable typo. + [[ -z "${base}" ]] && continue EXCLUDED_BY_FILE["${base}"]=1 done excluded_list="" @@ -123,7 +247,376 @@ jobs: log_summary "Excluded workflows: ${excluded_list:-}" # ------------------------------------------------------------------ - # 1. Bootstrap or check out the watchdog-state orphan branch. + # 1. Enumerate queued runs older than MIN_QUEUE_AGE_SECONDS. + # ------------------------------------------------------------------ + now_epoch="$(date -u +%s)" + queued_runs_json="$(mktemp)" + # `gh api --paginate` over the runs endpoint returns one JSON object + # per page; `jq -s '[.[] | .workflow_runs[]?]'` flattens all pages + # into a single array. Equivalent to using `--slurp` on newer gh + # versions but works on all gh versions shipped with ubuntu-latest. + if ! gh api --paginate "repos/${REPO}/actions/runs?status=queued&per_page=100" \ + | jq -s '[.[] | (.workflow_runs // [])[]]' > "${queued_runs_json}"; then + fail_closed "failed to list queued runs for ${REPO}." + fi + + # `mapfile < <(jq ...)` throws the process substitution's exit status + # away, and `pipefail` does not reach inside one. A single unparseable + # `created_at` would abort jq, yield an empty list, and print + # "Queue is clean" over a genuinely stuck queue -- the exact fail-open + # this rewrite exists to remove. Capture, check, then split. + if ! queued_ids_raw="$(jq -r --argjson now "${now_epoch}" --argjson min "${MIN_QUEUE_AGE_SECONDS}" ' + .[] + | select(.created_at != null) + | (.created_at | fromdateiso8601) as $created + | select(($now - $created) >= $min) + | .id + ' < "${queued_runs_json}" | sort -u)"; then + fail_closed "could not parse the queued-run listing for ${REPO}; the queue cannot be evaluated." + fi + queued_ids=() + if [[ -n "${queued_ids_raw}" ]]; then + mapfile -t queued_ids <<< "${queued_ids_raw}" + fi + + log_summary "Queued runs older than ${MIN_QUEUE_AGE_SECONDS}s: ${#queued_ids[@]}" + if [[ ${#queued_ids[@]} -eq 0 ]]; then + log_summary "Queue is clean. No action." + finish 0 + fi + + # ------------------------------------------------------------------ + # 2. Read the organization runner inventory visible to this + # repository, through the reader App. Fail closed on any failure: + # an unreadable inventory cannot distinguish "nothing is stuck" + # from "everything is stuck and we cannot see it". + # ------------------------------------------------------------------ + gh_reader() { + GH_TOKEN="${RUNNER_INVENTORY_TOKEN}" gh "$@" + } + + repo_name="${REPO##*/}" + runner_groups_json="$(mktemp)" + if ! gh_reader api --paginate \ + "orgs/${OWNER}/actions/runner-groups?visible_to_repository=${repo_name}&per_page=100" \ + | jq -s '[.[] | (.runner_groups // [])[] | select(.id != null)]' > "${runner_groups_json}"; then + fail_closed "could not read the organization runner groups visible to ${REPO}." + fi + group_count="$(jq 'length' < "${runner_groups_json}")" + if (( group_count == 0 )); then + fail_closed "the organization reported no runner groups visible to ${REPO}; the inventory cannot be trusted." + fi + log_summary "Runner groups visible to ${REPO}: $(jq -r '[.[].name] | join(", ")' < "${runner_groups_json}")" + + # mapfile, not `while read < <(...)`: the loop body runs `gh`, and a + # child that consumes the loop's stdin silently eats the remaining + # group ids. + if ! runner_group_ids_raw="$(jq -r '.[].id' < "${runner_groups_json}")"; then + fail_closed "could not parse the organization runner-group inventory." + fi + runner_group_ids=() + if [[ -n "${runner_group_ids_raw}" ]]; then + mapfile -t runner_group_ids <<< "${runner_group_ids_raw}" + fi + runners_json="$(mktemp)" + : > "${runners_json}" + group_runners_json="$(mktemp)" + for group_id in "${runner_group_ids[@]}"; do + if ! gh_reader api --paginate \ + "orgs/${OWNER}/actions/runner-groups/${group_id}/runners?per_page=100" \ + | jq -s '[.[] | (.runners // [])[]]' > "${group_runners_json}"; then + fail_closed "could not read runners in organization runner group ${group_id}." + fi + cat "${group_runners_json}" >> "${runners_json}" + done + + # Labels are compared case-insensitively in both directions: a job + # declaring `windows` and a runner registered as `Windows` are the + # same requirement, and an exact match would silently report every + # such run as unmatched. + inventory_json="$(mktemp)" + jq -s ' + [ .[][] + | { id: .id, + name: .name, + online: (.status == "online"), + busy: (.busy == true), + labels: ([(.labels // [])[] | (.name // "") | ascii_downcase | select(. != "")]) } + ] + | unique_by(.id) + ' < "${runners_json}" > "${inventory_json}" + + registered_count="$(jq 'length' < "${inventory_json}")" + online_count="$(jq '[.[] | select(.online)] | length' < "${inventory_json}")" + idle_count="$(jq '[.[] | select(.online and (.busy | not))] | length' < "${inventory_json}")" + log_summary "Runner inventory (groups visible to ${REPO}): ${registered_count} registered, ${online_count} online, ${idle_count} idle." + + # ------------------------------------------------------------------ + # 3. Classify every queued run. Nothing is cancelled in this pass, so + # the state branch is not needed unless something lands in + # `stuck_candidates`. + # ------------------------------------------------------------------ + stuck_candidates="$(mktemp)" + : > "${stuck_candidates}" + + # Reads the per-run loop variables directly; called from both the + # `busy` branch (where the run can still proceed) and the not-idle + # branch (where it cannot). No-op when nothing starved. + report_starvation() { + [[ -z "${starved_kind}" ]] && return 0 + local detail + if [[ "${starved_kind}" == "offline" ]]; then + detail="every runner carrying [${starved_labels}] is registered but offline" + else + detail="no runner registered in a group visible to ${REPO} carries [${starved_labels}]" + fi + log_summary "run ${run_id} (${run_path_base}, event=${run_event}): starved -- ${detail}. Not dispatcher-stuck; no action." + echo "::warning::run ${run_id} (${run_path_base}) queued past ${MIN_QUEUE_AGE_SECONDS}s: ${detail}. Bring that runner online or fix the label set." + printf '* run %s (%s, event=%s) -- %s -- %s\n' "${run_id}" "${run_path_base}" "${run_event}" "${detail}" "${run_html_url}" >> "${starved_runs_file}" + } + + for run_id in "${queued_ids[@]}"; do + # Skip the watchdog's own run id (defense in depth - the watchdog + # workflow file is excluded by name above, but if someone renames + # the file or runs ad-hoc the id check still protects us). + if [[ "${run_id}" == "${SELF_RUN_ID}" ]]; then + log_summary "run ${run_id}: this is the watchdog's own run; skipping." + continue + fi + + # Pull the run metadata we need from the cached list (workflow + # file path, event, head_branch). + run_meta="$( + jq -c --argjson id "${run_id}" ' + .[] + | select(.id == $id) + | { + path: (.path // ""), + event: (.event // ""), + workflow_id: (.workflow_id // 0), + head_branch: (.head_branch // ""), + html_url: (.html_url // "") + } + ' < "${queued_runs_json}" | head -n 1 + )" + if [[ -z "${run_meta}" ]]; then + fail_closed "run ${run_id} appeared in the queued listing but carries no metadata; the queue cannot be evaluated." + fi + run_path="$(jq -r '.path' <<< "${run_meta}")" + run_event="$(jq -r '.event' <<< "${run_meta}")" + run_html_url="$(jq -r '.html_url' <<< "${run_meta}")" + run_path_base="${run_path##*/}" + + # An empty subscript is a hard bash error, and `path: (.path // "")` + # above already anticipates a null path. A run whose workflow file + # cannot be named cannot be checked against the exclusion list, and + # cancelling is the one irreversible thing this job does, so it is + # reported and left alone rather than risked. + if [[ -z "${run_path_base}" ]]; then + log_summary "run ${run_id} (event=${run_event}): no workflow path reported; cannot check the exclusion list, so taking no action." + echo "::warning::run ${run_id} reported no workflow path; the watchdog cannot classify it and will not cancel it." + printf '* run %s (unknown workflow, event=%s) -- unclassifiable, left alone -- %s\n' "${run_id}" "${run_event}" "${run_html_url}" >> "${excluded_runs_file}" + continue + fi + + # Workflow-file exclusion (must NOT cancel release.yml). + if [[ -n "${EXCLUDED_BY_FILE[${run_path_base}]+x}" ]]; then + log_summary "run ${run_id} (${run_path_base}, event=${run_event}): workflow is excluded; operator action needed if stuck." + printf '* run %s (%s, event=%s) -- %s\n' "${run_id}" "${run_path_base}" "${run_event}" "${run_html_url}" >> "${excluded_runs_file}" + continue + fi + + jobs_json="$(mktemp)" + if ! gh api --paginate "repos/${REPO}/actions/runs/${run_id}/jobs?per_page=100" \ + | jq -s '[.[] | (.jobs // [])[]]' > "${jobs_json}"; then + fail_closed "failed to list jobs for run ${run_id}; that run could not be evaluated." + fi + + in_progress_count="$(jq '[ .[] | select(.status == "in_progress") ] | length' < "${jobs_json}")" + queued_count="$(jq '[ .[] | select(.status == "queued") ] | length' < "${jobs_json}")" + + # Classification runs BEFORE the early exits, because starvation has + # to be known on every path that reports one. The `in_progress` + # exit used to `continue` before labels were evaluated at all, so + # the moment a busy runner picked up the healthy cell the run became + # in-progress and a co-resident starved job went silent again for + # the rest of the matrix -- reopening the gap the `busy` fix had + # just closed, in the state that lasts longest. (Cursor Bugbot.) + # + # The exits keep their original order and messages below; only the + # point at which `starved_kind` becomes known has moved. + # + # Resolve each queued job's label set against the inventory. Only + # `idle` is dispatcher-stuck; the other three verdicts are the + # states a naive "no matching idle runner" test conflates, and + # cancelling any of them would destroy work the fleet still + # intends to run: + # idle - an online, not-busy runner carries every label + # busy - a matching runner is online but working + # offline - a matching runner is registered but disconnected + # unregistered - no registered runner carries the label set + # `unregistered` splits again on whether the job asked for + # `self-hosted`. The inventory covers only self-hosted runners, so + # a GitHub-hosted job (`ubuntu-latest`) is unregistered by + # definition and says nothing about our fleet; reporting it as + # starved would fire a warning on every queued hosted run. + # An empty label set would vacuously satisfy `all`, matching every + # runner and manufacturing a dispatcher-stuck verdict, so a job + # whose labels GitHub did not report is dropped rather than + # trusted. + if ! queued_job_labels_raw="$(jq -c ' + [ .[] | select(.status == "queued") | [ (.labels // [])[] | ascii_downcase ] ] + | map(select(length > 0)) + | unique + | .[] + ' < "${jobs_json}")"; then + fail_closed "could not parse the job labels for run ${run_id}; that run could not be evaluated." + fi + queued_job_labels=() + if [[ -n "${queued_job_labels_raw}" ]]; then + mapfile -t queued_job_labels <<< "${queued_job_labels_raw}" + fi + # Deferred, not `continue`d: a run can have BOTH an in-progress job + # and unlabelled queued jobs, and the in-progress verdict is the one + # that describes it. Recording the condition here keeps the exit + # order below identical to what it was before classification moved. + labels_not_usable=0 + if (( queued_count == 0 )) || [[ ${#queued_job_labels[@]} -eq 0 ]]; then + labels_not_usable=1 + fi + + # `match_kind` is the best verdict across ALL of the run's label + # sets and decides the action. `starved_*` is tracked SEPARATELY and + # only from self-hosted sets, because those are the only ones this + # inventory can speak to. Deriving the report from "the first set" + # while deciding from "all sets" made both wrong: a GitHub-hosted + # set sorting first suppressed a real starvation warning entirely, + # and when it did warn it named the hosted label rather than the + # self-hosted one, pointing the operator at a machine that does not + # exist. + match_kind="unregistered" + starved_kind="" + starved_labels="" + for labels_csv in "${queued_job_labels[@]+"${queued_job_labels[@]}"}"; do + kind="$(jq -r --argjson labels "${labels_csv}" ' + def satisfies($r): $labels | all(. as $l | $r.labels | index($l) | type == "number"); + if any(.[]; .online and (.busy | not) and satisfies(.)) then "idle" + elif any(.[]; .online and satisfies(.)) then "busy" + elif any(.[]; satisfies(.)) then "offline" + else "unregistered" end + ' < "${inventory_json}")" + # Precedence idle > busy > offline > unregistered. The loop does + # NOT break on `idle`: breaking meant the remaining label sets were + # never scanned, so a run with one dispatchable cell and one + # starved cell was cancelled with no starvation warning at all -- + # and because the stuck sibling keeps matching idle, that starved + # cell could stay invisible across repeated cancel cycles. Scanning + # every set costs one jq call per set and is what lets the verdict + # and the starvation report be independent. (Cursor Bugbot.) + if [[ "${kind}" == "idle" ]]; then + match_kind="idle" + elif [[ "${kind}" == "busy" && "${match_kind}" != "idle" ]]; then + match_kind="busy" + elif [[ "${kind}" == "offline" && "${match_kind}" == "unregistered" ]]; then + match_kind="offline" + fi + # Only a self-hosted set can starve on OUR fleet. Prefer reporting + # an `unregistered` set over an `offline` one: a label set nothing + # carries needs a human, while an offline machine may come back. + if jq -e 'index("self-hosted") | type == "number"' <<< "${labels_csv}" > /dev/null; then + if [[ "${kind}" == "offline" || "${kind}" == "unregistered" ]]; then + if [[ -z "${starved_kind}" ]] \ + || { [[ "${starved_kind}" == "offline" ]] && [[ "${kind}" == "unregistered" ]]; }; then + starved_kind="${kind}" + starved_labels="$(jq -r 'join(", ")' <<< "${labels_csv}")" + fi + fi + fi + done + + if (( in_progress_count > 0 )); then + # A run with even one in-progress job is by definition not + # dispatcher-stuck. This covers matrix cells or later jobs + # waiting while another job from the same run is active, and the + # general case of a run that has at least one runner. It is still + # not a reason to hide a sibling nothing can pick up. + log_summary "run ${run_id} (${run_path_base}, event=${run_event}): healthy queued (${in_progress_count} in_progress, ${queued_count} queued -- waiting on concurrency/matrix slot)." + printf '* run %s (%s, event=%s) -- %d in_progress, %d queued\n' "${run_id}" "${run_path_base}" "${run_event}" "${in_progress_count}" "${queued_count}" >> "${healthy_runs_file}" + report_starvation + continue + fi + + if (( queued_count == 0 )); then + # No queued jobs at all - the run is in some other transitional + # state, not the dispatcher-stuck pattern. + log_summary "run ${run_id} (${run_path_base}, event=${run_event}): no queued jobs yet (early state); skipping." + continue + fi + + if (( labels_not_usable == 1 )); then + log_summary "run ${run_id} (${run_path_base}, event=${run_event}): queued jobs report no labels; not evaluable. No action." + continue + fi + + # A busy sibling does not make a starved sibling less starved. The + # `busy` verdict used to `continue` without consulting `starved_kind`, + # so a matrix run with one cell queued behind a busy runner and + # another needing an offline or unregistered one reported only + # "healthy backpressure" -- no warning, no Starved row -- until every + # busy leg finished, which on Unity is hours. That is the #328 + # blindness in miniature: a starved job invisible because a sibling + # looks fine. (Cursor Bugbot.) + if [[ "${match_kind}" == "busy" ]]; then + log_summary "run ${run_id} (${run_path_base}, event=${run_event}): every matching runner is online but busy; healthy backpressure. No action." + printf '* run %s (%s, event=%s) -- waiting on a busy matching runner\n' "${run_id}" "${run_path_base}" "${run_event}" >> "${healthy_runs_file}" + report_starvation + continue + fi + + # Reaching here implies no queued job asked for `self-hosted`: + # a self-hosted set that is idle breaks out above, one that is busy + # returns above, and one that is offline or unregistered sets + # `starved_kind`. So the only surviving shape is a run this + # inventory genuinely cannot speak to. + # + # The old wording claimed such a run "targets GitHub-hosted + # capacity", which is not the same statement and can be false: a + # registered runner may carry `ubuntu-latest` and simply be offline. + # What is always true is that no queued job ASKED for self-hosted. + if [[ "${match_kind}" != "idle" && -z "${starved_kind}" ]]; then + log_summary "run ${run_id} (${run_path_base}): no queued job requests a self-hosted runner; not evaluable. No action." + continue + fi + + if [[ "${match_kind}" != "idle" ]]; then + report_starvation + continue + fi + + # Log the verdict HERE, not only when the cancel succeeds. Classification + # and action are separated by the state-branch materialization, which can + # fail closed -- and then the summary said "a stuck run is pending" while + # the Stuck section sat empty, so an operator could not tell WHICH run + # needed attention. Every other verdict already announces itself at the + # point it is reached; this one did not. (Cursor Bugbot.) + # A cancellable sibling does not make a starved one less starved, + # and this is the branch that acts, so it is the one most likely to + # be read. + report_starvation + log_summary "run ${run_id} (${run_path_base}, event=${run_event}): dispatcher-stuck; queued for cancel -- ${run_html_url}" + printf '%s\n' "${run_meta}" | jq -c --argjson id "${run_id}" '. + {id: $id}' >> "${stuck_candidates}" + done + + if [[ ! -s "${stuck_candidates}" ]]; then + log_summary "No dispatcher-stuck run found. No cancel issued." + finish 0 + fi + + # ------------------------------------------------------------------ + # 4. Materialize the cancel-cap state branch. Only reached when a + # cancel is actually about to be issued, so a state-branch problem + # can no longer fail a cycle that had nothing to do -- and when it + # IS needed, an unreadable cap means we must not cancel blind. # ------------------------------------------------------------------ work_dir="$(mktemp -d)" pushd "${work_dir}" > /dev/null @@ -155,16 +648,12 @@ jobs: # rewriting it as a fresh orphan. state_branch_probe="$(mktemp)" if ! git_auth ls-remote --heads "${remote_url}" "${STATE_BRANCH}" > "${state_branch_probe}" 2>/dev/null; then - log_summary "WARN: 'git ls-remote --heads ${remote_url} ${STATE_BRANCH}' failed (transient?); refusing to bootstrap. Skipping this cycle." - popd > /dev/null - flush_summary_and_exit 0 + fail_closed "'git ls-remote --heads ${STATE_BRANCH}' failed; the cancel cap is unreadable and a stuck run is pending." fi if [[ -s "${state_branch_probe}" ]]; then if ! git_auth clone --depth 1 --single-branch --branch "${STATE_BRANCH}" "${remote_url}" repo; then - log_summary "WARN: state branch '${STATE_BRANCH}' exists per ls-remote but clone failed; skipping this cycle." - popd > /dev/null - flush_summary_and_exit 0 + fail_closed "state branch '${STATE_BRANCH}' exists per ls-remote but the clone failed; the cancel cap is unreadable." fi cd repo log_summary "State branch '${STATE_BRANCH}' checked out." @@ -180,9 +669,7 @@ jobs: git add "${STATE_DIR}/.gitkeep" git "${GIT_AUTHOR_ID[@]}" commit -m "Initialize watchdog state" || true if ! git_auth push origin "${STATE_BRANCH}"; then - log_summary "WARN: bootstrap push failed; will retry on next run." - popd > /dev/null - flush_summary_and_exit 0 + fail_closed "bootstrap push of '${STATE_BRANCH}' failed; the cancel cap cannot be persisted." fi fi mkdir -p "${STATE_DIR}" @@ -232,169 +719,21 @@ jobs: } # ------------------------------------------------------------------ - # 2. Enumerate queued runs older than MIN_QUEUE_AGE_SECONDS. + # 5. Cancel each dispatcher-stuck run, capped at MAX_CANCELS_PER_DAY + # per run-id, then re-dispatch where a safe path exists. # ------------------------------------------------------------------ - now_epoch="$(date -u +%s)" - queued_runs_json="$(mktemp)" - # `gh api --paginate` over the runs endpoint returns one JSON object - # per page; `jq -s '[.[] | .workflow_runs[]?]'` flattens all pages - # into a single array. Equivalent to using `--slurp` on newer gh - # versions but works on all gh versions shipped with ubuntu-latest. - if ! gh api --paginate "repos/${REPO}/actions/runs?status=queued&per_page=100" \ - | jq -s '[.[] | (.workflow_runs // [])[]]' > "${queued_runs_json}"; then - log_summary "ERROR: failed to list queued runs." - popd > /dev/null - flush_summary_and_exit 0 - fi - - mapfile -t queued_ids < <(jq -r --argjson now "${now_epoch}" --argjson min "${MIN_QUEUE_AGE_SECONDS}" ' - .[] - | select(.created_at != null) - | (.created_at | fromdateiso8601) as $created - | select(($now - $created) >= $min) - | .id - ' < "${queued_runs_json}" | sort -u) - - log_summary "Queued runs older than ${MIN_QUEUE_AGE_SECONDS}s: ${#queued_ids[@]}" - if [[ ${#queued_ids[@]} -eq 0 ]]; then - log_summary "Queue is clean. No action." - popd > /dev/null - flush_summary_and_exit 0 - fi - - # ------------------------------------------------------------------ - # 3. Fetch idle runners (org first, fall back to repo on 403). - # `gh api --paginate` over an object-shaped endpoint returns one - # object per page; without `jq -s`, downstream evaluation runs - # against only the last page (silent false negatives once the - # org grows past 100 runners - see cli/cli#1268). - # ------------------------------------------------------------------ - runners_json="$(mktemp)" - runners_scope="org" - if ! gh api --paginate "orgs/${OWNER}/actions/runners?per_page=100" \ - | jq -s '[.[] | (.runners // [])[]]' > "${runners_json}" 2>/dev/null; then - runners_scope="repo" - log_summary "WARN: org-level runner list unavailable (likely 403). Falling back to repo runners." - if ! gh api --paginate "repos/${REPO}/actions/runners?per_page=100" \ - | jq -s '[.[] | (.runners // [])[]]' > "${runners_json}"; then - log_summary "ERROR: repo-level runner list also failed; cannot evaluate idle runners. No action issued." - popd > /dev/null - flush_summary_and_exit 0 - fi - fi - log_summary "Runner inventory scope: ${runners_scope}" - - idle_runners_json="$(mktemp)" - jq ' - [ .[] - | select(.status == "online") - | select(.busy == false) - | { id: .id, name: .name, labels: ([(.labels // [])[].name]) } ] - ' < "${runners_json}" > "${idle_runners_json}" - idle_count="$(jq 'length' < "${idle_runners_json}")" - log_summary "Idle runners (online + not busy): ${idle_count}" - - # ------------------------------------------------------------------ - # 4. For each queued run, decide stuck vs healthy vs excluded. - # ------------------------------------------------------------------ - state_dirty=0 - for run_id in "${queued_ids[@]}"; do - # Skip the watchdog's own run id (defense in depth - the watchdog - # workflow file is excluded by name above, but if someone renames - # the file or runs ad-hoc the id check still protects us). - if [[ "${run_id}" == "${SELF_RUN_ID}" ]]; then - log_summary "run ${run_id}: this is the watchdog's own run; skipping." - continue - fi - - # Pull the run metadata we need from the cached list (workflow - # file path, event, head_branch). - run_meta="$( - jq -c --argjson id "${run_id}" ' - .[] - | select(.id == $id) - | { - path: (.path // ""), - event: (.event // ""), - workflow_id: (.workflow_id // 0), - head_branch: (.head_branch // ""), - html_url: (.html_url // "") - } - ' < "${queued_runs_json}" | head -n 1 - )" - if [[ -z "${run_meta}" ]]; then - log_summary "run ${run_id}: metadata unavailable; skipping." - continue - fi - run_path="$(jq -r '.path' <<< "${run_meta}")" - run_event="$(jq -r '.event' <<< "${run_meta}")" - run_workflow_id="$(jq -r '.workflow_id' <<< "${run_meta}")" - run_head_branch="$(jq -r '.head_branch' <<< "${run_meta}")" - run_html_url="$(jq -r '.html_url' <<< "${run_meta}")" - run_path_base="${run_path##*/}" - - # Workflow-file exclusion (must NOT cancel release.yml). - if [[ -n "${EXCLUDED_BY_FILE[${run_path_base}]+x}" ]]; then - log_summary "run ${run_id} (${run_path_base}, event=${run_event}): workflow is excluded; operator action needed if stuck." - printf '* run %s (%s, event=%s) -- %s\n' "${run_id}" "${run_path_base}" "${run_event}" "${run_html_url}" >> "${excluded_runs_file}" - continue - fi - - jobs_json="$(mktemp)" - if ! gh api --paginate "repos/${REPO}/actions/runs/${run_id}/jobs?per_page=100" \ - | jq -s '[.[] | (.jobs // [])[]]' > "${jobs_json}" 2>/dev/null; then - log_summary "run ${run_id}: failed to list jobs; skipping." - continue - fi + # mapfile, not `while read < file`: `gh run cancel` inherits the + # loop's stdin and would swallow the remaining candidates. + mapfile -t stuck_candidate_lines < "${stuck_candidates}" + for candidate in "${stuck_candidate_lines[@]}"; do + run_id="$(jq -r '.id' <<< "${candidate}")" + run_event="$(jq -r '.event' <<< "${candidate}")" + run_workflow_id="$(jq -r '.workflow_id' <<< "${candidate}")" + run_head_branch="$(jq -r '.head_branch' <<< "${candidate}")" + run_html_url="$(jq -r '.html_url' <<< "${candidate}")" + run_path_base="$(jq -r '.path' <<< "${candidate}")" + run_path_base="${run_path_base##*/}" - in_progress_count="$(jq '[ .[] | select(.status == "in_progress") ] | length' < "${jobs_json}")" - queued_count="$(jq '[ .[] | select(.status == "queued") ] | length' < "${jobs_json}")" - - if (( in_progress_count > 0 )); then - # A run with even one in-progress job is by definition not - # dispatcher-stuck. This covers matrix cells or later jobs - # waiting while another job from the same run is active, and the - # general case of a run that has at least one runner. - log_summary "run ${run_id} (${run_path_base}, event=${run_event}): healthy queued (${in_progress_count} in_progress, ${queued_count} queued -- waiting on concurrency/matrix slot)." - printf '* run %s (%s, event=%s) -- %d in_progress, %d queued\n' "${run_id}" "${run_path_base}" "${run_event}" "${in_progress_count}" "${queued_count}" >> "${healthy_runs_file}" - continue - fi - - if (( queued_count == 0 )); then - # No queued jobs at all - the run is in some other transitional - # state, not the dispatcher-stuck pattern. - log_summary "run ${run_id} (${run_path_base}, event=${run_event}): no queued jobs yet (early state); skipping." - continue - fi - - mapfile -t queued_job_labels < <(jq -c ' - .[] - | select(.status == "queued") - | (.labels // []) - ' < "${jobs_json}") - - matched=0 - for labels_csv in "${queued_job_labels[@]}"; do - # labels_csv is a JSON array like ["self-hosted","Windows","RAM-64GB"] - if jq -e --argjson labels "${labels_csv}" ' - map( - . as $r - | ($labels | all(. as $l | $r.labels | index($l) | type == "number")) - ) | any - ' < "${idle_runners_json}" > /dev/null; then - matched=1 - break - fi - done - - if [[ ${matched} -eq 0 ]]; then - log_summary "run ${run_id} (${run_path_base}, event=${run_event}): no matching idle runner -- investigate label config. No action." - continue - fi - - # ------------------------------------------------------------------ - # 5. Genuinely stuck. Cap at MAX_CANCELS_PER_DAY per run-id. - # ------------------------------------------------------------------ state_file="${STATE_DIR}/${run_id}.json" cancels=0 last_cancel=0 @@ -501,31 +840,4 @@ jobs: popd > /dev/null - # ------------------------------------------------------------------ - # Final categorized step-summary table. - # ------------------------------------------------------------------ - { - echo "## Watchdog summary" - cat "${summary_file}" - echo "" - echo "### Healthy queued (waiting on concurrency / matrix slot)" - if [[ -s "${healthy_runs_file}" ]]; then - cat "${healthy_runs_file}" - else - echo "_(none)_" - fi - echo "" - echo "### Stuck (auto-cancelled)" - if [[ -s "${stuck_runs_file}" ]]; then - cat "${stuck_runs_file}" - else - echo "_(none)_" - fi - echo "" - echo "### Stuck but excluded (operator action needed)" - if [[ -s "${excluded_runs_file}" ]]; then - cat "${excluded_runs_file}" - else - echo "_(none)_" - fi - } >> "${GITHUB_STEP_SUMMARY}" + finish 0 diff --git a/.github/workflows/unity-benchmarks.yml b/.github/workflows/unity-benchmarks.yml index 886defd6..2e5f31d7 100644 --- a/.github/workflows/unity-benchmarks.yml +++ b/.github/workflows/unity-benchmarks.yml @@ -361,7 +361,19 @@ jobs: timeout-minutes: 2 uses: Ambiguous-Interactive/ambiguous-organization-build-lock/.github/actions/require-confirmed-unity-cleanup@ce12fde93e67a27062350601fb97e4a2f546d405 with: - acquired: ${{ steps.acquire_lock.outputs.acquired }} + # A leg that aborts before `Acquire organization Unity lock` runs never + # took a seat, and this gate's own contract treats `acquired: false` as + # proof that licensed cleanup was not required. Feeding it the skipped + # step's EMPTY output instead made it fail closed on work that never + # started, so a leg with one honest failure reported two (#327). + # + # This does not weaken the gate. It reads `outcome`, not `acquired`, and + # only the two values that prove the step never executed map to 'false'. + # An acquire that RAN and failed part-way reports outcome=failure while + # `acquired` reads empty, so the empty value still flows through and the + # gate still fails -- that ambiguous shape is the seat leak the bare + # `if: always()` above exists to catch, and it stays caught. + acquired: ${{ (steps.acquire_lock.outcome == 'skipped' || steps.acquire_lock.outcome == '') && 'false' || steps.acquire_lock.outputs.acquired }} classification-complete: ${{ steps.cleanup_classification.outputs.classification-complete }} cleanup-status: ${{ steps.cleanup_classification.outputs.resource-cleanup-status }} cleanup-health: ${{ steps.cleanup_classification.outputs.resource-health }} diff --git a/.github/workflows/unity-tests.yml b/.github/workflows/unity-tests.yml index 43055b46..1883832e 100644 --- a/.github/workflows/unity-tests.yml +++ b/.github/workflows/unity-tests.yml @@ -467,7 +467,19 @@ jobs: timeout-minutes: 2 uses: Ambiguous-Interactive/ambiguous-organization-build-lock/.github/actions/require-confirmed-unity-cleanup@ce12fde93e67a27062350601fb97e4a2f546d405 with: - acquired: ${{ steps.acquire_lock.outputs.acquired }} + # A leg that aborts before `Acquire organization Unity lock` runs never + # took a seat, and this gate's own contract treats `acquired: false` as + # proof that licensed cleanup was not required. Feeding it the skipped + # step's EMPTY output instead made it fail closed on work that never + # started, so a leg with one honest failure reported two (#327). + # + # This does not weaken the gate. It reads `outcome`, not `acquired`, and + # only the two values that prove the step never executed map to 'false'. + # An acquire that RAN and failed part-way reports outcome=failure while + # `acquired` reads empty, so the empty value still flows through and the + # gate still fails -- that ambiguous shape is the seat leak the bare + # `if: always()` above exists to catch, and it stays caught. + acquired: ${{ (steps.acquire_lock.outcome == 'skipped' || steps.acquire_lock.outcome == '') && 'false' || steps.acquire_lock.outputs.acquired }} classification-complete: ${{ steps.cleanup_classification.outputs.classification-complete }} cleanup-status: ${{ steps.cleanup_classification.outputs.resource-cleanup-status }} cleanup-health: ${{ steps.cleanup_classification.outputs.resource-health }} diff --git a/.github/workflows/update-issue-template-versions.yml b/.github/workflows/update-issue-template-versions.yml deleted file mode 100644 index e20e9761..00000000 --- a/.github/workflows/update-issue-template-versions.yml +++ /dev/null @@ -1,183 +0,0 @@ ---- -name: Update Issue Template Versions - -# Regenerates the bug-report issue template's package-version dropdown and, on the -# default branch, commits it back. This is the self-healing backstop for the -# `check:issue-template-versions` gate: whenever package.json / CHANGELOG.md change -# on master (a release lands, a version is added) the dropdown is brought current -# even if a contributor bumped the version without running the generator. The -# release PR (release-prepare.yml) already regenerates it proactively; this keeps -# it correct for every other path. -# -# Mirrors update-llms-txt.yml: the built-in GITHUB_TOKEN cannot push to a protected -# branch, so the push is authenticated by a GitHub App installation token -# (secrets.AUTO_COMMIT_APP_ID + secrets.AUTO_COMMIT_APP_PRIVATE_KEY) whose App is -# allowed to bypass branch protection. If those secrets are absent the commit -# degrades to a logged no-op (regeneration + validation still run). See -# docs/runbooks/perf-numbers-auto-commit.md. -on: - schedule: - # Weekly on Monday at 10:00 AM UTC (offset from update-llms-txt's 09:00 slot). - - cron: "0 10 * * 1" - workflow_dispatch: - push: - branches: - - main - - master - paths: - - package.json - - CHANGELOG.md - - .github/ISSUE_TEMPLATE/bug_report.yml - - scripts/generate-issue-template-versions.js - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -# The push is authenticated by the GitHub App installation token, not the job's -# GITHUB_TOKEN, so contents:read is all this job's own token needs (checkout). -permissions: - contents: read - -jobs: - update: - name: Update Issue Template Versions - runs-on: ubuntu-latest - timeout-minutes: 5 - - steps: - - name: Check for the auto-commit GitHub App credentials - id: check-autocommit-app - # Degrade to a logged no-op when the App is not provisioned, instead of a - # red build. See docs/runbooks/perf-numbers-auto-commit.md. - env: - AUTO_COMMIT_APP_ID: ${{ secrets.AUTO_COMMIT_APP_ID }} - AUTO_COMMIT_APP_PRIVATE_KEY: ${{ secrets.AUTO_COMMIT_APP_PRIVATE_KEY }} - run: | - set -euo pipefail - if [ -n "${AUTO_COMMIT_APP_ID:-}" ] && \ - [ -n "${AUTO_COMMIT_APP_PRIVATE_KEY:-}" ]; then - echo "has-app=true" >> "${GITHUB_OUTPUT}" - else - echo "::warning::AUTO_COMMIT_APP_* secrets are not set; skipping the issue-template-versions auto-commit. See docs/runbooks/perf-numbers-auto-commit.md." - echo "has-app=false" >> "${GITHUB_OUTPUT}" - fi - - - name: Generate auto-commit GitHub App token - id: app-token - if: steps.check-autocommit-app.outputs.has-app == 'true' - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 - with: - app-id: ${{ secrets.AUTO_COMMIT_APP_ID }} - private-key: ${{ secrets.AUTO_COMMIT_APP_PRIVATE_KEY }} - - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - token: ${{ steps.app-token.outputs.token || github.token }} - persist-credentials: false - - - name: Refresh branch before generating - if: steps.check-autocommit-app.outputs.has-app == 'true' - env: - GH_FETCH_TOKEN: ${{ steps.app-token.outputs.token }} - run: | - set -euo pipefail - auth_header="$(printf 'x-access-token:%s' "${GH_FETCH_TOKEN}" | base64 | tr -d '\n')" - git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic ${auth_header}" \ - fetch --no-tags --depth=1 origin \ - "+refs/heads/${GITHUB_REF_NAME}:refs/remotes/origin/${GITHUB_REF_NAME}" - git checkout -B "${GITHUB_REF_NAME}" "refs/remotes/origin/${GITHUB_REF_NAME}" - echo "Refreshed ${GITHUB_REF_NAME} to $(git rev-parse HEAD) before regenerating the dropdown." - - - name: Setup Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: "22.18.0" - cache: npm - cache-dependency-path: package.json - - - name: Install dependencies - run: | - if [ -f package-lock.json ]; then - npm ci - else - npm i --no-audit --no-fund - fi - - - name: Update issue-template versions - run: npm run update:issue-template-versions - - - name: Validate issue-template versions generator contract - run: npm run check:issue-template-versions - - - name: Check for changes - id: git-check - run: | - git diff --exit-code -- .github/ISSUE_TEMPLATE/bug_report.yml || echo "changed=true" >> "$GITHUB_OUTPUT" - - - name: Commit and push changes - if: steps.git-check.outputs.changed == 'true' && steps.check-autocommit-app.outputs.has-app == 'true' - env: - GH_PUSH_TOKEN: ${{ steps.app-token.outputs.token }} - run: | - set -euo pipefail - auth_header="$(printf 'x-access-token:%s' "${GH_PUSH_TOKEN}" | base64 | tr -d '\n')" - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - target_file=".github/ISSUE_TEMPLATE/bug_report.yml" - for attempt in 1 2 3; do - git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic ${auth_header}" \ - fetch --no-tags --depth=1 origin \ - "+refs/heads/${GITHUB_REF_NAME}:refs/remotes/origin/${GITHUB_REF_NAME}" - latest_sha="$(git rev-parse "refs/remotes/origin/${GITHUB_REF_NAME}")" - current_sha="$(git rev-parse HEAD)" - if [ "${latest_sha}" != "${current_sha}" ]; then - echo "::warning::Default branch ${GITHUB_REF_NAME} advanced from ${current_sha} to ${latest_sha}; regenerating on the latest head before pushing." - git checkout -f -B "${GITHUB_REF_NAME}" "refs/remotes/origin/${GITHUB_REF_NAME}" - npm run update:issue-template-versions - npm run check:issue-template-versions - fi - - if git diff --quiet -- "${target_file}"; then - echo "The version dropdown is already current; nothing to commit." - exit 0 - fi - - base_sha="$(git rev-parse HEAD)" - git add -- "${target_file}" - if git diff --cached --quiet -- "${target_file}"; then - echo "No staged dropdown changes remain; nothing to commit." - exit 0 - fi - git commit -m "chore: update issue-template package versions" - - set +e - git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic ${auth_header}" \ - push origin "HEAD:${GITHUB_REF_NAME}" - push_status=$? - set -e - if [ "${push_status}" -eq 0 ]; then - exit 0 - fi - - git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic ${auth_header}" \ - fetch --no-tags --depth=1 origin \ - "+refs/heads/${GITHUB_REF_NAME}:refs/remotes/origin/${GITHUB_REF_NAME}" - latest_after_push="$(git rev-parse "refs/remotes/origin/${GITHUB_REF_NAME}")" - if [ "${latest_after_push}" = "${base_sha}" ]; then - echo "::error::issue-template-versions auto-commit push failed while ${GITHUB_REF_NAME} still pointed at ${base_sha}." - exit "${push_status}" - fi - - if [ "${attempt}" -eq 3 ]; then - echo "::warning::Default branch ${GITHUB_REF_NAME} kept advancing during the auto-commit; skipping this stale attempt. A newer run will regenerate from the latest head." - exit 0 - fi - - echo "::warning::Default branch ${GITHUB_REF_NAME} advanced during the push; retrying from ${latest_after_push}." - git checkout -f -B "${GITHUB_REF_NAME}" "refs/remotes/origin/${GITHUB_REF_NAME}" - npm run update:issue-template-versions - npm run check:issue-template-versions - done diff --git a/.github/workflows/update-llms-txt.yml b/.github/workflows/update-llms-txt.yml deleted file mode 100644 index e85e61b1..00000000 --- a/.github/workflows/update-llms-txt.yml +++ /dev/null @@ -1,184 +0,0 @@ ---- -name: Update llms.txt - -# Regenerates llms.txt and, on the default branch, commits it back. The built-in -# GITHUB_TOKEN cannot push to a protected branch (github-actions[bot] is a system -# account, not a selectable branch-protection bypass actor), so the push is -# authenticated by a GitHub App installation token (secrets.AUTO_COMMIT_APP_ID + -# secrets.AUTO_COMMIT_APP_PRIVATE_KEY) whose App is allowed to bypass branch -# protection. If those secrets are absent the commit degrades to a logged no-op (the -# regeneration + validation still run). See docs/runbooks/perf-numbers-auto-commit.md. -on: - schedule: - # Run weekly on Monday at 9:00 AM UTC - - cron: "0 9 * * 1" - workflow_dispatch: - push: - branches: - - main - - master - paths: - - package.json - - .llm/skills/** - - scripts/update-llms-txt.js - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -# The push is authenticated by the GitHub App installation token, not the job's -# GITHUB_TOKEN, so contents:read is all this job's own token needs (checkout). -permissions: - contents: read - -jobs: - update: - name: Update llms.txt - runs-on: ubuntu-latest - timeout-minutes: 5 - - steps: - - name: Check for the auto-commit GitHub App credentials - id: check-autocommit-app - # Degrade to a logged no-op when the App is not provisioned, instead of a - # red build. See docs/runbooks/perf-numbers-auto-commit.md. - env: - AUTO_COMMIT_APP_ID: ${{ secrets.AUTO_COMMIT_APP_ID }} - AUTO_COMMIT_APP_PRIVATE_KEY: ${{ secrets.AUTO_COMMIT_APP_PRIVATE_KEY }} - run: | - set -euo pipefail - if [ -n "${AUTO_COMMIT_APP_ID:-}" ] && \ - [ -n "${AUTO_COMMIT_APP_PRIVATE_KEY:-}" ]; then - echo "has-app=true" >> "${GITHUB_OUTPUT}" - else - echo "::warning::AUTO_COMMIT_APP_* secrets are not set; skipping the llms.txt auto-commit. See docs/runbooks/perf-numbers-auto-commit.md." - echo "has-app=false" >> "${GITHUB_OUTPUT}" - fi - - - name: Generate auto-commit GitHub App token - id: app-token - if: steps.check-autocommit-app.outputs.has-app == 'true' - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 - with: - app-id: ${{ secrets.AUTO_COMMIT_APP_ID }} - private-key: ${{ secrets.AUTO_COMMIT_APP_PRIVATE_KEY }} - - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - token: ${{ steps.app-token.outputs.token || github.token }} - persist-credentials: false - - - name: Refresh branch before generating - if: steps.check-autocommit-app.outputs.has-app == 'true' - env: - GH_FETCH_TOKEN: ${{ steps.app-token.outputs.token }} - run: | - set -euo pipefail - auth_header="$(printf 'x-access-token:%s' "${GH_FETCH_TOKEN}" | base64 | tr -d '\n')" - git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic ${auth_header}" \ - fetch --no-tags --depth=1 origin \ - "+refs/heads/${GITHUB_REF_NAME}:refs/remotes/origin/${GITHUB_REF_NAME}" - git checkout -B "${GITHUB_REF_NAME}" "refs/remotes/origin/${GITHUB_REF_NAME}" - echo "Refreshed ${GITHUB_REF_NAME} to $(git rev-parse HEAD) before generating llms.txt." - - - name: Setup Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: "22.18.0" - cache: npm - cache-dependency-path: package.json - - - name: Install dependencies - run: | - if [ -f package-lock.json ]; then - npm ci - else - npm i --no-audit --no-fund - fi - - - name: Update llms.txt - run: npm run update:llms-txt - - - name: Validate llms.txt generator contract - run: npm run check:llms-txt - - - name: Check for changes - id: git-check - run: | - # update:llms-txt regenerates llms.txt and syncs the skill-count claim in - # README.md; both are committed so neither can drift on the default branch. - git diff --exit-code -- llms.txt README.md || echo "changed=true" >> "$GITHUB_OUTPUT" - - - name: Commit and push changes - if: steps.git-check.outputs.changed == 'true' && steps.check-autocommit-app.outputs.has-app == 'true' - env: - GH_PUSH_TOKEN: ${{ steps.app-token.outputs.token }} - run: | - set -euo pipefail - auth_header="$(printf 'x-access-token:%s' "${GH_PUSH_TOKEN}" | base64 | tr -d '\n')" - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - for attempt in 1 2 3; do - git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic ${auth_header}" \ - fetch --no-tags --depth=1 origin \ - "+refs/heads/${GITHUB_REF_NAME}:refs/remotes/origin/${GITHUB_REF_NAME}" - latest_sha="$(git rev-parse "refs/remotes/origin/${GITHUB_REF_NAME}")" - current_sha="$(git rev-parse HEAD)" - if [ "${latest_sha}" != "${current_sha}" ]; then - message="Default branch ${GITHUB_REF_NAME} advanced from ${current_sha} " - message+="to ${latest_sha}; regenerating llms.txt on the latest head " - message+="before pushing." - echo "::warning::${message}" - git checkout -f -B "${GITHUB_REF_NAME}" "refs/remotes/origin/${GITHUB_REF_NAME}" - npm run update:llms-txt - npm run check:llms-txt - fi - - if git diff --quiet -- llms.txt README.md; then - echo "llms.txt and README are already current; nothing to commit." - exit 0 - fi - - base_sha="$(git rev-parse HEAD)" - git add -- llms.txt README.md - if git diff --cached --quiet -- llms.txt README.md; then - echo "No staged llms.txt/README changes remain; nothing to commit." - exit 0 - fi - git commit -m "chore: update llms.txt" - - set +e - git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic ${auth_header}" \ - push origin "HEAD:${GITHUB_REF_NAME}" - push_status=$? - set -e - if [ "${push_status}" -eq 0 ]; then - exit 0 - fi - - git -c "http.https://github.com/.extraheader=AUTHORIZATION: basic ${auth_header}" \ - fetch --no-tags --depth=1 origin \ - "+refs/heads/${GITHUB_REF_NAME}:refs/remotes/origin/${GITHUB_REF_NAME}" - latest_after_push="$(git rev-parse "refs/remotes/origin/${GITHUB_REF_NAME}")" - if [ "${latest_after_push}" = "${base_sha}" ]; then - echo "::error::llms.txt auto-commit push failed while ${GITHUB_REF_NAME} still pointed at ${base_sha}." - exit "${push_status}" - fi - - if [ "${attempt}" -eq 3 ]; then - message="Default branch ${GITHUB_REF_NAME} kept advancing during " - message+="the llms.txt auto-commit; skipping this stale attempt. " - message+="A newer run will regenerate from the latest head." - echo "::warning::${message}" - exit 0 - fi - - message="Default branch ${GITHUB_REF_NAME} advanced during the llms.txt " - message+="push; retrying from ${latest_after_push}." - echo "::warning::${message}" - git checkout -f -B "${GITHUB_REF_NAME}" "refs/remotes/origin/${GITHUB_REF_NAME}" - npm run update:llms-txt - npm run check:llms-txt - done diff --git a/.llm/context.md b/.llm/context.md index 38c7e089..3095d746 100644 --- a/.llm/context.md +++ b/.llm/context.md @@ -62,7 +62,7 @@ discover it. `.llm/` is the single source of truth; the mirrors are generated po - Sync banner SVG version: `npm run sync:banner` (check-only: `npm run check:banner`) - Regenerate llms.txt: `npm run update:llms-txt` (check-only: `npm run check:llms-txt`) - Regenerate the skill index, registry block, and agent mirrors: `npm run llm:index` (check-only: `npm run llm:check`, gated in `validate:all`) -- Regenerate the bug-report version dropdown (`.github/ISSUE_TEMPLATE/bug_report.yml`, derived from `package.json` + `CHANGELOG.md` + git tags, append-only so a shallow checkout never drops history): `npm run update:issue-template-versions` (check-only: `npm run check:issue-template-versions`, gated in `validate:all`). Self-healing in three layers: `release-prepare.yml` regenerates + stages it in its version-sync step (so the release PR passes the gate); `update-issue-template-versions.yml` auto-commits it on any `package.json`/`CHANGELOG.md` push to the default branch + a weekly cron (the same App-token pattern as `update-llms-txt.yml`); and the `--check` gate blocks a stale hand-edit on PRs. +- Regenerate the bug-report version dropdown (`.github/ISSUE_TEMPLATE/bug_report.yml`, derived from `package.json` + `CHANGELOG.md` + git tags, append-only so a shallow checkout never drops history): `npm run update:issue-template-versions` (check-only: `npm run check:issue-template-versions`, gated in `validate:all`). Self-healing in three layers: `release-prepare.yml` regenerates + stages it in its version-sync step (so the release PR passes the gate); `post-merge-maintenance.yml` auto-commits it -- together with `llms.txt` in the SAME commit -- on any push to the default branch that touches either generator's inputs, plus a weekly cron; and the `--check` gate blocks a stale hand-edit on PRs. - Analyzer payload: `npm run check:analyzers` (refresh: `npm run refresh:analyzers`) - C# method naming auto-fix: `npm run fix:csharp-underscores` - Unity asmdef reference integrity: `npm run validate:asmdef-references` @@ -164,6 +164,16 @@ Reach GitHub in this order, and stop at the first one that works: Never echo a token into command output, a log, or a file. Read it into a shell variable in the same command that uses it. +**Do not poll.** Every `gh` invocation re-reads and re-presents credentials, so a +watch loop that runs `gh` on a timer turns one credential use into hundreds and +shows up as repeated credential activity on the account. This is what a long CI +wait actually costs: a 60-second loop over a 40-minute Unity matrix is 40 +authenticated calls, and several such loops at once multiply it. Wait on a +single blocking call, or check once when a notification says something changed. +Reuse one token read into a shell variable across a batch of API calls rather +than shelling out to `gh` per item. If a wait genuinely needs repetition, space +it in minutes and run exactly one loop. + ## Line Ending Policy - Mixed policy is required. diff --git a/docs/runbooks/perf-numbers-auto-commit.md b/docs/runbooks/perf-numbers-auto-commit.md index 0babafec..b1a41f1a 100644 --- a/docs/runbooks/perf-numbers-auto-commit.md +++ b/docs/runbooks/perf-numbers-auto-commit.md @@ -130,8 +130,7 @@ Add these as repository (or organization) **Actions** secrets: The workflow reads both via `actions/create-github-app-token`. The same App and secrets back every auto-commit workflow that writes repository refs: -`perf-numbers.yml`, `update-llms-txt.yml`, -`update-issue-template-versions.yml`, `release-prepare.yml`, and +`perf-numbers.yml`, `post-merge-maintenance.yml`, `release-prepare.yml`, and `release-tag.yml`. This one provisioning unblocks the doc auto-commits and the release branch/tag handoff. diff --git a/docs/runbooks/required-checks.md b/docs/runbooks/required-checks.md index 0bc5af17..76e90eef 100644 --- a/docs/runbooks/required-checks.md +++ b/docs/runbooks/required-checks.md @@ -163,7 +163,7 @@ Markdown`, and `Prettier and yamllint` gates are the correct required checks. `Validate Documentation Build` job in `ci.yml`), and the schedule/dispatch/release workflows, none of which have a `pull_request` trigger (`unity-benchmarks.yml`, `release*.yml`, `runner-bootstrap.yml`, - `update-llms-txt.yml`, `update-issue-template-versions.yml`, `sync-wiki.yml`, + `post-merge-maintenance.yml`, `sync-wiki.yml`, `markdown-link-validity.yml`, `stuck-job-watchdog.yml`, `unstick-run.yml`, `devcontainer-prebuild.yml`). diff --git a/scripts/__tests__/auto-commit-fetch-force-refspec.test.js b/scripts/__tests__/auto-commit-fetch-force-refspec.test.js index ad9c1399..56ed46d8 100644 --- a/scripts/__tests__/auto-commit-fetch-force-refspec.test.js +++ b/scripts/__tests__/auto-commit-fetch-force-refspec.test.js @@ -17,12 +17,11 @@ const path = require("node:path"); // the update succeeds, the subsequent SHA comparison runs, and the step proceeds (or // skips) deliberately. The invariant is uniform across the full ("refs/heads/x:...") // and short ("${BRANCH}:...") source forms, and whether the shallow boundary comes -// from `clone --depth 1` (stuck-job-watchdog) or `fetch --depth=1` (perf/llms). +// from `clone --depth 1` (stuck-job-watchdog) or `fetch --depth=1` (perf/maintenance). const WORKFLOW_DIR = path.resolve(__dirname, "..", "..", ".github", "workflows"); const WORKFLOWS = [ "perf-numbers.yml", - "update-llms-txt.yml", - "update-issue-template-versions.yml", + "post-merge-maintenance.yml", "stuck-job-watchdog.yml" ]; diff --git a/scripts/__tests__/update-llms-txt.test.js b/scripts/__tests__/update-llms-txt.test.js index b54db434..38362b15 100644 --- a/scripts/__tests__/update-llms-txt.test.js +++ b/scripts/__tests__/update-llms-txt.test.js @@ -24,6 +24,7 @@ const { normalizeToLf } = require("../lib/line-endings.js"); const ROOT_DIR = path.resolve(__dirname, "..", ".."); const SCRIPT = path.join(__dirname, "..", "update-llms-txt.js"); const SKILL_CLAIM = (n) => `- ${n}+ specialized skills:`; +const LAST_UPDATED = /^\*\*Last Updated:\*\*.*$/m; test("hasValidLastUpdatedLine accepts exactly one ISO-dated line", () => { assert.equal(hasValidLastUpdatedLine("intro\n**Last Updated:** 2026-06-10\n"), true); @@ -233,6 +234,60 @@ test("CLI update mode fixes a stale single claim, then --check passes (update/ch } }); +// A generator that restamps the date on every run makes the default-branch +// auto-commit fire on any day it runs: 1efb7326, the last llms.txt auto-commit +// on master, changed exactly one line -- the date (#330). Driven through the +// CLI so it covers the real write path, not just the helper. +test("CLI update mode keeps the committed date until the content actually changes", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "llms-date-")); + try { + const skills = path.join(dir, "skills"); + for (const name of ["alpha", "beta"]) { + fs.mkdirSync(path.join(skills, name), { recursive: true }); + fs.writeFileSync(path.join(skills, name, "SKILL.md"), "---\nname: x\n---\n"); + } + const llmsTxt = path.join(dir, "llms.txt"); + const readme = path.join(dir, "README.md"); + fs.writeFileSync(readme, `intro\n${SKILL_CLAIM(2)}\noutro\n`); + const env = { ...process.env, DX_SKILLS_DIR: skills, DX_LLMS_TXT: llmsTxt, DX_README: readme }; + + execFileSync("node", [SCRIPT], { env, stdio: "pipe" }); + const dated = fs + .readFileSync(llmsTxt, "utf8") + .replace(LAST_UPDATED, "**Last Updated:** 1999-01-01"); + fs.writeFileSync(llmsTxt, dated); + + // Same content, later day: the committed date must survive untouched. + execFileSync("node", [SCRIPT], { env, stdio: "pipe" }); + assert.equal(fs.readFileSync(llmsTxt, "utf8"), dated, "a no-op regeneration rewrote the date"); + execFileSync("node", [SCRIPT, "--check"], { env, stdio: "pipe" }); + + // A corrupt date line must still be repairable: the normalized comparison + // erases the date, so a file whose only defect IS the date looks identical + // to a fresh generation, and carrying the bad line forward would leave a + // state the post-write validator rejects -- update mode exiting 1 while + // advertising itself as the fix. + fs.writeFileSync(llmsTxt, dated.replace(LAST_UPDATED, "**Last Updated:** unknown")); + execFileSync("node", [SCRIPT], { env, stdio: "pipe" }); + assert.match( + fs.readFileSync(llmsTxt, "utf8"), + /^\*\*Last Updated:\*\* \d{4}-\d{2}-\d{2}$/m, + "a malformed date line was preserved instead of repaired" + ); + fs.writeFileSync(llmsTxt, dated); + + // Real content change: the date must move with it. + fs.mkdirSync(path.join(skills, "gamma"), { recursive: true }); + fs.writeFileSync(path.join(skills, "gamma", "SKILL.md"), "---\nname: x\n---\n"); + execFileSync("node", [SCRIPT], { env, stdio: "pipe" }); + const refreshed = fs.readFileSync(llmsTxt, "utf8"); + assert.match(refreshed, /\bgamma\b/, "the new skill was not generated"); + assert.doesNotMatch(refreshed, /1999-01-01/, "real content changed but the date did not"); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + test("collectValidationErrors (the shared --check/update contract) flags a malformed sibling", () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "llms-cve-")); try { diff --git a/scripts/update-llms-txt.js b/scripts/update-llms-txt.js index 372c3827..28e451fe 100755 --- a/scripts/update-llms-txt.js +++ b/scripts/update-llms-txt.js @@ -40,6 +40,7 @@ const LLM_SKILLS_DIR = process.env.DX_SKILLS_DIR || path.join(ROOT_DIR, ".llm", // entire "forgot to bump the skill count" failure class while still catching // genuinely false claims. See validateSkillCountClaim. const SKILL_CLAIM_REGEX = /(\d+)\+ specialized skills/g; +const LAST_UPDATED_LINE = /^\*\*Last Updated:\*\*.*$/m; // Docs whose skill-count claim is guarded (missing files are skipped). Resolved // per run so the CLI exit-code paths stay testable against a temp fixture via the @@ -81,7 +82,6 @@ function countSkillFiles() { return listSkillNames().length; } - /** * Extract every "N+ specialized skills" claim from content as integers, * in document order. @@ -145,10 +145,7 @@ function syncSkillCountClaim(filePath, actualCount) { return false; } - const updated = original.replace( - SKILL_CLAIM_REGEX, - `${actualCount}+ specialized skills` - ); + const updated = original.replace(SKILL_CLAIM_REGEX, `${actualCount}+ specialized skills`); if (updated === original) { return false; } @@ -199,6 +196,44 @@ function hasValidLastUpdatedLine(content) { const isoDatePattern = /^\*\*Last Updated:\*\*\s+\d{4}-\d{2}-\d{2}\s*$/; return isoDatePattern.test(line); } + +/** + * Keep the committed "Last Updated" date when a fresh generation is otherwise + * byte-identical, so the field means "last time this content changed" rather + * than "last time the generator ran". + * + * The date is the one field that moves without the content moving, so a + * generator that always rewrites it makes the default-branch auto-commit fire + * on any day the workflow runs. That is not hypothetical: 1efb7326, the last + * llms.txt auto-commit on master, changed exactly one line -- the date -- and + * still re-triggered the whole push-side workflow set (#330). + * + * The comparison normalizes ONLY the date, not the skill-count claim that + * `normalizeForComparison` also folds away: a count change is real content, and + * must move the date with it. + * + * Only a date this generator would itself accept is carried forward. Preserving + * a malformed one would take update mode's ability to repair it away: the + * normalized comparison erases the date, so a file whose ONLY defect is a + * corrupt date line looks identical to a fresh generation, and copying the + * corrupt line back leaves a state the post-write validator rejects. Update + * mode would then exit 1 while advertising itself as the remediation. + */ +function preserveUnchangedDate(newContent, llmsTxtPath) { + if (!fs.existsSync(llmsTxtPath)) { + return newContent; + } + const current = fs.readFileSync(llmsTxtPath, "utf8"); + if (!hasValidLastUpdatedLine(current)) { + return newContent; + } + const withoutDate = (text) => normalizeToLf(text).replace(LAST_UPDATED_LINE, ""); + if (withoutDate(current) !== withoutDate(newContent)) { + return newContent; + } + const committed = current.match(LAST_UPDATED_LINE); + return committed === null ? newContent : newContent.replace(LAST_UPDATED_LINE, committed[0]); +} function getPackageInfo() { const pkg = JSON.parse(fs.readFileSync(PACKAGE_JSON_PATH, "utf8")); return { @@ -535,7 +570,7 @@ function normalizeForComparison(str) { // Normalize the Last Updated line by replacing the date with a fixed // placeholder, while keeping the marker text so structural differences are // still detected. - .replace(/^\*\*Last Updated:\*\*.*$/m, "**Last Updated:** ") + .replace(LAST_UPDATED_LINE, "**Last Updated:** ") // Normalize the floored skill-count claim. The exact number is validated // separately (no-overstatement, see validateSkillCountClaim); keeping it out // of the structural diff means adding/removing a skill never trips --check. @@ -626,7 +661,11 @@ function main() { // Update mode: write llms.txt (LF endings, matching .gitattributes for // *.txt), then keep every other doc that advertises the skill count in sync // so the single documented remediation fixes the whole class of drift. - fs.writeFileSync(llmsTxtPath, normalizeToLf(newContent), "utf8"); + fs.writeFileSync( + llmsTxtPath, + normalizeToLf(preserveUnchangedDate(newContent, llmsTxtPath)), + "utf8" + ); console.log("[ok] Updated llms.txt"); for (const { label, filePath } of claimFiles) { diff --git a/scripts/validate-js-loc-budget.js b/scripts/validate-js-loc-budget.js index 0e9a8226..b17dbb57 100644 --- a/scripts/validate-js-loc-budget.js +++ b/scripts/validate-js-loc-budget.js @@ -77,7 +77,26 @@ const path = require("path"); // hand-authored scaffolding that the sibling marker-scoping fix exists to // protect. +37 source and +112 test lines, including a loose timing guard // that only a return to quadratic can breach: 17500. -const TOTAL_BUDGET = 17500; +// 077 Stop llms.txt auto-committing a date it restamped for no reason. The +// generator rewrote "Last Updated" on every run, so the default-branch +// auto-commit fired on any day the workflow ran: 1efb7326, the last +// llms.txt commit on master, changed exactly one line -- the date -- and +// still re-triggered the whole push-side workflow set (#330). The date now +// survives a byte-identical regeneration and moves only when the content +// does, so it finally means what it says. +21 source lines +// (preserveUnchangedDate plus a shared LAST_UPDATED_LINE that replaces a +// duplicated literal) and +36 test lines driving it through the CLI, so the +// real write path is covered rather than the helper alone; both mutations +// (unconditional restamp, preserve-through-a-content-change) fail the +// suite. Offset by -1 line in the fetch-refspec guard, whose two merged +// workflow entries collapsed to one, and the comment lines of this very +// entry. Adversarial review then added the malformed-date guard and its +// regression case: preserving a date the generator itself rejects took +// update mode's ability to REPAIR one away, because the normalized +// comparison erases the date, so a file whose only defect was a corrupt +// date line looked identical to a fresh generation and update exited 1 +// while advertising itself as the fix: 17612. +const TOTAL_BUDGET = 17612; const LARGEST_FILE_COUNT = 10; const REPO_ROOT = path.resolve(__dirname, ".."); diff --git a/scripts/validate-unity-pr-policy.py b/scripts/validate-unity-pr-policy.py index 0639382f..0752becb 100644 --- a/scripts/validate-unity-pr-policy.py +++ b/scripts/validate-unity-pr-policy.py @@ -3,15 +3,18 @@ from __future__ import annotations +import base64 import json import os import re import subprocess import tempfile +from datetime import datetime, timezone from pathlib import Path WORKFLOW = Path(".github/workflows/unity-tests.yml") +WATCHDOG = Path(".github/workflows/stuck-job-watchdog.yml") LOCK_ACTION_PREFIX = "Ambiguous-Interactive/ambiguous-organization-build-lock/.github/actions/" REGISTERED_UNITY_AUTOMATION = { ".github/actions/validate-unity-license/action.yml", @@ -668,6 +671,43 @@ def find_workflow_editor_mutations(files: dict[str, str]) -> list[str]: return violations +def validate_cleanup_gate_not_attempted_input(job: str, label: str) -> None: + """Pin how a lock window tells the cleanup gate that acquisition never ran. + + The gate's contract is that `acquired: false` proves licensed cleanup was + not required, and a bare `if: always()` is what catches a seat leak when + acquisition fails part-way. Both have to hold at once (#327): a leg that + aborts before `Acquire organization Unity lock` must report exactly one + failure, and a leg whose acquire failed after taking the lock must still + fail. The distinction is `outcome`, which is empty or `skipped` only when + the step did not execute; gating on `acquired == 'true'` instead would + skip the gate in precisely the case it must never miss. + """ + gate = step_block(job, "Require confirmed Unity cleanup") + require( + "\n if: always()\n" in gate, + f"{label}: the cleanup gate must keep a bare `if: always()`", + ) + acquired = re.search(r"\n acquired: (.*)\n", gate) + require(acquired is not None, f"{label}: the cleanup gate must pass `acquired`") + expression = acquired.group(1) + for fragment in ( + "steps.acquire_lock.outcome == 'skipped'", + "steps.acquire_lock.outcome == ''", + "&& 'false' ||", + "steps.acquire_lock.outputs.acquired", + ): + require( + fragment in expression, + f"{label}: the cleanup gate's `acquired` input must map only a step that " + f"never executed to 'false' and pass everything else through; missing {fragment!r}", + ) + require( + "outcome == 'failure'" not in expression and "outcome != " not in expression, + f"{label}: a failed acquire must not be laundered into a not-attempted verdict", + ) + + def validate_lock_window_timeout_budget(job: str, label: str) -> None: steps = top_level_steps_through_cleanup_gate(job, label) bounded_minutes = 0 @@ -818,6 +858,1447 @@ def run_head_check(script: str, event: str, live_head: str) -> str: return written[len("superseded=") :] +def watchdog_gh_stub(routes: dict[str, tuple[int, object]], cancel_ok: bool) -> str: + """A `gh` that answers only the stubbed endpoints and rejects the rest. + + An unstubbed call exits 3 rather than printing nothing, so a watchdog that + reaches for an endpoint this table does not model fails the case instead of + quietly reading an empty body. + """ + # A `str` payload is emitted verbatim rather than JSON-encoded. The audit + # pipes some calls through `gh --jq`, which this stub does not implement, so + # a route that feeds such a call has to supply the already-extracted value. + cases = "".join( + f" *{needle}*)\n cat <<'PAYLOAD'\n" + f"{payload if isinstance(payload, str) else (json.dumps(payload) if payload is not None else '')}" + f"\nPAYLOAD\n exit {code} ;;\n" + for needle, (code, payload) in routes.items() + ) + # Organization endpoints answer ONLY when the call carries the reader App + # token. A static grep can prove the token is minted; this proves it is the + # one actually used, which is the credential shape #328 was about -- the + # audit read inventory with a token that could never reach those endpoints. + return ( + "#!/bin/bash\n" + 'if [ "$1" = "run" ] && [ "$2" = "cancel" ]; then\n' + f' echo "cancelled $3"; exit {0 if cancel_ok else 1}\n' + "fi\n" + 'case "$*" in\n' + " *orgs/*)\n" + ' if [ "${GH_TOKEN:-}" != "stub-reader" ]; then\n' + ' echo "gh: Resource not accessible by integration (HTTP 403)" >&2\n' + " exit 1\n" + " fi ;;\n" + "esac\n" + f'case "$*" in\n{cases} *) echo "unstubbed gh: $*" >&2; exit 3 ;;\nesac\n' + ) + + +# The watchdog never needs a real remote: `ls-remote` reporting an empty branch +# list sends it down the bootstrap path, and `push` succeeds. Everything else +# (init, checkout, add, commit) runs against the real git in a temp directory. +def watchdog_git_stub( + push_ok: bool = True, + ls_remote_ok: bool = True, + existing_state: dict[int, int] | None = None, + clone_ok: bool = True, + state_age_seconds: int = 3600, +) -> str: + """A `git` that passes through except for the three remote operations. + + `existing_state` makes `ls-remote` report the branch as PRESENT and has + `clone` materialize it with the given per-run cancel counts. Without it the + stub could only ever exercise the bootstrap path, which left both the + clone-failure guard and the cancel cap unreachable from the suite. + """ + if existing_state is None: + ls_remote = f"exit {0 if ls_remote_ok else 1}" + clone = "exit 1" + else: + ls_remote = ( + f"echo 'deadbeefdeadbeefdeadbeefdeadbeefdeadbeef\trefs/heads/state'; " + f"exit {0 if ls_remote_ok else 1}" + ) + if clone_ok: + # `last_cancel` defaults to an hour ago, inside the 24h reset + # window; `state_age_seconds` moves it outside so the reset itself + # becomes reachable. A far-future stamp would make elapsed time + # negative, and then no window size could change the verdict at all. + writes = "".join( + f"printf '{{\"cancels\": {n}, \"last_cancel\": " + f"'$(( $(date -u +%s) - {state_age_seconds} ))'}}' " + f'> "$target/.watchdog-state/{run_id}.json"; ' + for run_id, n in existing_state.items() + ) + clone = ( + 'target="${@: -1}"; /usr/bin/git init -q "$target"; ' + 'mkdir -p "$target/.watchdog-state"; ' + f"{writes}" + '/usr/bin/git -C "$target" add -A; ' + '/usr/bin/git -C "$target" -c user.email=t@t -c user.name=t commit -qm state; ' + "exit 0" + ) + else: + clone = "exit 1" + return ( + "#!/bin/bash\n" + 'for a in "$@"; do\n' + ' case "$a" in\n' + f" ls-remote) {ls_remote} ;;\n" + " fetch) exit 0 ;;\n" + f" clone) {clone} ;;\n" + f' push) echo "pushed"; exit {0 if push_ok else 1} ;;\n' + " esac\n" + "done\n" + 'exec /usr/bin/git "$@"\n' + ) + + +# Values the runner supplies from the workflow context; everything else in the +# step's `env:` is a literal the workflow OWNS, and the harness must read those +# from the file rather than restate them. Restating them is what makes a case +# assert its own stub: a truth table that hardcodes +# `DEFAULT_EXCLUDED_WORKFLOWS: release.yml` still passes after the workflow +# stops excluding release.yml. +WATCHDOG_CONTEXT_ENV = { + "GH_TOKEN": "stub", + "RUNNER_INVENTORY_TOKEN": "stub-reader", + "REPO": "Ambiguous-Interactive/DxMessaging", + "OWNER": "Ambiguous-Interactive", + "SELF_RUN_ID": "999", + "EXTRA_EXCLUDED_WORKFLOWS": "", +} +# Literals the cases below actually depend on. Each is exercised by at least one +# case, so a rename or deletion fails rather than silently dropping the behavior +# it configures. Adding a name here without a case that depends on its VALUE +# would restore the false guarantee this list used to advertise. +WATCHDOG_REQUIRED_ENV_LITERALS = ( + "STATE_BRANCH", + "STATE_DIR", + "MAX_CANCELS_PER_DAY", + "MIN_QUEUE_AGE_SECONDS", + "DEFAULT_EXCLUDED_WORKFLOWS", +) + + +def step_env_literals(step: str) -> dict[str, str]: + """Return the step's `env:` entries whose values are workflow literals.""" + start = step.find(" env:\n") + require(start >= 0, "watchdog audit step must declare env") + literals: dict[str, str] = {} + for line in step[start + len(" env:\n") :].splitlines(): + entry = re.fullmatch(r" ([A-Z0-9_]+): (.*)", line) + if entry is None: + break + name, value = entry.group(1), entry.group(2).strip() + if "${{" in value: + continue + literals[name] = value[1:-1] if len(value) >= 2 and value[0] == value[-1] == '"' else value + return literals + + +class WatchdogOutput(str): + """Combined job-log + step-summary text, with the summary kept separately. + + `log_summary` tees to stdout, and the harness used to hand back + `summary + stdout + stderr` as one blob -- so every needle was satisfiable + from the job log alone and NO case could prove a line actually reached + `GITHUB_STEP_SUMMARY`. Three of the four step-summary buckets and the whole + audit narrative were deletable with the suite green. Behaves as the same + string it always did; `.summary` is the channel-specific view. + """ + + summary: str + + def __new__(cls, combined: str, summary: str) -> "WatchdogOutput": + value = super().__new__(cls, combined) + value.summary = summary + return value + + +def run_watchdog( + script: str, + environment_literals: dict[str, str], + routes: dict[str, tuple[int, object]], + cancel_ok: bool = True, + push_ok: bool = True, + ls_remote_ok: bool = True, + existing_state: dict[int, int] | None = None, + clone_ok: bool = True, + state_age_seconds: int = 3600, + break_date: bool = False, + extra_env: dict[str, str] | None = None, +) -> tuple[int, str]: + """Execute the watchdog audit script against a stubbed `gh` and `git`. + + `break_date` fails the unguarded `now_epoch="$(date ...)"` assignment, which + is the cheapest way to force an abort the script does NOT anticipate -- the + only thing that exercises the EXIT trap rather than a `finish` call. + """ + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + stubs = [ + ("gh", watchdog_gh_stub(routes, cancel_ok)), + ( + "git", + watchdog_git_stub( + push_ok, ls_remote_ok, existing_state, clone_ok, state_age_seconds + ), + ), + ] + if break_date: + stubs.append(("date", "#!/bin/bash\nexit 1\n")) + for name, body in stubs: + stub = root / name + stub.write_text(body, encoding="utf-8") + stub.chmod(0o755) + summary = root / "summary.md" + summary.touch() + environment = os.environ.copy() + environment.update(environment_literals) + environment.update(WATCHDOG_CONTEXT_ENV) + environment.update(extra_env or {}) + environment["PATH"] = f"{root}{os.pathsep}{environment['PATH']}" + environment["GITHUB_STEP_SUMMARY"] = str(summary) + # The audit calls `mktemp` freely, which is free on an ephemeral runner + # but permanent in a developer's /tmp -- roughly 440 entries per run of + # this validator. Point it at the case's own directory instead. + environment["TMPDIR"] = str(root) + result = subprocess.run( + ["bash", "-c", script], env=environment, capture_output=True, text=True, check=False + ) + summary_text = summary.read_text(encoding="utf-8") + return result.returncode, WatchdogOutput( + summary_text + result.stdout + result.stderr, summary_text + ) + + +def watchdog_queued_runs(*runs: dict[str, object]) -> tuple[int, object]: + return 0, {"workflow_runs": list(runs)} + + +def watchdog_run( + run_id: int, + workflow: str = "perf-numbers.yml", + event: str = "push", + created_at: str = "2020-01-01T00:00:00Z", +) -> dict[str, object]: + # Created in 2020, so it is unconditionally older than MIN_QUEUE_AGE_SECONDS. + return { + "id": run_id, + "created_at": created_at, + "path": f".github/workflows/{workflow}", + "event": event, + "workflow_id": 42, + "head_branch": "master", + "html_url": f"https://github.com/Ambiguous-Interactive/DxMessaging/actions/runs/{run_id}", + } + + +def watchdog_jobs(*label_sets: list[str], extra: list[dict] | None = None) -> tuple[int, object]: + """Queued jobs carrying `label_sets`, plus any raw jobs in `extra`. + + `extra` exists so a case can put a job in a NON-queued state. Without it the + fixture could only ever produce queued jobs, and the two guards that read job + status -- the in-progress early-continue and the zero-queued-jobs check -- + were unreachable from the whole suite. + """ + jobs = [{"status": "queued", "labels": labels} for labels in label_sets] + return 0, {"jobs": jobs + list(extra or [])} + + +def watchdog_runners(*specs: tuple[str, str, bool, list[str]]) -> tuple[int, object]: + return 0, { + "runners": [ + { + "id": index, + "name": name, + "status": status, + "busy": busy, + "labels": [{"name": label} for label in labels], + } + for index, (name, status, busy, labels) in enumerate(specs, start=1) + ] + } + + +# The redispatch branch reads the workflow file through `gh --jq .content`, which +# is base64. Encoding here rather than embedding the literal keeps the fixture +# readable and keeps a meaningless base64 blob out of the spell checker. +STARVED_SECTION_EMPTY = "required labels)\n_(none)_" + + +DISPATCHABLE_WORKFLOW_BODY = base64.b64encode(b"on:\n workflow_dispatch:\n").decode() +# A body that decodes cleanly but declares no `workflow_dispatch` trigger. It +# is the only input that separates "detection said no" from "the base64 decode +# failed", which is what an empty `content` actually exercises. +NON_DISPATCHABLE_WORKFLOW_BODY = base64.b64encode(b"on:\n push:\n").decode() + + +def validate_stuck_job_watchdog() -> None: + """Execute the watchdog's audit script across its verdict space. + + The watchdog is the automation that recovers the licensed Unity legs this + file governs, so its failure modes belong to the same policy: #328 was a + watchdog that could not read runner inventory on either endpoint it tried, + reported `success` anyway, and let a `Performance Numbers` run sit queued + for ten hours. The rule these cases pin is that a green watchdog run means + the queue was evaluated -- never that evaluation was skipped. + """ + source = WATCHDOG.read_text(encoding="utf-8") + require( + "actions/create-github-app-token@" in source + and "app-id: ${{ secrets.BUILD_LOCK_READER_APP_ID }}" in source, + "watchdog must read runner inventory with the organization reader App; " + "the job GITHUB_TOKEN is repository-scoped and cannot list organization runners", + ) + require( + "repos/${REPO}/actions/runners" not in source, + "watchdog must not fall back to repository-level runners: this repository " + "registers none, so the call succeeds with an empty inventory that reads " + "identically to 'no runner matches' (#328)", + ) + audit_step = step_block(job_block(source, "audit-queue"), "Audit + cancel-and-redispatch") + script = run_script(audit_step) + environment_literals = step_env_literals(audit_step) + for name in WATCHDOG_REQUIRED_ENV_LITERALS: + require(name in environment_literals, f"watchdog env must declare {name} as a literal") + require( + "release.yml" in environment_literals["DEFAULT_EXCLUDED_WORKFLOWS"].split(), + "watchdog must never cancel a queued release run", + ) + if os.name == "nt": + return + + self_hosted = ["self-hosted", "Windows", "RAM-64GB", "fast"] + queued = "actions/runs?status=queued" + inventory = "runner-groups?visible_to_repository" + one_group = (0, {"runner_groups": [{"id": 7, "name": "Default"}]}) + + # name, gh routes, expected exit, must appear, must NOT appear + FRESH_TIMESTAMP = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + STUCK_ROUTES = { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs(self_hosted), + "actions/workflows/42": (0, {"path": ".github/workflows/perf-numbers.yml"}), + "contents/.github/workflows/perf-numbers.yml": (0, {"content": ""}), + } + + cases: tuple[tuple[str, dict[str, tuple[int, object]], int, tuple[str, ...], tuple[str, ...]], ...] = ( + ( + "clean queue", + {queued: (0, {"workflow_runs": []})}, + 0, + ("Queue is clean",), + ("::error::",), + ), + ( + "unreadable queue fails closed", + {queued: (1, None)}, + 1, + ("failed to list queued runs",), + (), + ), + ( + "unreadable runner inventory fails closed", + {queued: watchdog_queued_runs(watchdog_run(1)), inventory: (1, None)}, + 1, + ("could not read the organization runner groups",), + (), + ), + ( + "empty visible runner-group set fails closed", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: (0, {"runner_groups": []}), + }, + 1, + ("no runner groups visible",), + (), + ), + ( + "unreadable jobs fail closed", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + "actions/runs/1/jobs": (1, None), + }, + 1, + ("failed to list jobs for run 1",), + (), + ), + ( + "idle matching runner is dispatcher-stuck", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs(self_hosted), + "actions/workflows/42": (0, {"path": ".github/workflows/perf-numbers.yml"}), + "contents/.github/workflows/perf-numbers.yml": (0, {"content": ""}), + }, + 0, + ("dispatcher-stuck", "cancelled 1", "does not support workflow_dispatch"), + ("re-dispatching",), + ), + ( + "busy matching runner is healthy backpressure", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", True, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs(self_hosted), + }, + 0, + ("healthy backpressure",), + ("cancelled 1", "::warning::"), + ), + ( + "registered but offline runner is starved, not stuck", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "offline", False, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs(self_hosted), + }, + 0, + ("starved", "registered but offline", "::warning::"), + ("cancelled 1", STARVED_SECTION_EMPTY), + ), + ( + "no registered self-hosted runner is starved, not stuck", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("MAC", "online", False, ["self-hosted", "macOS"])), + "actions/runs/1/jobs": watchdog_jobs(self_hosted), + }, + 0, + ("starved", "no runner registered", "::warning::"), + ("cancelled 1", STARVED_SECTION_EMPTY), + ), + ( + "GitHub-hosted run is not reported as starved", + { + queued: watchdog_queued_runs(watchdog_run(1, workflow="ci.yml")), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs(["ubuntu-latest"]), + }, + 0, + ("no queued job requests a self-hosted runner",), + ("cancelled 1", "::warning::"), + ), + ( + "label match is case-insensitive in both directions", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", True, ["Self-Hosted", "WINDOWS"])), + "actions/runs/1/jobs": watchdog_jobs(["self-hosted", "windows"]), + }, + 0, + ("healthy backpressure",), + ("::warning::",), + ), + ( + "a job reporting no labels is never cancelled", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs([]), + }, + 0, + ("queued jobs report no labels",), + ("cancelled 1",), + ), + ( + # Finding: `mapfile < <(jq ...)` discarded jq's exit status, so one + # unparseable timestamp printed "Queue is clean" over a stuck queue. + "an unparseable queued-run timestamp fails closed", + { + queued: ( + 0, + { + "workflow_runs": [ + watchdog_run(1) | {"created_at": "not-a-date"}, + watchdog_run(2), + ] + }, + ) + }, + 1, + ("could not parse the queued-run listing",), + ("Queue is clean",), + ), + ( + # Finding: the starvation report latched on the FIRST label set while + # the verdict accumulated across all of them, so a GitHub-hosted set + # sorting first suppressed a real starvation warning outright. + "a hosted label set never suppresses a self-hosted starvation", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + # "macos-latest" sorts before "self-hosted", so jq `unique` puts + # the hosted set first -- the ordering that used to lose. + "actions/runs/1/jobs": watchdog_jobs( + ["macos-latest"], ["self-hosted", "Windows", "unicorn"] + ), + }, + 0, + ("starved", "::warning::", "self-hosted, windows, unicorn"), + ( + "no queued job requests a self-hosted runner", + "cancelled 1", + "macos-latest] is registered", + ), + ), + ( + # Finding: the warning interpolated the first set's labels while + # branching on the whole-run verdict, so it told the operator to + # bring online a runner named by a GitHub-hosted label. + "the starvation warning names the self-hosted labels, not a hosted one", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "offline", False, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs(["macos-latest"], self_hosted), + }, + 0, + ("registered but offline", "self-hosted, windows, ram-64gb, fast"), + ("[macos-latest] is registered", "cancelled 1"), + ), + ( + # Finding: an empty workflow path is a hard bash error on the + # exclusion-list subscript, which aborted with an EMPTY summary. + "a run with no workflow path is reported, not crashed on", + { + queued: ( + 0, + {"workflow_runs": [watchdog_run(1) | {"path": None}]}, + ), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + }, + 0, + ("no workflow path reported", "Watchdog summary"), + ("bad array subscript", "cancelled 1"), + ), + ( + # THE guard that matters most. A matrix cell executing on one runner + # while a sibling cell waits is not dispatcher-stuck -- it holds a + # Unity licence seat. Cancelling it kills a live Unity session + # mid-test, which is the seat leak `require-confirmed-unity-cleanup` + # exists to catch. Nothing in the suite reached this branch before: + # the job fixture could only produce QUEUED jobs. + "a run with an in-progress job is never cancelled", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs( + self_hosted, extra=[{"status": "in_progress", "labels": self_hosted}] + ), + }, + 0, + ("healthy queued", "1 in_progress"), + ("cancelled 1", "queued for cancel"), + ), + ( + # A run whose jobs have all finished is in a transitional state, not + # the dispatcher-stuck pattern. + "a run with no queued jobs is never cancelled", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs( + extra=[{"status": "completed", "labels": self_hosted}] + ), + }, + 0, + ("no queued jobs yet",), + ("cancelled 1",), + ), + ( + # The audit must never cancel itself, however the exclusion list is + # configured. SELF_RUN_ID is 999 in the harness, so the fixture run + # has to carry that id for the guard to be exercised at all. + "the watchdog never cancels its own run", + { + queued: watchdog_queued_runs(watchdog_run(999)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + }, + 0, + ("this is the watchdog's own run",), + ("cancelled 999",), + ), + ( + # The inventory is read in TWO calls. Only the runner-GROUP listing + # was covered; a failure of the per-group RUNNER listing could be + # downgraded to a log line and the audit would report exit 0 over an + # empty inventory -- precisely the #328 shape. + "an unreadable runner listing inside a group fails closed", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": (1, None), + }, + 1, + ("could not read runners in organization runner group",), + (), + ), + ( + # The redispatch branch had never executed: the fixture returned an + # empty `content`, so `base64 -d` always failed and the workflow was + # always treated as non-dispatchable. + "a dispatchable push run is cancelled and re-dispatched", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs(self_hosted), + # Ordered BEFORE the plain workflow route: the stub matches + # substrings in insertion order, so `actions/workflows/42` would + # otherwise swallow the dispatches POST and answer it silently. + "actions/workflows/42/dispatches": (0, "DISPATCH-REQUESTED"), + "actions/workflows/42": (0, {"path": ".github/workflows/perf-numbers.yml"}), + "contents/.github/workflows/perf-numbers.yml": (0, DISPATCHABLE_WORKFLOW_BODY), + }, + 0, + # "cancelled 1" and the log line both precede the POST, so neither + # proves it happened. Only the stub's response to the dispatches + # call proves the request was actually made. + ("cancelled 1", "re-dispatching workflow 42", "DISPATCH-REQUESTED"), + (), + ), + ( + # A pull_request run must NEVER be re-dispatched: the dispatches + # endpoint cannot re-trigger one, and the documented path is a + # cancel plus an operator instruction. The workflow body here IS + # dispatchable, so only the event check can hold the line. + "a pull_request run is cancelled but never re-dispatched", + { + queued: watchdog_queued_runs(watchdog_run(1, event="pull_request")), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs(self_hosted), + "actions/workflows/42": (0, {"path": ".github/workflows/perf-numbers.yml"}), + "contents/.github/workflows/perf-numbers.yml": (0, DISPATCHABLE_WORKFLOW_BODY), + }, + 0, + ("cancelled 1", "pull_request-triggered", "Re-run all jobs"), + ("re-dispatching",), + ), + ( + # A busy sibling must not hide a starved one. The run is legitimately + # healthy (something can proceed) AND a second label set can never be + # picked up; both are true and both have to be reported, or the + # starvation stays invisible for as long as the busy leg runs. + "a busy sibling does not suppress a co-resident starvation", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", True, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs( + self_hosted, ["self-hosted", "Windows", "unicorn"] + ), + }, + 0, + ("healthy backpressure", "starved", "::warning::", "unicorn"), + ("cancelled 1", STARVED_SECTION_EMPTY), + ), + ( + # The longest-lived form of the same blindness: once a runner picks + # up the healthy cell the run is in-progress, and that state lasts + # for the rest of the matrix. If the in-progress exit skips the + # starvation report, the starved sibling is silent for hours. + "an in-progress sibling does not suppress a co-resident starvation", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", True, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs( + ["self-hosted", "Windows", "unicorn"], + extra=[{"status": "in_progress", "labels": self_hosted}], + ), + }, + 0, + ("healthy queued", "1 in_progress", "starved", "::warning::", "unicorn"), + ("cancelled 1", STARVED_SECTION_EMPTY), + ), + ( + # The third and last shape of sibling blindness, and the one that + # persists longest: while the dispatchable cell keeps matching idle, + # the run is cancelled and re-dispatched over and over, and the + # starved cell stays invisible across every cycle. `self_hosted` + # sorts BEFORE the unicorn set under jq `unique`, so the idle match + # is seen first -- exactly the ordering a `break` would lose. + "a cancellable sibling does not suppress a co-resident starvation", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs( + self_hosted, ["self-hosted", "Windows", "unicorn"] + ), + "actions/workflows/42": (0, {"path": ".github/workflows/perf-numbers.yml"}), + "contents/.github/workflows/perf-numbers.yml": (0, {"content": ""}), + }, + 0, + ("cancelled 1", "dispatcher-stuck", "starved", "::warning::", "unicorn"), + (STARVED_SECTION_EMPTY,), + ), + ( + # Precedence: a later `busy` set must not demote an earlier `idle` + # one. Without the break, that demotion is what would silently stop + # the run being cancelled. The zebra set sorts AFTER self_hosted + # under jq `unique`, so it is scanned second. + "a later busy set does not demote an earlier idle match", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners( + ("ELI", "online", False, self_hosted), + ("DAD", "online", True, ["self-hosted", "Windows", "zebra"]), + ), + "actions/runs/1/jobs": watchdog_jobs( + self_hosted, ["self-hosted", "Windows", "zebra"] + ), + "actions/workflows/42": (0, {"path": ".github/workflows/perf-numbers.yml"}), + "contents/.github/workflows/perf-numbers.yml": (0, {"content": ""}), + }, + 0, + ("cancelled 1", "dispatcher-stuck"), + ("healthy backpressure",), + ), + ( + # Precedence: a later `offline` set must not demote an earlier + # `busy` one. The run is still waiting on a real runner, so it stays + # healthy -- while the offline set is still reported as starved. + "a later offline set does not demote an earlier busy match", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners( + ("ELI", "online", True, ["self-hosted", "Windows", "aaa"]), + ("DAD", "offline", False, ["self-hosted", "Windows", "zzz"]), + ), + "actions/runs/1/jobs": watchdog_jobs( + ["self-hosted", "Windows", "aaa"], ["self-hosted", "Windows", "zzz"] + ), + }, + 0, + ("healthy backpressure", "starved", "zzz"), + ("cancelled 1", STARVED_SECTION_EMPTY), + ), + ( + # Pins the `workflow_dispatch:` grep itself. The dispatcher-stuck + # case above uses an empty `content`, so `base64 -d` fails and the + # grep never runs -- it reads as if it asserts detection but only + # asserts the decode-failure path. A regression that dispatched + # unconditionally would POST to a workflow with no such trigger, get + # a 422, and leave the run cancelled with no recovery. + "a run whose workflow declares no workflow_dispatch is not re-dispatched", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs(self_hosted), + "actions/workflows/42": (0, {"path": ".github/workflows/perf-numbers.yml"}), + "contents/.github/workflows/perf-numbers.yml": ( + 0, + NON_DISPATCHABLE_WORKFLOW_BODY, + ), + }, + 0, + ("cancelled 1", "does not support workflow_dispatch", "Re-run all jobs"), + ("re-dispatching",), + ), + ( + # Pins the starvation PRECEDENCE. Two self-hosted sets starve for + # different reasons: one has a registered-but-offline runner, the + # other has nothing at all. The unregistered set must win, because a + # label nothing carries needs a human while an offline machine may + # reconnect on its own. Reporting the offline one instead points the + # operator at a machine that exists and will come back. + "an unregistered label set outranks an offline one in the report", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + # `aaa` sorts BEFORE `zzz` under jq `unique`, so the OFFLINE set + # is scanned first and plain first-wins would report it. Only the + # upgrade clause promotes the unregistered set. With the sets the + # other way round the clause never executes and its deletion + # survives -- which is how this case originally passed. + "runner-groups/7/runners": watchdog_runners( + ("DAD", "offline", False, ["self-hosted", "aaa"]) + ), + "actions/runs/1/jobs": watchdog_jobs( + ["self-hosted", "aaa"], ["self-hosted", "zzz"] + ), + }, + 0, + ("starved", "no runner registered", "zzz"), + ("registered but offline", "cancelled 1", STARVED_SECTION_EMPTY), + ), + ( + # Two dispatcher-stuck runs in one cycle: the cancel loop must handle + # both, which is what exercises the `mapfile`-not-`while read` stdin + # defense and per-run cap independence. + "two dispatcher-stuck runs are both cancelled", + { + queued: watchdog_queued_runs(watchdog_run(1), watchdog_run(2)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs(self_hosted), + "actions/runs/2/jobs": watchdog_jobs(self_hosted), + "actions/workflows/42": (0, {"path": ".github/workflows/perf-numbers.yml"}), + "contents/.github/workflows/perf-numbers.yml": (0, {"content": ""}), + }, + 0, + ("cancelled 1", "cancelled 2"), + (), + ), + ( + # An unusable cancel-cap state branch must fail rather than cancel + # blind: without the cap a run could be cancelled without limit. The + # pending run must still be NAMED, or a fail-closed cap leaves the + # operator nothing to act on. + "an unusable state branch fails closed and names the pending run", + STUCK_ROUTES, + 1, + ("run 1", "queued for cancel"), + ("cancelled 1",), + {"push_ok": False}, + ), + ( + # A transient `ls-remote` failure must not be read as "the branch does + # not exist": bootstrapping on that would push-corrupt the real branch + # by rewriting it as a fresh orphan, resetting every cancel cap. + "an unreadable state branch fails closed", + STUCK_ROUTES, + 1, + (), + ("cancelled 1",), + {"ls_remote_ok": False}, + ), + ( + "a state branch that cannot be cloned fails closed", + STUCK_ROUTES, + 1, + ("clone failed",), + (), + {"existing_state": {}, "clone_ok": False}, + ), + ( + # The cap is what stops a run being cancelled without limit. + "a run past its daily cancel cap is not cancelled again", + STUCK_ROUTES, + 0, + ("cap reached",), + ("cancelled 1",), + {"existing_state": {1: 2}}, + ), + ( + # ...and the cap resets after 24h, which the fixture could not reach + # while it hardcoded an age inside the window. + "a cancel cap older than 24h resets", + STUCK_ROUTES, + 0, + ("cancelled 1",), + ("cap reached",), + {"existing_state": {1: 2}, "state_age_seconds": 90000}, + ), + ( + # A rejected cancel must not increment the cap or re-dispatch: that + # would duplicate a run that is still live. + "a rejected cancel is not treated as a cancel", + STUCK_ROUTES, + 0, + ("will try again next cycle",), + ("re-dispatching",), + {"cancel_ok": False}, + ), + ( + # RESTORED: deleted by the table fold in 98cd2094. Labels that are not + # strings break `ascii_downcase` INSIDE the label parse, after the job + # listing itself parsed cleanly -- a different guard from the listing + # read, and the only one that can leave a run silently unevaluated. + "job labels that are not strings fail closed", + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs(extra=[{"status": "queued", "labels": [17]}]), + }, + 1, + ("could not parse the job labels",), + (), + ), + ( + # RESTORED: deleted by the table fold. A queued listing whose ids do + # not round-trip to their own metadata (here an id typed as a string) + # leaves the run unclassifiable, and an unclassifiable run must fail + # the audit rather than be skipped past. + "a run whose id does not round-trip fails closed", + { + queued: (0, {"workflow_runs": [watchdog_run(1) | {"id": "1"}]}), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + }, + 1, + ("carries no metadata",), + (), + ), + ( + # RESTORED: deleted by the table fold. A run younger than + # MIN_QUEUE_AGE_SECONDS is not a candidate at all. The timestamp is + # derived from the workflow's own literal, so lowering that literal + # to 0 -- or deleting the age filter -- changes the verdict here. + "a just-created run is below the age gate", + {queued: watchdog_queued_runs(watchdog_run(1, created_at=FRESH_TIMESTAMP))}, + 0, + ("Queue is clean",), + ("cancelled 1",), + ), + ( + "the excluded release workflow is never cancelled", + { + queued: watchdog_queued_runs(watchdog_run(1, workflow="release.yml")), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + }, + 0, + ("workflow is excluded",), + ("cancelled 1",), + ), + ( + # Pins `base="${wf##*/}"`. `release.yml` is in the workflow's OWN + # default list, so it stays excluded even with the strip removed -- + # only a workflow excluded SOLELY by a prefixed repo-variable entry + # can prove the normalization runs. + "a repo-variable exclusion given with its directory prefix still matches", + { + queued: watchdog_queued_runs(watchdog_run(1, workflow="ci.yml")), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + }, + 0, + ("workflow is excluded",), + ("cancelled 1",), + {"extra_env": {"EXTRA_EXCLUDED_WORKFLOWS": ".github/workflows/ci.yml"}}, + ), + ) + + + # The exclusion list is built from a repo variable, so an operator entry + # ending in `/` strips to an EMPTY associative-array subscript -- a hard + # bash error that kills the audit before it reads anything, every five + # minutes. Guarding only the read site left this reachable. + code, text = run_watchdog( + script, + environment_literals, + {queued: (0, {"workflow_runs": []})}, + extra_env={"EXTRA_EXCLUDED_WORKFLOWS": "some/dir/ release.yml"}, + ) + require( + code == 0, + f"watchdog exclusion-list normalization: a trailing-slash entry aborted the audit " + f"(exit {code})\n{text}", + ) + require( + "bad array subscript" not in text, + "watchdog exclusion-list normalization: empty subscript reached the array\n" + text, + ) + + # RESTORED. The fold that turned standalone cases into table rows deleted a + # span that included this block and the summary-channel one below, and + # nothing failed -- they only fail when the WORKFLOW is mutated, so the suite + # stayed green while silently losing the coverage. That is the exact class + # this file exists to catch, arriving via a cleanup. (Cursor Bugbot.) + # + # The EXIT trap. Without it, an abort the script does not anticipate exits + # red with a COMPLETELY EMPTY step summary and no annotation -- the worst + # signal for a job that runs 288 times a day. Forced through a failing + # `date`, which is unguarded on purpose so this stays reachable. + code, text = run_watchdog(script, environment_literals, {}, break_date=True) + require(code != 0, f"watchdog unexpected abort: expected a non-zero exit, got {code}\n{text}") + for needle in ("Watchdog summary", "aborted unexpectedly", "::error::"): + require( + needle in text, + f"watchdog unexpected abort: the EXIT trap did not emit {needle!r}\n{text}", + ) + + # The step summary is the operator-facing artifact, and nothing else proves + # anything reached it: `log_summary` tees to stdout, so every other needle is + # satisfiable from the job log alone. These assert the SUMMARY channel. + code, text = run_watchdog( + script, + environment_literals, + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "offline", False, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs(self_hosted), + }, + ) + require(code == 0, f"watchdog summary channel: expected exit 0, got {code}\n{text}") + for needle in ( + "## Watchdog summary", + "### Healthy queued", + "### Stuck (auto-cancelled)", + "### Starved", + "### Stuck but excluded", + "Queued runs older than", + "Runner groups visible to", + "registered but offline", + ): + require( + needle in text.summary, + f"watchdog summary channel: {needle!r} never reached GITHUB_STEP_SUMMARY " + f"(it may only be in the job log)\n{text.summary}", + ) + + # RESTORED: deleted by the table fold, and the most dangerous of the six. + # State is pushed immediately after EACH successful cancel, not only in the + # final sync. Deleting `state_dirty=1` makes `persist_state_changes` return + # early on every call, so NO cancel count is ever committed: every cycle + # reads `cancels=0` and cancels the same run again, without limit, against + # live Unity runs holding licence seats. The git stub echoes "pushed" per + # push, so the count separates bootstrap + one push per cancel (3) from + # bootstrap + a single final sync (2). + two_stuck = { + queued: watchdog_queued_runs(watchdog_run(1), watchdog_run(2)), + inventory: one_group, + "runner-groups/7/runners": watchdog_runners(("ELI", "online", False, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs(self_hosted), + "actions/runs/2/jobs": watchdog_jobs(self_hosted), + "actions/workflows/42": (0, {"path": ".github/workflows/perf-numbers.yml"}), + "contents/.github/workflows/perf-numbers.yml": (0, {"content": ""}), + } + code, text = run_watchdog(script, environment_literals, two_stuck) + require(code == 0, f"watchdog per-cancel persist: expected exit 0, got {code}\n{text}") + require( + text.count("pushed") >= 3, + "watchdog per-cancel persist: the cap was not pushed after each cancel " + f"(saw {text.count('pushed')} pushes, expected at least 3)\n{text}", + ) + + # The summary is emitted exactly once. A second copy on the normal path + # would mean `finish` and the trap both fired. + code, text = run_watchdog( + script, environment_literals, {queued: (0, {"workflow_runs": []})} + ) + require(code == 0, f"watchdog emit-once: expected exit 0, got {code}\n{text}") + require( + text.count("### Stuck (auto-cancelled)") == 1, + "watchdog emit-once: the step summary was written more than once\n" + text, + ) + + # The runner groups actually counted are named in the summary. That log line + # is the only way a second, restricted group ever becomes visible, which is + # what the visibility comment tells a future reader to watch for. + code, text = run_watchdog( + script, + environment_literals, + { + queued: watchdog_queued_runs(watchdog_run(1)), + inventory: (0, {"runner_groups": [{"id": 7, "name": "Default"}]}), + "runner-groups/7/runners": watchdog_runners(("ELI", "online", True, self_hosted)), + "actions/runs/1/jobs": watchdog_jobs(self_hosted), + }, + ) + require( + "Runner groups visible to" in text and "Default" in text, + "watchdog inventory: the counted runner-group names were not reported\n" + text, + ) + + # Rows may carry a 6th element: kwargs for `run_watchdog`. That is what lets + # a case needing a failing push, a pre-populated cap, or a repo variable be a + # ROW rather than yet another hand-rolled run-and-assert block. + for case in cases: + name, routes, expected_code, expected, forbidden = case[:5] + options = case[5] if len(case) > 5 else {} + code, text = run_watchdog(script, environment_literals, routes, **options) + require( + code == expected_code, + f"watchdog {name}: expected exit {expected_code}, got {code}\n{text}", + ) + for needle in expected: + require(needle in text, f"watchdog {name}: summary omitted {needle!r}\n{text}") + for needle in forbidden: + require(needle not in text, f"watchdog {name}: summary leaked {needle!r}\n{text}") + + +MAINTENANCE = Path(".github/workflows/post-merge-maintenance.yml") + + +def run_maintenance_push_loop( + script: str, + root: Path, + failing_check_calls: int, + failing_pushes: int, + failing: str = "issue-template", + advance_before_run: bool = False, + push_fails_without_advance: bool = False, + quiet_other: bool = False, +) -> tuple[int, dict[str, str]]: + """Execute the push loop against a real local remote; return exit + pushed files. + + The loop is the only place in this repository where generator failure, + concurrent-merge retry, and the job's exit status interact, and it has now + produced two defects in review: a `regenerate` that always returned 0 (so an + not-yet-converged generator shipped green) and then a failure flag that never + cleared (so a transient failure reddened a job that had converged). Both are + invisible to a text assertion, so this runs the real thing. + + `failing_pushes` forces the concurrent-merge retry: each failed push also + advances the remote, which is what makes the loop regenerate on a new head. + """ + bin_dir, origin, work, other = (root / n for n in ("bin", "origin.git", "work", "other")) + bin_dir.mkdir() + subprocess.run(["git", "init", "-q", "--bare", str(origin)], check=True) + for clone in (work, other): + subprocess.run( + ["git", "clone", "-q", str(origin), str(clone)], check=True, capture_output=True + ) + for key, value in (("user.email", "t@t"), ("user.name", "t")): + subprocess.run(["git", "-C", str(clone), "config", key, value], check=True) + (work / ".github" / "ISSUE_TEMPLATE").mkdir(parents=True) + for name, body in (("llms.txt", "a"), ("README.md", "b"), (".github/ISSUE_TEMPLATE/bug_report.yml", "c")): + (work / name).write_text(body, encoding="utf-8") + subprocess.run(["git", "-C", str(work), "add", "-A"], check=True) + subprocess.run(["git", "-C", str(work), "commit", "-qm", "seed"], check=True) + subprocess.run(["git", "-C", str(work), "branch", "-M", "master"], check=True) + subprocess.run(["git", "-C", str(work), "push", "-q", "origin", "master"], check=True) + + calls, pushes = root / "check-calls", root / "push-calls" + calls.write_text("0", encoding="utf-8") + pushes.write_text("0", encoding="utf-8") + npm = bin_dir / "npm" + # Whichever generator is under test counts its own calls and fails the first + # `failing_check_calls` of them; the other always converges. Both branches of + # `regenerate` set the failure flag, so both have to be exercised. + # + # `quiet_other` stops the OTHER generator writing anything. Without it every + # regeneration left a pending change, so the loop always exited through the + # commit-and-push branch and the "already current; nothing to commit" exit + # was unreachable -- one of two sites where the regenerate verdict could be + # dropped undetected. + flaky = "check:llms-txt" if failing == "llms" else "check:issue-template-versions" + steady = "check:issue-template-versions" if failing == "llms" else "check:llms-txt" + writes_template = "exit 0" if quiet_other else ( + 'echo "GEN-$(date +%s%N)" > .github/ISSUE_TEMPLATE/bug_report.yml; exit 0' + ) + writes_llms = "exit 0" if (quiet_other and failing != "llms") else ( + 'echo "GEN-$(date +%s%N)" > llms.txt; exit 0' + ) + npm.write_text( + "#!/bin/bash\n" + 'case "$*" in\n' + f' "run update:llms-txt") {writes_llms} ;;\n' + f' "run update:issue-template-versions") {writes_template} ;;\n' + f' "run {steady}") exit 0 ;;\n' + f' "run {flaky}")\n' + f" n=$(cat {calls}); n=$((n+1)); echo $n > {calls}\n" + f' [ "$n" -le {failing_check_calls} ] && exit 1\n' + " exit 0 ;;\n" + "esac\nexit 0\n", + encoding="utf-8", + ) + git = bin_dir / "git" + # A failed push normally ALSO advances the remote, which is what drives the + # concurrent-merge retry. `push_fails_without_advance` withholds that, which + # is the genuinely different case: a push rejected while master stood still + # (branch protection, a bad token) is a real failure with no retry to make, + # and the loop must surface it rather than exit 0 having pushed nothing. + if push_fails_without_advance: + on_push_failure = " exit 1\n" + else: + on_push_failure = ( + f" (cd {other} && /usr/bin/git pull -q --rebase; echo $n > z$n.txt; " + "/usr/bin/git add -A; /usr/bin/git commit -qm concurrent; " + "/usr/bin/git push -q origin master) > /dev/null 2>&1\n" + " exit 1\n" + ) + git.write_text( + "#!/bin/bash\n" + 'for a in "$@"; do\n' + ' if [ "$a" = "push" ]; then\n' + f" n=$(cat {pushes}); n=$((n+1)); echo $n > {pushes}\n" + f' if [ "$n" -le {failing_pushes} ]; then\n' + f"{on_push_failure}" + " fi\n fi\ndone\n" + 'exec /usr/bin/git "$@"\n', + encoding="utf-8", + ) + for stub in (npm, git): + stub.chmod(0o755) + + (work / "llms.txt").write_text("modified", encoding="utf-8") + if advance_before_run: + # Master advances between the generator steps and the commit step. That + # drives the TOP-of-loop regeneration, which the fixture could not reach + # before -- only the bottom-of-loop one after a failed push. It is the + # branch the whole stale-head design exists for: without it the loop + # commits the old head's generated output onto a new head. + # `other` was cloned from the still-empty bare repo, so it has to catch + # up to the seed commit before it can advance master. + subprocess.run( + ["git", "-C", str(other), "fetch", "-q", "origin"], check=True, capture_output=True + ) + subprocess.run( + ["git", "-C", str(other), "checkout", "-q", "-B", "master", "origin/master"], + check=True, + capture_output=True, + ) + (other / "concurrent.txt").write_text("x", encoding="utf-8") + subprocess.run(["git", "-C", str(other), "add", "-A"], check=True) + subprocess.run(["git", "-C", str(other), "commit", "-qm", "advance"], check=True) + subprocess.run(["git", "-C", str(other), "push", "-q", "origin", "master"], check=True) + environment = os.environ.copy() + environment.update( + { + "PATH": f"{bin_dir}{os.pathsep}{environment['PATH']}", + "LLMS_PATHS": "llms.txt README.md", + "ISSUE_TEMPLATE_PATHS": ".github/ISSUE_TEMPLATE/bug_report.yml", + "GH_PUSH_TOKEN": "stub", + "GITHUB_REF_NAME": "master", + } + ) + code = subprocess.run( + ["bash", "-c", script], cwd=work, env=environment, capture_output=True, text=True, check=False + ).returncode + # What actually reached the remote matters as much as the exit code: a + # generator whose own checker rejected its output must never have that + # output pushed. Asserting only the exit code let the revert be deleted. + # Per-file, not concatenated: the generator that SUCCEEDS legitimately writes + # its marker, so a combined blob cannot tell whose output was reverted. + pushed = {} + for path in ("llms.txt", ".github/ISSUE_TEMPLATE/bug_report.yml"): + shown = subprocess.run( + ["git", "-C", str(origin), "show", f"master:{path}"], + capture_output=True, + text=True, + check=False, + ) + pushed[path] = shown.stdout if shown.returncode == 0 else "" + return code, pushed + + +def run_maintenance_gate(script: str, llms: str, issue_template: str) -> int: + """Execute the terminal `Require every generator to have converged` step.""" + environment = os.environ.copy() + environment.update({"LLMS_OUTCOME": llms, "ISSUE_TEMPLATE_OUTCOME": issue_template}) + return subprocess.run( + ["bash", "-c", script], env=environment, capture_output=True, text=True, check=False + ).returncode + + +def validate_post_merge_terminal_gate() -> None: + """Pin the step that decides the job's verdict. + + The commit step deliberately runs after a failed generator, so the job's own + verdict comes from this gate alone -- its comment says "without this a + generator that never converged would be reported green". It had no coverage + at all: neither its condition nor its outcome table was executed or asserted. + """ + source = MAINTENANCE.read_text(encoding="utf-8") + script = run_script( + step_block(job_block(source, "regenerate"), "Require every generator to have converged") + ) + if os.name == "nt": + return + # llms outcome, issue-template outcome, expected exit + for llms, issue_template, expected in ( + ("success", "success", 0), + ("success", "skipped", 0), + ("skipped", "skipped", 0), + ("failure", "success", 1), + ("success", "failure", 1), + ("failure", "failure", 1), + ("cancelled", "success", 1), + ("", "success", 1), + ): + code = run_maintenance_gate(script, llms, issue_template) + require( + code == expected, + f"post-merge terminal gate ({llms!r}, {issue_template!r}): " + f"expected exit {expected}, got {code}", + ) + + +def validate_post_merge_cancellation_policy() -> None: + """No step that writes or pushes may survive a cancellation. + + `always()` includes CANCELLED, and this workflow sets + `cancel-in-progress: true`, so a second push kills the first mid-flight as a + matter of routine. A generator killed mid-write never reaches its own revert, + so any step that keeps generating, probing, or pushing after a cancel can + commit a half-written file to the default branch. The two workflows this one + replaced had no `always()` at all and simply stopped. (Cursor Bugbot, high.) + """ + source = MAINTENANCE.read_text(encoding="utf-8") + require( + "cancel-in-progress: true" in source, + "post-merge cancellation: the concurrency policy changed; re-check whether " + "cancellation is still a routine path before relaxing the guards below", + ) + job = job_block(source, "regenerate") + for step_name in ( + "Regenerate the issue-template version dropdown", + "Check for changes", + "Commit and push changes", + "Require every generator to have converged", + ): + step = step_block(job, step_name) + condition = re.search(r"\n if: (?:>-\n\s+)?(.*?)\n \w", step, re.S) + require(condition is not None, f"post-merge cancellation: {step_name} has no `if:`") + text = condition.group(1) + require( + "!cancelled()" in text, + f"post-merge cancellation: {step_name} must be gated on `!cancelled()`; " + f"got {text!r}", + ) + require( + "always()" not in text, + f"post-merge cancellation: {step_name} uses `always()`, which runs after a " + f"cancel and can push a half-written tree; got {text!r}", + ) + + +def validate_post_merge_push_loop() -> None: + """Pin how the post-merge push loop turns generator outcomes into an exit code.""" + source = MAINTENANCE.read_text(encoding="utf-8") + script = run_script(step_block(job_block(source, "regenerate"), "Commit and push changes")) + if os.name == "nt": + return + # `failing_pushes` is what forces the loop to regenerate on a refreshed head: + # each failed push also advances the remote. With zero failed pushes the loop + # never calls `regenerate` at all, so a generator failure there belongs to the + # step that ran it, not to this loop -- which is why the no-retry row expects 0. + # + # name, failing check calls, failing pushes, which generator, expected exit + # A head that advanced BEFORE the commit step forces the top-of-loop + # regeneration; a generator that fails there must still fail the job. + with tempfile.TemporaryDirectory() as directory: + code, pushed = run_maintenance_push_loop( + script, Path(directory), 99, 0, "issue-template", advance_before_run=True + ) + require( + code == 1, + f"post-merge push loop stale head: a generator failing on the refreshed head " + f"must fail the job, got exit {code}", + ) + with tempfile.TemporaryDirectory() as directory: + code, pushed = run_maintenance_push_loop( + script, Path(directory), 0, 0, "issue-template", advance_before_run=True + ) + require( + code == 0, + f"post-merge push loop stale head: a clean regeneration on the refreshed head " + f"must succeed, got exit {code}", + ) + + # The "already current" early exit must also carry the regenerate verdict. + # A failing llms generator reverts its own file, which removes the only + # pending change, so the loop leaves through that branch -- and it was one of + # two exit sites where `exit "${regenerate_failed}"` could become `exit 0` + # undetected, reinstating the always-green bug 764540b1 claims to have fixed. + with tempfile.TemporaryDirectory() as directory: + code, pushed = run_maintenance_push_loop( + script, Path(directory), 99, 1, "llms", quiet_other=True + ) + require( + code == 1, + f"post-merge push loop nothing-to-commit exit: a failed generator that " + f"leaves no change must still fail the job, got exit {code}", + ) + + # The "No staged changes remain" exit is the second of the two sites where + # the regenerate verdict could be dropped undetected. It is reached when a + # change exists when the loop starts but is gone by staging time, which is + # what a failing generator's revert does on the refreshed head. + with tempfile.TemporaryDirectory() as directory: + code, pushed = run_maintenance_push_loop( + script, + Path(directory), + 99, + 1, + "llms", + quiet_other=True, + advance_before_run=True, + ) + require( + code == 1, + f"post-merge push loop no-staged-changes exit: a failed generator must fail " + f"the job through this branch too, got exit {code}", + ) + + # A push rejected while master stood still is a genuine failure with no + # retry to make -- branch protection, a bad token, a lost credential. The + # loop must surface it. The stub previously always advanced the remote on a + # failed push, so this branch was unreachable and `exit "${push_status}"` + # could be replaced with `exit 0`: a job reporting green having pushed + # nothing at all. + with tempfile.TemporaryDirectory() as directory: + code, pushed = run_maintenance_push_loop( + script, Path(directory), 0, 1, "issue-template", push_fails_without_advance=True + ) + require( + code != 0, + f"post-merge push loop rejected push: a push rejected with master unchanged " + f"must fail the job, got exit {code}", + ) + require( + "GEN-" not in pushed["llms.txt"], + "post-merge push loop rejected push: reported a push that never landed\n" + + repr(pushed["llms.txt"]), + ) + + for name, failing_checks, failing_pushes, failing, expected in ( + ("a clean run exits 0", 0, 0, "issue-template", 0), + ("a straight retry with no generator failure exits 0", 0, 2, "issue-template", 0), + ("an issue-template generator that never converges fails the job", 99, 1, "issue-template", 1), + ("an llms.txt generator that never converges fails the job", 99, 1, "llms", 1), + ("a transient issue-template failure that later converges exits 0", 1, 2, "issue-template", 0), + # No transient-llms row on purpose. llms.txt carries the only change in + # these fixtures, so reverting it on failure leaves nothing to commit and + # the loop exits at the "already current" branch before a second + # regeneration can occur. The failure still propagates (the row above), + # which is the property that matters; a row asserting a path the loop + # cannot reach would only look like coverage. + ): + with tempfile.TemporaryDirectory() as directory: + code, pushed = run_maintenance_push_loop( + script, Path(directory), failing_checks, failing_pushes, failing + ) + require(code == expected, f"post-merge push loop {name}: expected exit {expected}, got {code}") + # A generator whose checker rejected its output must have that output + # REVERTED, so the marker its update step wrote can never reach master. + # Only the FAILING generator's file is checked; the other one converged + # and its marker is expected there. + # Only when the generator NEVER converged (which is why the job fails). + # A transient failure that later succeeds legitimately pushes its output. + if expected == 1: + owned = ( + "llms.txt" if failing == "llms" else ".github/ISSUE_TEMPLATE/bug_report.yml" + ) + require( + "GEN-" not in pushed[owned], + f"post-merge push loop {name}: pushed {owned} from a generator that " + f"never converged\n{pushed[owned]!r}", + ) + + def find_unregistered_unity_automation(files: dict[str, str]) -> list[str]: return sorted( path @@ -1534,10 +3015,9 @@ def validate() -> None: "healthy-existing CI validation must require the central return action's canonical editor leaf", ) for workflow, job_id in LICENSED_LOCK_WINDOWS: - validate_lock_window_timeout_budget( - job_block(workflow.read_text(encoding="utf-8"), job_id), - f"{workflow}:{job_id}", - ) + window = job_block(workflow.read_text(encoding="utf-8"), job_id) + validate_lock_window_timeout_budget(window, f"{workflow}:{job_id}") + validate_cleanup_gate_not_attempted_input(window, f"{workflow}:{job_id}") source = WORKFLOW.read_text(encoding="utf-8") licensed = validate_licensed_workflow_policy(source) @@ -1767,6 +3247,11 @@ def validate() -> None: f"{name}: expected {expected}, got {result.returncode}\n{result.stdout}\n{result.stderr}", ) + validate_stuck_job_watchdog() + validate_post_merge_push_loop() + validate_post_merge_terminal_gate() + validate_post_merge_cancellation_policy() + print("Unity pull-request policy validation passed.")