Skip to content

Commit e979b7b

Browse files
ci: don't let a dead Depot runner cancel a merge-queue run (#1376)
Between 22:15Z and 22:52Z on 2026-10-09, 58 of 65 failed Depot jobs (merge_group, pull_request and push alike, on depot-ubuntu-24.04-4 and -16) died because their runner went away: "Step canceled by GitHub", "The runner has received a shutdown signal", or "lost communication". The step-cancel deaths came 34-91 s (median ~63 s) after the job started, whatever it ran (Checkout, clippy, cargo, unittest, bundler), and the lost-communication ones are GitHub's 10-minute heartbeat timeout on runners that died at about the same point. None of the victim runs had been cancelled through the API first. merge-queue-fail-fast.py treated each death as a failed job and cancelled the whole run, so every dead runner evicted the queue head and threw away its other jobs. It now reads the failed job's check-run annotations and skips jobs whose runner died, leaving the run to finish so ci-ok decides. A job re-run is not possible here: the jobs rerun API refuses until the whole run has completed, by which time ci-ok has reported. Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1 parent 92e001a commit e979b7b

2 files changed

Lines changed: 112 additions & 3 deletions

File tree

‎scripts/merge-queue-fail-fast.py‎

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,13 @@
1313
failed and cancelled `needs`, and reports failure within seconds, and the
1414
queue evicts the entry and rebuilds the ones behind it.
1515
16+
A job whose runner died is the exception (see `runner_lost`): it says
17+
nothing about the PR, and cancelling the run for it turned one dead runner
18+
into an eviction plus a burst of cancelled jobs. Those are left alone, so
19+
`ci-ok` decides when the run finishes. The job can't be re-run instead:
20+
`POST .../actions/jobs/{id}/rerun` is refused until the whole run has
21+
completed, and by then `ci-ok` has already reported.
22+
1623
The script never fails its own job: any API error is a warning and the
1724
watch carries on (or gives up), leaving the run to finish as before.
1825
"""
@@ -31,14 +38,32 @@
3138
# Job conclusions that make `ci-ok` fail. `cancelled` is left out: it
3239
# means someone (the queue, a janitor, a person) is already cancelling.
3340
FATAL = {"failure", "timed_out"}
41+
# Check-run annotations that mean the job's runner went away, not that a
42+
# step failed. On Depot runners these arrive as a job `failure` (its step
43+
# shows `cancelled`) about a minute into the job, whatever the job runs;
44+
# "lost communication" is the same death noticed by GitHub's 10-minute
45+
# heartbeat. Matched case-insensitively against failure-level messages.
46+
RUNNER_LOSS = ("lost communication", "shutdown signal", "step canceled by github")
3447

3548

3649
def fatal_jobs(jobs):
3750
"""Names of the completed jobs whose conclusion dooms `ci-ok`."""
38-
return sorted(
39-
j["name"]
51+
return sorted(j["name"] for j in failed_jobs(jobs))
52+
53+
54+
def failed_jobs(jobs):
55+
return [
56+
j
4057
for j in jobs
4158
if j.get("status") == "completed" and j.get("conclusion") in FATAL and j.get("name") != GATE
59+
]
60+
61+
62+
def runner_lost(annotations):
63+
"""True when a job's failure annotations say its runner died."""
64+
return any(
65+
a.get("annotation_level") == "failure" and any(m in (a.get("message") or "").lower() for m in RUNNER_LOSS)
66+
for a in annotations
4267
)
4368

4469

@@ -74,6 +99,11 @@ def list_jobs(repo, run_id, token):
7499
page += 1
75100

76101

102+
def annotations(repo, job_id, token):
103+
# A job's id is also its check-run id.
104+
return request("GET", f"repos/{repo}/check-runs/{job_id}/annotations?per_page=100", token)
105+
106+
77107
def summary(text):
78108
print(text)
79109
path = os.environ.get("GITHUB_STEP_SUMMARY")
@@ -89,6 +119,7 @@ def main():
89119
interval = int(os.environ.get("POLL_SECONDS", "30"))
90120
deadline = time.monotonic() + int(os.environ.get("WATCH_MINUTES", "100")) * 60
91121
run = None
122+
lost = {} # job id -> runner_lost verdict; a completed job's annotations don't change
92123
while time.monotonic() < deadline:
93124
try:
94125
if run is None:
@@ -100,7 +131,15 @@ def main():
100131
if request("GET", f"repos/{repo}/actions/runs/{run['id']}", token)["status"] == "completed":
101132
print("CI run completed; nothing to do")
102133
return 0
103-
failed = fatal_jobs(list_jobs(repo, run["id"], token))
134+
failed = []
135+
for j in failed_jobs(list_jobs(repo, run["id"], token)):
136+
if j["id"] not in lost:
137+
lost[j["id"]] = runner_lost(annotations(repo, j["id"], token))
138+
if lost[j["id"]]:
139+
summary(f"Not cancelling for `{j['name']}`: its runner died ({j.get('html_url', '')}); `ci-ok` decides.")
140+
if not lost[j["id"]]:
141+
failed.append(j["name"])
142+
failed.sort()
104143
if failed:
105144
request("POST", f"repos/{repo}/actions/runs/{run['id']}/cancel", token)
106145
summary(

‎scripts/tests/test_merge_queue_fail_fast.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
"""Tests for scripts/merge-queue-fail-fast.py."""
22

33
import importlib.util
4+
import io
45
import re
56
import unittest
7+
from contextlib import redirect_stdout
68
from pathlib import Path
9+
from unittest import mock
710

811
ROOT = Path(__file__).parents[2]
912
CI = ROOT / ".github" / "workflows" / "ci.yml"
@@ -37,6 +40,73 @@ def test_cancelled_jobs_and_the_gate_itself_are_ignored(self):
3740
self.assertEqual(ff.fatal_jobs(jobs), [])
3841

3942

43+
def note(message, level="failure"):
44+
return {"annotation_level": level, "message": message}
45+
46+
47+
class RunnerLost(unittest.TestCase):
48+
"""Annotations GitHub attached to jobs whose Depot runner died (2026-10-09)."""
49+
50+
def test_depot_runner_deaths_are_recognised(self):
51+
for message in (
52+
"Step canceled by GitHub (see: https://depot.dev/docs/github-actions/troubleshooting#error-step-canceled-by-github)",
53+
"The runner has received a shutdown signal. This can happen when the runner service is stopped, "
54+
"or a manually started runner is canceled.",
55+
"The self-hosted runner lost communication with the server. Verify the machine is running and has "
56+
"a healthy network connection.",
57+
):
58+
with self.subTest(message=message[:30]):
59+
self.assertTrue(ff.runner_lost([note("Node.js 20 is deprecated.", "warning"), note(message)]))
60+
61+
def test_a_real_failure_is_not_a_runner_loss(self):
62+
self.assertFalse(ff.runner_lost([]))
63+
self.assertFalse(ff.runner_lost([note("Process completed with exit code 101.")]))
64+
65+
def test_only_failure_level_annotations_count(self):
66+
self.assertFalse(ff.runner_lost([note("Step canceled by GitHub", "warning")]))
67+
68+
69+
class Watch(unittest.TestCase):
70+
"""main() against a fake API: which failed jobs make it cancel the run."""
71+
72+
def watch(self, notes_by_job):
73+
jobs = [
74+
{"id": i, "name": f"job{i}", "status": "completed", "conclusion": "failure", "html_url": f"u{i}"}
75+
for i in notes_by_job
76+
]
77+
calls = []
78+
79+
def request(method, path, token):
80+
calls.append((method, path))
81+
if "/annotations" in path:
82+
return notes_by_job[int(path.split("/")[-2])]
83+
if path.endswith("/jobs?filter=latest&per_page=100&page=1"):
84+
return {"jobs": jobs}
85+
if "/workflows/ci.yml/runs" in path:
86+
return {"workflow_runs": [{"id": 7, "html_url": "run"}]}
87+
if path.endswith("/runs/7"):
88+
# Report completion on the second poll so the watch ends.
89+
done = sum(1 for _, p in calls if p.endswith("/runs/7")) > 1
90+
return {"status": "completed" if done else "in_progress"}
91+
return None
92+
93+
env = {"GITHUB_REPOSITORY": "o/r", "HEAD_SHA": "abc", "GH_TOKEN": "t", "POLL_SECONDS": "0"}
94+
with mock.patch.dict(ff.os.environ, env, clear=True), mock.patch.object(ff, "request", request), \
95+
mock.patch.object(ff.time, "sleep"), redirect_stdout(io.StringIO()):
96+
ff.main()
97+
return calls
98+
99+
def test_a_runner_loss_does_not_cancel_the_run(self):
100+
calls = self.watch({1: [note("Step canceled by GitHub (see: https://depot.dev/...)")]})
101+
self.assertNotIn(("POST", "repos/o/r/actions/runs/7/cancel"), calls)
102+
# The verdict is cached: annotations are fetched once across polls.
103+
self.assertEqual(sum("/annotations" in p for _, p in calls), 1)
104+
105+
def test_a_real_failure_still_cancels_the_run(self):
106+
calls = self.watch({1: [note("The runner has received a shutdown signal.")], 2: [note("exit code 1")]})
107+
self.assertIn(("POST", "repos/o/r/actions/runs/7/cancel"), calls)
108+
109+
40110
class CiWorkflowContract(unittest.TestCase):
41111
"""Cancelling on a failed job is safe only while ci.yml keeps these."""
42112

0 commit comments

Comments
 (0)