fix(engine): the API's daily run sweep deletes only finished runs, in batches, each through the run delete with an audit row

cli/cmd/mcp_tools_runs.go

@@ -30,7 +30,7 @@ import (
 // Every tool goes through addReadTool and reads only with GET. get_run and
 // assess_run read the run with GET /api/tests/{id}, which makes the backend
 // poll a run that is still active and save what it finds
-// (TestResource.java:460-466, TestOrchestrator.refreshStatus); both
+// (TestResource.java:458-464, TestOrchestrator.refreshStatus); both
 // descriptions say so. Third-party text (task errors, scenario and phase
 // names, labels, plan names, audit details, backend messages) is fenced;
 // ids, types and statuses are cleaned so an agent can pass them back.
@@ -1285,7 +1285,7 @@ func mcpRunHasRequested(r *client.MCPRun) bool {
 
 // mcpAssessBaseline finds the baseline run: among the runs the band's scan
 // read, or else with one read of its own. The baseline may be any run, of
-// any type or status (TestResource.java:592-620 checks only that it exists),
+// any type or status (TestResource.java:590-618 checks only that it exists),
 // so reading it polls it if it is still active, as get_run does.
 func mcpAssessBaseline(ctx context.Context, call *mcpCall, baselineID string, band mcpBandScan) *mcpRunIdentity {
 	if b, ok := band.byID[baselineID]; ok {
@@ -1998,10 +1998,10 @@ func mcpActivityAuditSince(ctx context.Context, call *mcpCall, since time.Time,
 	out.Total = pg.Total
 	for _, a := range pg.Items {
 		out.Rows = append(out.Rows, mcpActivityAuditRow{
-			ID:        a.ID,
-			Action:    mcpSanitizeLine(a.Action, 32),
-			EventType: mcpSanitizeLine(a.EventType, 32),
-			Target:    mcpSanitizeLine(a.Target, 128),
+			ID:            a.ID,
+			Action:        mcpSanitizeLine(a.Action, 32),
+			EventType:     mcpSanitizeLine(a.EventType, 32),
+			Target:        mcpSanitizeLine(a.Target, 128),
 			Details:       call.FenceN(a.Details, 200),
 			Actor:         mcpSanitizeLine(a.Actor, 64),
 			PrincipalType: mcpSanitizeLine(a.PrincipalType, 16),
@@ -2269,7 +2269,7 @@ var mcpCaveatsRuns = []mcpCaveat{
 		Refs: []string{
 			mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:2011-2089",
 				`"Cancelled by user"`, `EventKind.FAILED, "cancelled"`, "return run.withResults(updatedResults);"),
-			mcpAnchoredRef(mcpJava+"api/TestResource.java:499-544",
+			mcpAnchoredRef(mcpJava+"api/TestResource.java:497-542",
 				`@Path("/{id}/cancel")`, `auditService.record("CANCEL"`, `"Test cancelled; it is stored as FAILED"`),
 			mcpJava + "domain/TestResult.java:25-31",
 		},
@@ -2353,11 +2353,15 @@ var mcpCaveatsRuns = []mcpCaveat{
 			"principalType says whether a person (human), an agent or a schedule (system, actor system:scheduler) " +
 			"acted. Every REST and gRPC call that changes something leaves a row, a refused or failed one too, " +
 			"with how it ended in details, and its target is the id of the run or disruption it started when it " +
-			"started one. A call without a valid key, and a dry run, leave none. Rows written before the Kates " +
+			"started one. The Kates API's daily retention sweep leaves one as system:scheduler for each finished " +
+			"run it deletes. A call without a valid key, and a dry run, leave none. Rows written before the Kates " +
 			"API recorded actors name no one, and then only the REST test endpoints wrote rows at all. Runs and " +
 			"disruption reports record no owner of their own: who started one shows only in its audit row. The " +
 			"endpoint reads at most the 500 newest matching rows.",
 		Refs: []string{
+			mcpAnchoredRef(mcpJava+"service/RunRetention.java:82",
+				`auditService.record("DELETE", "test", id, auditDetails, actor);`),
+			mcpAnchoredRef(mcpJava+"service/TestCleanupScheduler.java:99", "Actor.SCHEDULER"),
 			mcpAnchoredRef(mcpJava+"audit/AuditResponseFilter.java:42-62",
 				"public void filter(ContainerRequestContext request", "if (actor == null)", "static String idOf("),
 			mcpAnchoredRef(mcpJava+"security/GrpcApiKeyInterceptor.java:67-70,100,156-159",

cli/cmd/mcp_tools_scenario.go

@@ -1242,7 +1242,7 @@ var mcpCaveatsScenario = []mcpCaveat{
 			"and kates test apply sends a scenario with whatever key it holds.",
 		Refs: []string{
 			"plans/mcp-server.md:387-405",
-			mcpJava + "api/TestResource.java:104-131",
+			mcpJava + "api/TestResource.java:108-135",
 			mcpJava + "domain/TestSpec.java:34-113",
 			mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1356-1421",
 				"Map<String, String> inapplicableFields(", "the trogdor backend cannot run a transactional producer"),

docs/book/10-cli-reference.md

@@ -718,7 +718,7 @@ The command first counts the runs, deleting none, and prints how many it found,
 
 The command stops after 100 calls, or after a call that deleted nothing. When runs still match then, it exits 1 and says how many are left; run it again to go on. With `-o json` it prints one answer for the whole command, in the fields of [DELETE /api/tests](11-api-reference.md#delete-apitests). `deleted` adds up every call, `matched` is the count the first delete found and `remaining` the one after the last; with `--dry-run`, or when none match, it prints the count.
 
-Once a day the Kates API deletes, on its own, every run older than 90 days that is not `RUNNING` or `PENDING`, so `prune` is for keeping finished runs for less long. To prune on a schedule, enable the `kates` chart's cleanup CronJob, described in [Where Kates Stores Test Data](12-deployment.md#where-kates-stores-test-data). A Kates API too old to have `DELETE /api/tests` answers `405`, and the command fails, saying to upgrade it.
+Once a day the Kates API prunes by itself, the same way: it deletes the `DONE` and `FAILED` runs created more than 90 days ago (`kates.cleanup.retention-days`), each with an audit row. It leaves `PENDING`, `RUNNING` and `STOPPING` runs alone, however old they are. So `prune` is for keeping finished runs for less long. To prune on a schedule, enable the `kates` chart's cleanup CronJob, described in [Where Kates Stores Test Data](12-deployment.md#where-kates-stores-test-data). A Kates API too old to have `DELETE /api/tests` answers `405`, and the command fails, saying to upgrade it.
 
 #### test watch
 

docs/book/11-api-reference.md

@@ -373,7 +373,9 @@ Stop and delete a test run with its results. A run that is still `PENDING` or `R
 
 Delete the finished runs created before a cutoff, to keep the run history to a retention period of your own. Only `DONE` and `FAILED` runs match, and a cancelled run is stored as `FAILED`, so it matches too. A run still `PENDING`, `RUNNING` or `STOPPING` is never deleted here; [test cleanup](10-cli-reference.md#test-cleanup) deals with one left `RUNNING`. `kates test prune` and the `kates` chart's cleanup CronJob call this endpoint.
 
-It needs a key with the `admin` scope, as the other deletes do. A call deletes the oldest matching runs first, by `createdAt`, and at most `limit` of them. Each goes as `DELETE /api/tests/{id}` deletes it, with its results, and leaves a row in the audit log naming who pruned it: action `DELETE`, type `test`, the run's ID, and the details `retention: created before <cutoff>`. A dry run leaves none, and a call that deletes nothing, or is refused, leaves one row with its status. `kates audit --type test` lists them. Once a day the Kates API also deletes, on its own, every run older than 90 days (`kates.cleanup.retention-days`) that is not `RUNNING` or `PENDING`, without an audit row; this endpoint is for keeping finished runs for less long.
+It needs a key with the `admin` scope, as the other deletes do. A call deletes the oldest matching runs first, by `createdAt`, and at most `limit` of them. Each goes as `DELETE /api/tests/{id}` deletes it, with its results, and leaves a row in the audit log naming who pruned it: action `DELETE`, type `test`, the run's ID, and the details `retention: created before <cutoff>`. A dry run leaves none, and a call that deletes nothing, or is refused, leaves one row with its status. `kates audit --type test` lists them.
+
+Once a day the Kates API also prunes by itself, through the same delete: it deletes the `DONE` and `FAILED` runs created more than `kates.cleanup.retention-days` days ago, 90 by default, oldest first and 1,000 at a time. Each run it deletes leaves an audit row too, naming `system:scheduler`, with the details `retention sweep: created before <cutoff> (kates.cleanup.retention-days=<n>)`. A run still `PENDING`, `RUNNING` or `STOPPING` is left alone, however old it is. This endpoint is for keeping finished runs for less long.
 
 **Query Parameters:**
 

docs/book/12-deployment.md

@@ -75,7 +75,11 @@ The default deployment uses **NodePort** for all services (Grafana on 30080, Kaf
 
 ### Where Kates Stores Test Data
 
-The Kates API keeps every run in PostgreSQL. The `kates` chart runs its own PostgreSQL on a 1 GiB PersistentVolumeClaim on the cluster's default StorageClass, so runs survive pod restarts; there is no emptyDir mode. For production, point the chart at an external database instead, with `postgresql.enabled: false` and `externalDatabase`, as `values-prod.yaml` does. Either way, the Kates API deletes finished runs older than 90 days by default, and [Architecture & Design](02-architecture.md) describes what else it keeps.
+The Kates API keeps every run in PostgreSQL. The `kates` chart runs its own PostgreSQL on a 1 GiB PersistentVolumeClaim on the cluster's default StorageClass, so runs survive pod restarts; there is no emptyDir mode. For production, point the chart at an external database instead, with `postgresql.enabled: false` and `externalDatabase`, as `values-prod.yaml` does. [Architecture & Design](02-architecture.md) describes what else the Kates API keeps.
+
+Whichever database holds them, each replica of the Kates API sweeps out old runs when it starts and every 24 hours after. A sweep deletes the `DONE` and `FAILED` runs created more than `kates.cleanup.retention-days` days ago, 90 by default, so a lower setting deletes as soon as a replica starts with it. It deletes each run as `DELETE /api/tests/{id}` does, with its results, oldest first and 1,000 at a time. Each leaves a row in the audit log naming `system:scheduler`: action `DELETE`, type `test`, the run's ID, and the details `retention sweep: created before <cutoff> (kates.cleanup.retention-days=<n>)`. When two replicas find the same run at once, it is deleted, and audited, once. An error ends that sweep with an `ERROR` line in the Kates API's log, and the next sweep goes on from there.
+
+A sweep leaves a run still `RUNNING` alone, however old it is, and the first sweep after the run ends deletes it. The Kates API fails a run still `RUNNING` five minutes after the time it was set to last. A run left `PENDING` by a submission that died, or `STOPPING` by an older version of Kates, never ends, so no sweep deletes it. Delete such a run with `DELETE /api/tests/{id}`, or `kates test delete`.
 
 To keep runs for less time, set `cleanup.enabled: true` in the `kates` chart's values. Its CronJob then deletes the finished runs older than `cleanup.retentionDays`, through [DELETE /api/tests](11-api-reference.md#delete-apitests), with the key from the Secret the `apiKey` values name. The table gives the values and their defaults:
 
@@ -87,7 +91,7 @@ To keep runs for less time, set `cleanup.enabled: true` in the `kates` chart's v
 | `cleanup.statuses` | `[DONE, FAILED]` | Which finished runs go; a cancelled run is stored as `FAILED` |
 | `cleanup.image` | `curlimages/curl:8.7.1` | The image the job runs `curl` in |
 
-Each time it runs, the job deletes the oldest first, up to 1,000 runs a call, until none older are left or it has made 100 calls, and the next job goes on from there. Its log ends with how many it deleted. A call the Kates API refuses fails the job, with the answer in its log. A `401` or `403` there means the key is missing or wrong, and a `405` that the Kates API is too old to have `DELETE /api/tests`. The Kates API's own daily sweep deletes every run that is not `RUNNING` or `PENDING` at 90 days by default, so a `retentionDays` of 90 or more adds nothing to it. To prune once, by hand, run `kates test prune`.
+Each time it runs, the job deletes the oldest first, up to 1,000 runs a call, until none older are left or it has made 100 calls, and the next job goes on from there. Its log ends with how many it deleted. A call the Kates API refuses fails the job, with the answer in its log. A `401` or `403` there means the key is missing or wrong, and a `405` that the Kates API is too old to have `DELETE /api/tests`. The Kates API's own daily sweep already deletes the `DONE` and `FAILED` runs older than 90 days by default, so a `retentionDays` of 90 or more adds nothing to it. To prune once, by hand, run `kates test prune`.
 
 For Kafka broker storage, **always use persistent volumes** — even in development. Kafka's log retention depends on data being durable, and losing broker data mid-test invalidates results.
 

kates/docs/api-reference.md

@@ -225,7 +225,9 @@ DELETE /api/tests?createdBefore=2026-09-07T00:00:00Z&status=FAILED&dryRun=true
 
 Deletes the finished runs created before a cutoff, to keep the run history to a retention period of your own. Only `DONE` and `FAILED` runs match; a cancelled run is stored as `FAILED`, so it matches too. A run still pending, running or stopping is never deleted here: `kates test cleanup` deals with one left running. `kates test prune` and the chart's cleanup CronJob (`cleanup.*` in `charts/kates/values.yaml`) call this endpoint.
 
-A call deletes the oldest matching runs first, by `createdAt`, and at most `limit` of them. Each goes through the same delete as `DELETE /api/tests/{id}`, and leaves an audit row: action `DELETE`, type `test`, the run's ID, and the details `retention: created before <cutoff>`. Once a day the Kates API also deletes, on its own, every run older than `kates.cleanup.retention-days` (90) that is not `RUNNING` or `PENDING`, without an audit row; this endpoint is for keeping finished runs for less long.
+A call deletes the oldest matching runs first, by `createdAt`, and at most `limit` of them. Each goes through the same delete as `DELETE /api/tests/{id}`, and leaves an audit row: action `DELETE`, type `test`, the run's ID, and the details `retention: created before <cutoff>`.
+
+Once a day the Kates API also prunes by itself, through the same delete: it deletes the `DONE` and `FAILED` runs created more than `kates.cleanup.retention-days` (90) days ago, oldest first and 1,000 at a time. Each leaves an audit row too, naming `system:scheduler`, with the details `retention sweep: created before <cutoff> (kates.cleanup.retention-days=<n>)`. A run still pending, running or stopping is left alone, however old it is. This endpoint is for keeping finished runs for less long.
 
 **Query Parameters:**
 

kates/src/main/java/com/bmscomp/kates/api/TestResource.java

@@ -45,6 +45,7 @@
 import com.bmscomp.kates.security.Scopes;
 import com.bmscomp.kates.service.AuditService;
 import com.bmscomp.kates.service.BaselineService;
+import com.bmscomp.kates.service.RunRetention;
 import com.bmscomp.kates.service.TestRunRepository;
 
 @RolesAllowed(Scopes.READ)
@@ -86,6 +87,9 @@ public class TestResource {
     @Inject
     AuditService auditService;
 
+    @Inject
+    RunRetention retention;
+
     @Inject
     AuditTrail auditTrail;
 
@@ -366,29 +370,23 @@ public Response pruneTests(
         }
 
         List<String> names = statuses.stream().map(Enum::name).toList();
-        long matched = repository.countByStatusCreatedBefore(statuses, before);
         if (countOnly) {
             // A count changes nothing, so it leaves no audit row.
             auditTrail.done();
+            long matched = retention.count(statuses, before);
             return Response.ok(new PruneResponse(before.toString(), names, true, matched, 0, matched))
                     .build();
         }
-        int deleted = 0;
-        for (String id : repository.findIdsByStatusCreatedBefore(statuses, before, max)) {
-            // The single delete, as the bulk one uses. A run another delete
-            // took first is gone, and is not counted.
-            if (orchestrator.deleteTest(id)) {
-                auditService.record("DELETE", "test", id, "retention: created before " + before);
-                deleted++;
-            }
-        }
-        long remaining = repository.countByStatusCreatedBefore(statuses, before);
-        if (deleted > 0) {
+        // The delete TestCleanupScheduler sweeps with, each run through the
+        // single delete, as the bulk one uses.
+        RunRetention.Pass pass = retention.prune(statuses, before, max, "retention: created before " + before);
+        if (pass.deleted() > 0) {
             LOG.infof(
                     "Pruned %d %s run(s) created before %s; %d still match",
-                    deleted, String.join(" or ", names), before, remaining);
+                    pass.deleted(), String.join(" or ", names), before, pass.remaining());
         }
-        return Response.ok(new PruneResponse(before.toString(), names, false, matched, deleted, remaining))
+        return Response.ok(new PruneResponse(
+                        before.toString(), names, false, pass.matched(), pass.deleted(), pass.remaining()))
                 .build();
     }