[DEV-79] Enable bounded parallel execution across repositories in workspace mode

docs/code/dashboard.md

@@ -55,7 +55,7 @@ The dashboard is backed by a small read-only JSON API you can use directly, for
 | `GET /api/runs`             | Paginated run list (`limit`, `offset`, `status`, `origin`, `taskKey`); `origin=scheduled` is supported |
 | `GET /api/runs/:id`         | One run with its stage timeline                                       |
 | `GET /api/stats?window=30d` | Aggregate stats (`7d`, `30d`, `90d`, or `all`)                        |
-| `GET /api/worker`           | Worker liveness, queue counts, agent PRs, poll cursors                |
+| `GET /api/worker`           | Worker liveness, queue counts, agent PRs, poll cursors, per-repo fleet activity |
 | `GET /api/health`           | Health check                                                          |
 
 ## License

docs/code/workspaces.md

@@ -1,6 +1,6 @@
 ---
 title: "Workspaces (Multi-Repo Fleet)"
-description: "Drive many repositories with one devintern worker: a single workspace.toml, routing rules, and per-task worktrees"
+description: "Drive many repositories with one devintern worker: a single workspace.toml, routing rules, per-task worktrees, and opt-in parallel execution across repos"
 section: "Server Automation"
 order: 1
 dateModified: 2026-08-26
@@ -19,14 +19,18 @@ Workspace mode runs under the same automation license as the rest of the worker:
 - The worker polls your tracker with one fleet-wide query (a detect-then-evaluate loop with one cursor).
 - Each ready task is matched against your routing rules. A task runs only when the rules agree on exactly one repository. The worker never guesses: tasks that match no rule, or rules for different repositories, are skipped and recorded, and are retried only after the task changes again. **A 1-repo workspace needs no routing rules** — N=1 already implies the only checkout (`devintern worker init` starts this way).
 - The worker manages a bare clone of each repository under `~/.devintern/repos/` and runs every task in a fresh, disposable worktree under `~/.devintern/worktrees/`. Your own checkouts are never touched. Worktrees are removed after a successful run, kept for debugging when a run fails, and swept after `worktrees_ttl_days`.
-- All worker state (queue, cursors, agent PR registry, run records, routing skips) lives in one database at `~/.devintern/state/queue.db`.
-- Runs are serialized: one task at a time, with a per-repository lock. One systemd unit (or one terminal) drives the whole fleet.
+- All worker state (queue, cursors, agent PR registry, run records, routing skips, live fleet activity) lives in one database at `~/.devintern/state/queue.db`.
+- Runs are serialized **within each repository** by a per-repo run lock. By default the whole fleet runs one task at a time; you can opt in to running different repositories concurrently (see [Parallel execution](#parallel-execution-across-repositories)).
+- One systemd unit (or one terminal) drives the whole fleet.
 
 ## workspace.toml
 
 ```toml
 [workspace]
 worktrees_ttl_days = 7
+# Opt-in concurrency across repositories (default false):
+parallel_across_repos = false
+max_concurrency = 4
 dashboard = true
 # dashboard_port = 4400
 
@@ -77,6 +81,7 @@ prompt = "Review the frontend and clean up one source of recurring noise."
 - Repo names must be unique and filesystem-safe; they become directory names under `repos/` and `worktrees/`.
 - Rule criteria combine with AND; list values (`components`, `labels`) match when the task carries any of them. Comparisons are case-insensitive. `project` matches the task key prefix for `PROJ-123` style keys (Jira, Linear); trackers with numeric or opaque ids route via labels or components.
 - `[[automations]]` uses the same schema as single-repo `.devintern-code/automations.toml`. An entry must name `repo` when the workspace has more than one repository. See [Worker Daemon → Recurring automations](./worker.md#recurring-automations) for prompt-writing guidance and schedule semantics.
+- `[workspace].parallel_across_repos` must be `true` or `false`; `[workspace].max_concurrency` must be a positive whole number (`1`, `2`, …). Invalid values fail startup with a clear message.
 
 ### How workspace automations differ from single-repo ones
 
@@ -86,6 +91,38 @@ The scheduling is identical; only where the work runs changes:
 - It takes the normal per-repo run lock, so it never mutates a checkout concurrently with a task or PR run.
 - Occurrence task files land under the workspace home (`~/.devintern/automations/<id>/`), next to `repos/`, `worktrees/`, and the central database — not inside the repo worktrees.
 
+## Parallel execution across repositories
+
+By default the fleet executes one task at a time, exactly as it did before this option existed. Setting `parallel_across_repos = true` lets tasks routed to **different** repositories run at the same time:
+
+```toml
+[workspace]
+parallel_across_repos = true
+max_concurrency = 4   # optional; defaults to 4
+```
+
+Semantics, regardless of settings:
+
+- **One run per repository, always.** Work for the same repo is queued and runs FIFO — never overlapping, no matter which source submitted it (task polling, relay events, PR reviews, or @mentions all join the same per-repo lane).
+- **Global limit.** At most `max_concurrency` runs are in flight across the workspace. Extra ready tasks queue and start as slots free up. A cap larger than your repo count is fine — it is simply never filled.
+- **Cross-process safety.** The per-repo lock file under `~/.devintern/locks/` remains the safety boundary between processes. If another process holds a repo's lock, that work is *deferred* and retried automatically (every ~10s) instead of being counted as a failed attempt — its dedupe record is not consumed, so nothing is lost while waiting.
+- **Failure isolation.** A failed run in one repo does not cancel, block, or misreport concurrent runs elsewhere; failures are recorded per task as usual.
+- **Safe shared state.** The central `queue.db` runs in WAL mode with a busy timeout, so concurrent runs read and write history without database-lock errors.
+
+### Graceful shutdown
+
+On `SIGINT`/`SIGTERM` the worker stops acquiring new events, then:
+
+1. Queued (not yet started) tasks are cancelled **with their dedupe marks rolled back**, so the next start picks them up again automatically.
+2. In-flight runs are awaited to completion so their per-repo locks are released cleanly.
+3. Shared database handles are closed and the workspace lock is released.
+
+Press `Ctrl-C` a second time to exit immediately if a run appears hung; an interrupted in-flight run is recovered by the normal incomplete-attempt machinery on the next start.
+
+### Watching the fleet
+
+The dashboard (`devintern worker` serves it by default per `[workspace].dashboard`, or `devintern dashboard`) shows what every configured repo is doing — `idle`, `queued`, or `running`, plus the current task key / PR and the aggregate active/max concurrency. The same data is available from `GET /api/worker`. After a crash, leftover activity rows are marked stale until the next worker start clears them.
+
 ## Creating a workspace
 
 ```bash

packages/code/CHANGELOG.md

@@ -4,11 +4,15 @@
 
 ### Added
 
+- **Parallel execution across workspace repos (opt-in)**: `[workspace].parallel_across_repos = true` lets tasks routed to different repositories run concurrently while work within one repo stays strictly FIFO and serialized — task polling, relay task events, PR review runs, and mention-triggered runs all join the same per-repo lane through one bounded scheduler, so independent acquirers can never overlap in a repo. `[workspace].max_concurrency` (positive integer, default 4) caps total concurrent fleet runs; excess ready tasks queue and start as slots free up. Serial-by-default behavior is unchanged when the key is omitted or false. The per-repo lock file remains the cross-process boundary: contention defers the run with automatic retries instead of consuming its dedupe record, so contended tasks are not lost or miscounted as attempts
+- **Fleet activity in the dashboard and `GET /api/worker`**: the worker persists a per-repo snapshot (`idle` / `queued` / `running`, current task key or PR, start time) plus aggregate active/max concurrency to the central database on every scheduling transition; the dashboard header renders live per-repo chips. Snapshots written by a process that is no longer running are flagged stale (and never presented as live), a graceful shutdown clears them, and databases from older versions degrade gracefully to an empty fleet section
 - **High-signal worker analytics**: a worker emits one anonymous `worker_started` event after its sources start (`polling`, `relay`, `hybrid`, or `scheduled`) and one `worker_task_run` event per terminal task outcome. Worker task subprocesses no longer duplicate the ordinary `cli_run` event; no polling heartbeats, task keys, repository names, prompts, or error text are sent
 - **Workspace worker probes push access at startup**: each configured GitHub HTTPS remote gets a side-effect-free `git push --dry-run` against its bare clone, so an under-scoped token (fine-grained PAT without `Contents: Read and write`, or a `$GITHUB_TOKEN` that silently overrides the keyring login via `gh auth git-credential`) is reported as a clear startup warning instead of burning task pickups on 403 pushes
 
 ### Changed
 
+- **Graceful worker shutdown drains the fleet**: on SIGINT/SIGTERM the worker stops acquiring events, cancels queued-but-unstarted tasks **with their dedupe marks rolled back** (they re-enter on the next start instead of being silently skipped), awaits in-flight runs so every per-repo lock is released cleanly, closes shared SQLite handles, then releases the workspace lock. A second signal exits immediately if a run appears hung; interrupted runs recover through the normal incomplete-attempt machinery
+- **Central state database uses WAL**: `queue.db` connections (webhook queue, worker state, run records, routing skips, fleet activity) now enable WAL journaling, a busy timeout, and NORMAL sync so concurrent fleet runs read and write history without `SQLITE_BUSY` failures; `WebhookQueue.removeProcessed` rolls back dedupe marks for accepted-but-cancelled work
 - **`devintern worker init` is the complete unattended setup**: the wizard reuses tracker config from `devintern init` (or runs that subset), imports the current repo into `~/.devintern/workspace.toml` as a 1-repo workspace, dry-runs the ready-tasks query into `[defaults].task_query`, checks any automation license, offers zero-port relay pairing, and generates a user-level systemd unit or macOS launchd agent. It no longer asks about `--listen` or writes worker env vars
 - **Worker dashboard is on by default**: `devintern worker` serves localhost:4400 unless `[workspace].dashboard = false`; dashboard startup failures no longer stop task processing
 - **Worker CLI is toml-backed**: query, poll interval, per-task flags, dashboard on/off, and dashboard port live in `workspace.toml`. `--query`, `--interval`, `--ui` / `--no-ui`, `--ui-port`, `--sandbox`, and the `WORKER_TASK_QUERY` / `WORKER_TASK_ARGS` / `WORKER_POLL_INTERVAL` env vars are removed from `devintern worker`. `--help` no longer lists operational env vars

packages/code/src/lib/dashboard-api.ts

@@ -17,6 +17,8 @@ import type { RunOrigin, RunRecord, RunStageRecord, RunStats, RunStatus } from "
 import { resolveQueueDbPath, WebhookQueue } from "./webhook-queue";
 import { WorkerState } from "./worker-state";
 import type { Cursor } from "./worker-state";
+import { FleetActivityStore } from "./workspace/state";
+import type { FleetActivityReport } from "./workspace/state";
 
 const RUN_STATUSES: RunStatus[] = [
   "in_progress",
@@ -56,6 +58,7 @@ interface Stores {
   runs: RunStore;
   state: WorkerState;
   queue: WebhookQueue;
+  fleet: FleetActivityStore;
 }
 
 /**
@@ -85,6 +88,7 @@ export class DashboardData {
         runs: new RunStore(this.dbPath, { readonly: true }),
         state: new WorkerState(this.dbPath, { readonly: true }),
         queue: new WebhookQueue({ dbPath: this.dbPath, readonly: true }),
+        fleet: new FleetActivityStore(this.dbPath, { readonly: true }),
       };
       return this.stores;
     } catch {
@@ -150,12 +154,21 @@ export class DashboardData {
     return this.read([], (stores) => stores.state.listCursors());
   }
 
+  /**
+   * Latest per-repo fleet activity snapshot, or null when the worker never
+   * reported (or the database predates the activity table).
+   */
+  getFleetActivity(): FleetActivityReport | null {
+    return this.read(null, (stores) => stores.fleet.latest());
+  }
+
   /** Close the underlying SQLite connections (tests, shutdown). */
   close(): void {
     if (this.stores) {
       this.stores.runs.close();
       this.stores.state.close();
       this.stores.queue.close();
+      this.stores.fleet.close();
       this.stores = null;
     }
   }
@@ -236,12 +249,32 @@ export function handleStats(data: DashboardData, params: URLSearchParams): ApiRe
 }
 
 /**
- * `GET /api/worker` — worker liveness, queue counts, agent PRs, poll cursors.
+ * Read the worker daemon's liveness lock, checking both layouts: the
+ * workspace (fleet) daemon locks its workspace home directly
+ * (`<workspace>/.worker.lock`), while a single-repo daemon nests it under
+ * `.devintern-code/`.
+ *
+ * @param workingDir - Directory to inspect (workspace home or project root)
+ */
+function readWorkerLock(workingDir: string): ReturnType<typeof LockManager.readLockStatus> {
+  return (
+    LockManager.readLockStatus(workingDir, WORKER_LOCK_FILE, { plainDir: true }) ??
+    LockManager.readLockStatus(workingDir, WORKER_LOCK_FILE)
+  );
+}
+
+/**
+ * `GET /api/worker` — worker liveness, queue counts, agent PRs, poll cursors,
+ * and per-repo fleet activity.
+ *
+ * The fleet section is null when this database has no workspace activity
+ * (single-repo mode, standalone dashboard on an old database).
  *
  * @param data - Dashboard data source
  */
 export function handleWorkerStatus(data: DashboardData): ApiResponse {
-  const lock = LockManager.readLockStatus(data.workingDir, WORKER_LOCK_FILE);
+  const lock = readWorkerLock(data.workingDir);
+  const fleet = data.getFleetActivity();
   return {
     status: 200,
     body: {
@@ -254,8 +287,29 @@ export function handleWorkerStatus(data: DashboardData): ApiResponse {
         cursorValue: cursor.cursorValue,
         updatedAt: cursor.updatedAt,
       })),
+      fleet: fleet === null ? null : formatFleetActivity(fleet),
       dbPath: data.dbPath,
       dbMissing: data.dbMissing,
     },
   };
 }
+
+/** Shape the fleet snapshot for the API response. */
+function formatFleetActivity(fleet: FleetActivityReport) {
+  return {
+    parallel: fleet.parallel,
+    maxConcurrency: fleet.maxConcurrency,
+    // A crashed writer's rows stay until the next start clears them; never
+    // present dead work as live concurrency.
+    active: fleet.stale ? 0 : fleet.rows.filter((row) => row.status === "running").length,
+    stale: fleet.stale,
+    pid: fleet.pid,
+    updatedAt: fleet.updatedAt,
+    repos: fleet.rows.map((row) => ({
+      repo: row.repo,
+      status: row.stale && row.status !== "idle" ? ("stale" as const) : row.status,
+      label: row.label,
+      startedAt: row.startedAt,
+    })),
+  };
+}

packages/code/src/lib/lock-manager.ts

@@ -124,15 +124,22 @@ export class LockManager {
   /**
    * Read a lock file's status without acquiring or touching it.
    *
-   * @param workingDir - Project root used to locate the lock file
+   * @param workingDir - Directory used to locate the lock file
    * @param lockFileName - Lock file name (e.g. `.worker.lock`)
+   * @param options - `plainDir` reads the lock from `workingDir` itself
+   *                  instead of its `.devintern-code/` subdirectory
+   *                  (workspace locks live directly in `~/.devintern/`)
    * @returns Lock status, or `null` when no lock file exists (or it is unreadable)
    */
   static readLockStatus(
     workingDir: string = process.cwd(),
     lockFileName = ".pid.lock",
+    options: { plainDir?: boolean } = {},
   ): LockStatus | null {
-    const lockFilePath = join(resolve(workingDir, ".devintern-code"), lockFileName);
+    const configDir = options.plainDir
+      ? resolve(workingDir)
+      : resolve(workingDir, ".devintern-code");
+    const lockFilePath = join(configDir, lockFileName);
     if (!existsSync(lockFilePath)) {
       return null;
     }

packages/code/src/lib/run-recorder.ts

@@ -13,7 +13,11 @@
  */
 
 import { Database } from "bun:sqlite";
-import { prepareQueueDbDirectory, resolveQueueDbPath } from "./webhook-queue";
+import {
+  applySqliteConcurrencyPragmas,
+  prepareQueueDbDirectory,
+  resolveQueueDbPath,
+} from "./webhook-queue";
 
 export type RunOrigin = "task" | "pr_mention" | "conflict_resolution" | "scheduled";
 
@@ -157,15 +161,16 @@ export class RunStore {
   constructor(dbPath: string = resolveQueueDbPath(), options: { readonly?: boolean } = {}) {
     if (options.readonly) {
       this.db = new Database(dbPath, { readonly: true });
-      this.db.run("PRAGMA busy_timeout = 5000");
+      applySqliteConcurrencyPragmas(this.db);
       return;
     }
 
     prepareQueueDbDirectory(dbPath);
 
     this.db = new Database(dbPath);
-    // The webhook queue / worker state may hold connections to the same file.
-    this.db.run("PRAGMA busy_timeout = 5000");
+    // The webhook queue / worker state may hold connections to the same file;
+    // parallel fleet runs record stages concurrently.
+    applySqliteConcurrencyPragmas(this.db);
     this.initializeSchema();
   }