fix(finalize): retry transient Windows locks around the atomic seal — v0.1.6

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@testmuai/evidence-cli",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "description": "An open, framework-agnostic format for what a test run produced — the .evidence pack, and the library + CLI that validate and seal it.",
   "license": "Apache-2.0",
   "author": "TestMu AI (formerly LambdaTest)",

src/finalize/index.ts

@@ -3,6 +3,7 @@ import * as path from "node:path";
 import { createHash, randomBytes } from "node:crypto";
 import AdmZip from "adm-zip";
 import { parseYaml, parseDoc, stringifyDoc, stringifyYaml } from "../yaml";
+import { retryTransient } from "../fs-retry";
 import type { FinalizeResult, Totals, Verdict } from "../contract";
 
 /** One row of the generated run-level failure index (decision 0044). */
@@ -102,10 +103,22 @@ export async function finalize(dir: string, opts: FinalizeOptions): Promise<Fina
   } finally {
     await fh.close();
   }
-  await fs.rename(dir, bakPath); // move the live directory aside (atomic)
-  await fs.rename(tmpPath, sealedPath); // install the sealed file (atomic)
+  // Both renames retry on transient Windows locks (AV/indexer handles inside
+  // the tree make directory renames EPERM there — see fs-retry.ts). Budget is
+  // deliberately generous (~15s): the dir aside blocks on a handle to ANYTHING
+  // in the tree, and the fresh zip is a prime real-time-scan target; a
+  // permanently stuck handle fails either way, so the wait only costs time on
+  // the already-failing path (graceful-fs's precedent for this is 60s).
+  const sealRetry = { attempts: 20 };
+  await retryTransient(() => fs.rename(dir, bakPath), sealRetry); // move the live directory aside (atomic)
+  await retryTransient(() => fs.rename(tmpPath, sealedPath), sealRetry); // install the sealed file (atomic)
   await fsyncDir(parent); // persist the directory-entry changes (best-effort)
-  await fs.rm(bakPath, { recursive: true, force: true }); // cleanup (non-critical)
+  try {
+    await fs.rm(bakPath, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 });
+  } catch {
+    // cleanup is non-critical: the pack is sealed; a lingering .bak- aside is
+    // redundant and sweepIncomplete deletes it on the next startup.
+  }
 
   return { totals, sealedPath };
 }
@@ -205,16 +218,18 @@ export async function sweepIncomplete(parentDir: string): Promise<SweepResult> {
     const basePath = path.join(parentDir, base);
     const bakPath = path.join(parentDir, entry);
     if (await pathExists(basePath)) {
-      await fs.rm(bakPath, { recursive: true, force: true });
+      await fs.rm(bakPath, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 });
       removed.push(entry);
     } else {
-      await fs.rename(bakPath, basePath);
+      await retryTransient(() => fs.rename(bakPath, basePath));
       restored.push(base);
     }
   }
   for (const entry of entries) {
     if (!/\.evidence\.tmp-[0-9a-f]+$/.test(entry)) continue;
-    await fs.rm(path.join(parentDir, entry), { force: true });
+    // retryTransient, not rm's maxRetries: that option is IGNORED without
+    // recursive:true, and .tmp- is a plain file.
+    await retryTransient(() => fs.rm(path.join(parentDir, entry), { force: true }));
     removed.push(entry);
   }
   return { restored, removed };

src/finalize/win32-lock.test.ts

@@ -0,0 +1,137 @@
+// Regression tests for the Windows transient-lock failure: on win32 an
+// antivirus/indexer handle anywhere inside the pack tree makes fs.rename of
+// the live directory fail with EPERM (and fs.rm of the .bak aside likewise).
+// These inject that fault through fs.promises; the seal must ride it out.
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { promises as fs } from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { finalize, sweepIncomplete } from "./index";
+
+const SRC = path.resolve(__dirname, "../../fixtures/0.1/L0/finalized/running.evidence");
+let work: string | undefined;
+
+async function stageCopy(): Promise<string> {
+  const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "evi-win32-"));
+  const dst = path.join(tmp, "running.evidence");
+  await fs.cp(SRC, dst, { recursive: true });
+  work = tmp;
+  return dst;
+}
+
+function eperm(): Error {
+  const e = new Error("EPERM: operation not permitted") as Error & { code: string };
+  e.code = "EPERM";
+  return e;
+}
+
+afterEach(async () => {
+  vi.restoreAllMocks();
+  if (work) await fs.rm(work, { recursive: true, force: true });
+  work = undefined;
+});
+
+describe("finalize under transient Windows locks", () => {
+  it("seals despite a transient EPERM on the dir→bak rename", async () => {
+    const dir = await stageCopy();
+    const realRename = fs.rename.bind(fs);
+    let failures = 0;
+    vi.spyOn(fs, "rename").mockImplementation(async (from, to) => {
+      if (String(to).includes(".bak-") && failures < 2) {
+        failures++;
+        throw eperm();
+      }
+      return realRename(from, to);
+    });
+
+    const result = await finalize(dir, { endedAt: "2026-06-28T09:00:30Z" });
+
+    expect(failures).toBe(2); // the fault actually fired
+    expect(result.sealedPath).toBe(dir);
+    expect((await fs.stat(dir)).isFile()).toBe(true);
+  });
+
+  it("seals despite a transient EPERM on the tmp→sealed rename", async () => {
+    const dir = await stageCopy();
+    const realRename = fs.rename.bind(fs);
+    let failures = 0;
+    vi.spyOn(fs, "rename").mockImplementation(async (from, to) => {
+      if (String(from).includes(".tmp-") && failures < 2) {
+        failures++;
+        throw eperm();
+      }
+      return realRename(from, to);
+    });
+
+    const result = await finalize(dir, { endedAt: "2026-06-28T09:00:30Z" });
+
+    expect(failures).toBe(2);
+    expect((await fs.stat(dir)).isFile()).toBe(true);
+    expect(result.sealedPath).toBe(dir);
+  });
+
+  it("still succeeds when the .bak cleanup rm fails persistently (sweep collects it later)", async () => {
+    const dir = await stageCopy();
+    const realRm = fs.rm.bind(fs);
+    vi.spyOn(fs, "rm").mockImplementation(async (target, opts) => {
+      if (String(target).includes(".bak-")) throw eperm();
+      return realRm(target, opts);
+    });
+
+    const result = await finalize(dir, { endedAt: "2026-06-28T09:00:30Z" });
+
+    expect(result.sealedPath).toBe(dir);
+    expect((await fs.stat(dir)).isFile()).toBe(true);
+    // the aside is left behind for sweepIncomplete — redundant, not corrupt
+    const siblings = await fs.readdir(path.dirname(dir));
+    expect(siblings.some((s) => /\.evidence\.bak-/.test(s))).toBe(true);
+  });
+});
+
+describe("sweepIncomplete under transient Windows locks", () => {
+  it("removes a stale .tmp despite a transient EPERM on its rm", async () => {
+    const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "evi-win32-sweep-"));
+    work = tmp;
+    await fs.writeFile(path.join(tmp, "run.evidence.tmp-999"), "half-zip");
+
+    const realRm = fs.rm.bind(fs);
+    let failures = 0;
+    vi.spyOn(fs, "rm").mockImplementation(async (target, opts) => {
+      if (String(target).includes(".tmp-") && failures < 2) {
+        failures++;
+        throw eperm();
+      }
+      return realRm(target, opts);
+    });
+
+    const res = await sweepIncomplete(tmp);
+
+    expect(failures).toBe(2);
+    expect(res.removed).toContain("run.evidence.tmp-999");
+    expect(await fs.readdir(tmp)).toEqual([]);
+  });
+
+  it("restores a .bak despite a transient EPERM on the restore rename", async () => {
+    const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "evi-win32-sweep-"));
+    work = tmp;
+    const bak = path.join(tmp, "run.evidence.bak-abc123");
+    await fs.mkdir(path.join(bak, "tests"), { recursive: true });
+    await fs.writeFile(path.join(bak, "run.yaml"), 'evidence: "0.1"\n');
+
+    const realRename = fs.rename.bind(fs);
+    let failures = 0;
+    vi.spyOn(fs, "rename").mockImplementation(async (from, to) => {
+      if (failures < 2) {
+        failures++;
+        throw eperm();
+      }
+      return realRename(from, to);
+    });
+
+    const res = await sweepIncomplete(tmp);
+
+    expect(failures).toBe(2);
+    expect(res.restored).toContain("run.evidence");
+    expect((await fs.stat(path.join(tmp, "run.evidence"))).isDirectory()).toBe(true);
+  });
+});

src/fs-retry.test.ts

@@ -0,0 +1,92 @@
+import { describe, expect, it } from "vitest";
+import { retryTransient } from "./fs-retry";
+
+/** Build an errno-style error the way node:fs raises them. */
+function errnoError(code: string): Error {
+  const e = new Error(`${code}: operation not permitted`) as Error & { code: string };
+  e.code = code;
+  return e;
+}
+
+describe("retryTransient", () => {
+  it("returns the result on first success without sleeping", async () => {
+    let sleeps = 0;
+    const result = await retryTransient(async () => "ok", {
+      sleep: async () => { sleeps++; },
+    });
+    expect(result).toBe("ok");
+    expect(sleeps).toBe(0);
+  });
+
+  it.each(["EPERM", "EACCES", "EBUSY"])("retries on %s until the operation succeeds", async (code) => {
+    let attempts = 0;
+    const result = await retryTransient(
+      async () => {
+        attempts++;
+        if (attempts < 3) throw errnoError(code);
+        return "sealed";
+      },
+      { sleep: async () => {} },
+    );
+    expect(result).toBe("sealed");
+    expect(attempts).toBe(3);
+  });
+
+  it("rethrows a non-transient error immediately (single attempt)", async () => {
+    let attempts = 0;
+    await expect(
+      retryTransient(
+        async () => {
+          attempts++;
+          throw errnoError("ENOENT");
+        },
+        { sleep: async () => {} },
+      ),
+    ).rejects.toMatchObject({ code: "ENOENT" });
+    expect(attempts).toBe(1);
+  });
+
+  it("gives up after `attempts` tries and rethrows the last error", async () => {
+    let attempts = 0;
+    await expect(
+      retryTransient(
+        async () => {
+          attempts++;
+          throw errnoError("EPERM");
+        },
+        { attempts: 4, sleep: async () => {} },
+      ),
+    ).rejects.toMatchObject({ code: "EPERM" });
+    expect(attempts).toBe(4);
+  });
+
+  it("backs off linearly, capped at 1s", async () => {
+    const delays: number[] = [];
+    let attempts = 0;
+    await retryTransient(
+      async () => {
+        attempts++;
+        if (attempts < 14) throw errnoError("EBUSY");
+        return "ok";
+      },
+      { attempts: 20, delayMs: 100, sleep: async (ms) => { delays.push(ms); } },
+    );
+    expect(delays.slice(0, 3)).toEqual([100, 200, 300]);
+    expect(Math.max(...delays)).toBe(1000); // capped
+    expect(delays).toHaveLength(13);
+  });
+
+  it("rethrows an error without a code immediately", async () => {
+    let attempts = 0;
+    await expect(
+      retryTransient(
+        async () => {
+          attempts++;
+          throw new Error("plain failure");
+        },
+        { sleep: async () => {} },
+      ),
+    ).rejects.toThrow("plain failure");
+    expect(attempts).toBe(1);
+  });
+});

src/fs-retry.ts

@@ -0,0 +1,39 @@
+/**
+ * Bounded retry for filesystem operations that fail with a TRANSIENT lock.
+ *
+ * On Windows, renaming a directory (or removing it) fails with
+ * ERROR_ACCESS_DENIED — surfaced by Node as EPERM/EACCES — while any other
+ * process holds an open handle to anything inside its tree. Antivirus
+ * real-time scans and search indexers open freshly-written files for
+ * milliseconds-to-seconds, which is exactly when the atomic seal renames run.
+ * POSIX renames never block on open handles, so these codes there mean a real
+ * permission problem — the retry just delays the same failure by a few
+ * seconds, which is an acceptable cost for one shared code path.
+ */
+const TRANSIENT_CODES = new Set(["EPERM", "EACCES", "EBUSY"]);
+
+export interface RetryOptions {
+  /** Total tries including the first (default 10). */
+  attempts?: number;
+  /** Base backoff: try n waits min(n * delayMs, 1000) ms (default 100). */
+  delayMs?: number;
+  /** Injectable for tests; defaults to setTimeout. */
+  sleep?: (ms: number) => Promise<void>;
+}
+
+const defaultSleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
+
+export async function retryTransient<T>(op: () => Promise<T>, opts: RetryOptions = {}): Promise<T> {
+  const attempts = opts.attempts ?? 10;
+  const delayMs = opts.delayMs ?? 100;
+  const sleep = opts.sleep ?? defaultSleep;
+  for (let attempt = 1; ; attempt++) {
+    try {
+      return await op();
+    } catch (e) {
+      const code = (e as NodeJS.ErrnoException)?.code;
+      if (!code || !TRANSIENT_CODES.has(code) || attempt >= attempts) throw e;
+      await sleep(Math.min(attempt * delayMs, 1000));
+    }
+  }
+}