diff --git a/package.json b/package.json index f8dea92..b7b4b68 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@testmuai/evidence-cli", - "version": "0.1.5", + "version": "0.1.6", "description": "An open, framework-agnostic format for what a test run produced — the .evidence pack, and the library + CLI that validate and seal it.", "license": "Apache-2.0", "author": "TestMu AI (formerly LambdaTest)", diff --git a/src/finalize/index.ts b/src/finalize/index.ts index 097c904..4bb3d49 100644 --- a/src/finalize/index.ts +++ b/src/finalize/index.ts @@ -3,6 +3,7 @@ import * as path from "node:path"; import { createHash, randomBytes } from "node:crypto"; import AdmZip from "adm-zip"; import { parseYaml, parseDoc, stringifyDoc, stringifyYaml } from "../yaml"; +import { retryTransient } from "../fs-retry"; import type { FinalizeResult, Totals, Verdict } from "../contract"; /** One row of the generated run-level failure index (decision 0044). */ @@ -102,10 +103,22 @@ export async function finalize(dir: string, opts: FinalizeOptions): Promise fs.rename(dir, bakPath), sealRetry); // move the live directory aside (atomic) + await retryTransient(() => fs.rename(tmpPath, sealedPath), sealRetry); // install the sealed file (atomic) await fsyncDir(parent); // persist the directory-entry changes (best-effort) - await fs.rm(bakPath, { recursive: true, force: true }); // cleanup (non-critical) + try { + await fs.rm(bakPath, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); + } catch { + // cleanup is non-critical: the pack is sealed; a lingering .bak- aside is + // redundant and sweepIncomplete deletes it on the next startup. + } return { totals, sealedPath }; } @@ -205,16 +218,18 @@ export async function sweepIncomplete(parentDir: string): Promise { const basePath = path.join(parentDir, base); const bakPath = path.join(parentDir, entry); if (await pathExists(basePath)) { - await fs.rm(bakPath, { recursive: true, force: true }); + await fs.rm(bakPath, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); removed.push(entry); } else { - await fs.rename(bakPath, basePath); + await retryTransient(() => fs.rename(bakPath, basePath)); restored.push(base); } } for (const entry of entries) { if (!/\.evidence\.tmp-[0-9a-f]+$/.test(entry)) continue; - await fs.rm(path.join(parentDir, entry), { force: true }); + // retryTransient, not rm's maxRetries: that option is IGNORED without + // recursive:true, and .tmp- is a plain file. + await retryTransient(() => fs.rm(path.join(parentDir, entry), { force: true })); removed.push(entry); } return { restored, removed }; diff --git a/src/finalize/win32-lock.test.ts b/src/finalize/win32-lock.test.ts new file mode 100644 index 0000000..d166a00 --- /dev/null +++ b/src/finalize/win32-lock.test.ts @@ -0,0 +1,137 @@ +// Regression tests for the Windows transient-lock failure: on win32 an +// antivirus/indexer handle anywhere inside the pack tree makes fs.rename of +// the live directory fail with EPERM (and fs.rm of the .bak aside likewise). +// These inject that fault through fs.promises; the seal must ride it out. +import { afterEach, describe, expect, it, vi } from "vitest"; +import { promises as fs } from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { finalize, sweepIncomplete } from "./index"; + +const SRC = path.resolve(__dirname, "../../fixtures/0.1/L0/finalized/running.evidence"); +let work: string | undefined; + +async function stageCopy(): Promise { + const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "evi-win32-")); + const dst = path.join(tmp, "running.evidence"); + await fs.cp(SRC, dst, { recursive: true }); + work = tmp; + return dst; +} + +function eperm(): Error { + const e = new Error("EPERM: operation not permitted") as Error & { code: string }; + e.code = "EPERM"; + return e; +} + +afterEach(async () => { + vi.restoreAllMocks(); + if (work) await fs.rm(work, { recursive: true, force: true }); + work = undefined; +}); + +describe("finalize under transient Windows locks", () => { + it("seals despite a transient EPERM on the dir→bak rename", async () => { + const dir = await stageCopy(); + const realRename = fs.rename.bind(fs); + let failures = 0; + vi.spyOn(fs, "rename").mockImplementation(async (from, to) => { + if (String(to).includes(".bak-") && failures < 2) { + failures++; + throw eperm(); + } + return realRename(from, to); + }); + + const result = await finalize(dir, { endedAt: "2026-06-28T09:00:30Z" }); + + expect(failures).toBe(2); // the fault actually fired + expect(result.sealedPath).toBe(dir); + expect((await fs.stat(dir)).isFile()).toBe(true); + }); + + it("seals despite a transient EPERM on the tmp→sealed rename", async () => { + const dir = await stageCopy(); + const realRename = fs.rename.bind(fs); + let failures = 0; + vi.spyOn(fs, "rename").mockImplementation(async (from, to) => { + if (String(from).includes(".tmp-") && failures < 2) { + failures++; + throw eperm(); + } + return realRename(from, to); + }); + + const result = await finalize(dir, { endedAt: "2026-06-28T09:00:30Z" }); + + expect(failures).toBe(2); + expect((await fs.stat(dir)).isFile()).toBe(true); + expect(result.sealedPath).toBe(dir); + }); + + it("still succeeds when the .bak cleanup rm fails persistently (sweep collects it later)", async () => { + const dir = await stageCopy(); + const realRm = fs.rm.bind(fs); + vi.spyOn(fs, "rm").mockImplementation(async (target, opts) => { + if (String(target).includes(".bak-")) throw eperm(); + return realRm(target, opts); + }); + + const result = await finalize(dir, { endedAt: "2026-06-28T09:00:30Z" }); + + expect(result.sealedPath).toBe(dir); + expect((await fs.stat(dir)).isFile()).toBe(true); + // the aside is left behind for sweepIncomplete — redundant, not corrupt + const siblings = await fs.readdir(path.dirname(dir)); + expect(siblings.some((s) => /\.evidence\.bak-/.test(s))).toBe(true); + }); +}); + +describe("sweepIncomplete under transient Windows locks", () => { + it("removes a stale .tmp despite a transient EPERM on its rm", async () => { + const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "evi-win32-sweep-")); + work = tmp; + await fs.writeFile(path.join(tmp, "run.evidence.tmp-999"), "half-zip"); + + const realRm = fs.rm.bind(fs); + let failures = 0; + vi.spyOn(fs, "rm").mockImplementation(async (target, opts) => { + if (String(target).includes(".tmp-") && failures < 2) { + failures++; + throw eperm(); + } + return realRm(target, opts); + }); + + const res = await sweepIncomplete(tmp); + + expect(failures).toBe(2); + expect(res.removed).toContain("run.evidence.tmp-999"); + expect(await fs.readdir(tmp)).toEqual([]); + }); + + it("restores a .bak despite a transient EPERM on the restore rename", async () => { + const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "evi-win32-sweep-")); + work = tmp; + const bak = path.join(tmp, "run.evidence.bak-abc123"); + await fs.mkdir(path.join(bak, "tests"), { recursive: true }); + await fs.writeFile(path.join(bak, "run.yaml"), 'evidence: "0.1"\n'); + + const realRename = fs.rename.bind(fs); + let failures = 0; + vi.spyOn(fs, "rename").mockImplementation(async (from, to) => { + if (failures < 2) { + failures++; + throw eperm(); + } + return realRename(from, to); + }); + + const res = await sweepIncomplete(tmp); + + expect(failures).toBe(2); + expect(res.restored).toContain("run.evidence"); + expect((await fs.stat(path.join(tmp, "run.evidence"))).isDirectory()).toBe(true); + }); +}); diff --git a/src/fs-retry.test.ts b/src/fs-retry.test.ts new file mode 100644 index 0000000..60d39ef --- /dev/null +++ b/src/fs-retry.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from "vitest"; +import { retryTransient } from "./fs-retry"; + +/** Build an errno-style error the way node:fs raises them. */ +function errnoError(code: string): Error { + const e = new Error(`${code}: operation not permitted`) as Error & { code: string }; + e.code = code; + return e; +} + +describe("retryTransient", () => { + it("returns the result on first success without sleeping", async () => { + let sleeps = 0; + const result = await retryTransient(async () => "ok", { + sleep: async () => { sleeps++; }, + }); + expect(result).toBe("ok"); + expect(sleeps).toBe(0); + }); + + it.each(["EPERM", "EACCES", "EBUSY"])("retries on %s until the operation succeeds", async (code) => { + let attempts = 0; + const result = await retryTransient( + async () => { + attempts++; + if (attempts < 3) throw errnoError(code); + return "sealed"; + }, + { sleep: async () => {} }, + ); + expect(result).toBe("sealed"); + expect(attempts).toBe(3); + }); + + it("rethrows a non-transient error immediately (single attempt)", async () => { + let attempts = 0; + await expect( + retryTransient( + async () => { + attempts++; + throw errnoError("ENOENT"); + }, + { sleep: async () => {} }, + ), + ).rejects.toMatchObject({ code: "ENOENT" }); + expect(attempts).toBe(1); + }); + + it("gives up after `attempts` tries and rethrows the last error", async () => { + let attempts = 0; + await expect( + retryTransient( + async () => { + attempts++; + throw errnoError("EPERM"); + }, + { attempts: 4, sleep: async () => {} }, + ), + ).rejects.toMatchObject({ code: "EPERM" }); + expect(attempts).toBe(4); + }); + + it("backs off linearly, capped at 1s", async () => { + const delays: number[] = []; + let attempts = 0; + await retryTransient( + async () => { + attempts++; + if (attempts < 14) throw errnoError("EBUSY"); + return "ok"; + }, + { attempts: 20, delayMs: 100, sleep: async (ms) => { delays.push(ms); } }, + ); + expect(delays.slice(0, 3)).toEqual([100, 200, 300]); + expect(Math.max(...delays)).toBe(1000); // capped + expect(delays).toHaveLength(13); + }); + + it("rethrows an error without a code immediately", async () => { + let attempts = 0; + await expect( + retryTransient( + async () => { + attempts++; + throw new Error("plain failure"); + }, + { sleep: async () => {} }, + ), + ).rejects.toThrow("plain failure"); + expect(attempts).toBe(1); + }); +}); diff --git a/src/fs-retry.ts b/src/fs-retry.ts new file mode 100644 index 0000000..76e3074 --- /dev/null +++ b/src/fs-retry.ts @@ -0,0 +1,39 @@ +/** + * Bounded retry for filesystem operations that fail with a TRANSIENT lock. + * + * On Windows, renaming a directory (or removing it) fails with + * ERROR_ACCESS_DENIED — surfaced by Node as EPERM/EACCES — while any other + * process holds an open handle to anything inside its tree. Antivirus + * real-time scans and search indexers open freshly-written files for + * milliseconds-to-seconds, which is exactly when the atomic seal renames run. + * POSIX renames never block on open handles, so these codes there mean a real + * permission problem — the retry just delays the same failure by a few + * seconds, which is an acceptable cost for one shared code path. + */ +const TRANSIENT_CODES = new Set(["EPERM", "EACCES", "EBUSY"]); + +export interface RetryOptions { + /** Total tries including the first (default 10). */ + attempts?: number; + /** Base backoff: try n waits min(n * delayMs, 1000) ms (default 100). */ + delayMs?: number; + /** Injectable for tests; defaults to setTimeout. */ + sleep?: (ms: number) => Promise; +} + +const defaultSleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +export async function retryTransient(op: () => Promise, opts: RetryOptions = {}): Promise { + const attempts = opts.attempts ?? 10; + const delayMs = opts.delayMs ?? 100; + const sleep = opts.sleep ?? defaultSleep; + for (let attempt = 1; ; attempt++) { + try { + return await op(); + } catch (e) { + const code = (e as NodeJS.ErrnoException)?.code; + if (!code || !TRANSIENT_CODES.has(code) || attempt >= attempts) throw e; + await sleep(Math.min(attempt * delayMs, 1000)); + } + } +}