From 09f04be5484471a070a8d4fd013d94980eb07ff7 Mon Sep 17 00:00:00 2001 From: Thomas Hart Date: Fri, 31 Jul 2026 18:28:50 +0000 Subject: [PATCH] feat: Add self-verification with verified:true only on assertion pass Split post-condition checking into a pure selfVerify phase and tag every RunResult with a literal verified flag so trusted values only appear when the caller-supplied assertion holds. --- README.md | 34 ++++-- src/contract.ts | 44 +++++--- src/index.ts | 11 ++ src/run.ts | 32 ++++-- src/sandbox.ts | 4 +- src/verify.ts | 86 +++++++++++++++ src/worker.ts | 54 +++++++--- test/limits.test.ts | 2 +- test/run.test.ts | 6 +- test/sandbox.test.ts | 6 +- test/verify.test.ts | 244 +++++++++++++++++++++++++++++++++++++++++++ test/worker.test.ts | 8 +- 12 files changed, 473 insertions(+), 58 deletions(-) create mode 100644 src/verify.ts create mode 100644 test/verify.test.ts diff --git a/README.md b/README.md index 2e89e41..dcf7ee5 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,16 @@ Ephemeral, zero-credential, self-verifying execution for untrusted or agent-writ ## What this demonstrates -An airlock is the safe way to run code you do not trust: an LLM-generated snippet, a plugin, a user-submitted function. This repo builds that primitive from the ground up in TypeScript. The guarantee is that a caller never reads an output unless the run stayed inside its resource ceilings and its output satisfies a post-condition the caller supplied. Untrusted code is guilty until proven correct, and the type system makes you prove it before you can touch the value. +An airlock is the safe way to run code you do not trust: an LLM-generated snippet, a plugin, a user-submitted function. This repo builds that primitive from the ground up in TypeScript. The guarantee is that a caller never reads an output as trusted unless the run stayed inside its resource ceilings and its output satisfies a post-condition the caller supplied. Untrusted code is guilty until proven correct, and the type system makes you prove it before you can touch a verified value. -The first slice was the contract and the in-process runner that enforces it. The second slice added `run(code, opts)`, which executes untrusted source in a fresh `node:vm` context with no ambient authority. The third slice added `runInWorker(code, opts)`: the same contract on a `worker_threads` isolate with frozen globals and an empty env. The fourth slice hardens the **resource limit** layer: wall-clock deadline with hard terminate on abort, V8 heap cap, and output size caps. This slice adds a **deny-by-default module loader**: untrusted code has no `require` unless the caller opts in with `allowedModules`, and only the listed builtin or package ids resolve. Relative and absolute paths are always refused. Later slices add a Docker-backed tier and a growing suite of documented escape-attempt tests. +The first slice was the contract and the in-process runner that enforces it. The second slice added `run(code, opts)`, which executes untrusted source in a fresh `node:vm` context with no ambient authority. The third slice added `runInWorker(code, opts)`: the same contract on a `worker_threads` isolate with frozen globals and an empty env. The fourth slice hardens the **resource limit** layer: wall-clock deadline with hard terminate on abort, V8 heap cap, and output size caps. The fifth slice adds a **deny-by-default module loader**: untrusted code has no `require` unless the caller opts in with `allowedModules`. This slice makes **self-verification** explicit: every result carries a literal `verified: true | false`, and `verified: true` is returned only when the caller-supplied assertion passes. Later slices add a Docker-backed tier and a growing suite of documented escape-attempt tests. ## Concepts demonstrated -- **Verification-gated results.** The output value is reachable only through the `ok` variant of a discriminated union, so an unverified run is unrepresentable at the call site. -- **Post-condition contracts.** A run is trusted when a caller-supplied assertion holds over its output, a design-by-contract style check applied to untrusted code. +- **Self-verification.** Execution and verification are separate phases. A pure `selfVerify` post-condition tags a produced value as trusted only when a caller-supplied assertion holds; runners return `verified: true` exclusively on that path. +- **Verification-gated results.** Every `RunResult` arm carries a literal `verified: true | false`. The trusted value is reachable only through the `ok` + `verified: true` arm of a discriminated union, so an unverified run cannot be treated as success at the type level. +- **Post-condition contracts.** A run is trusted when a caller-supplied assertion holds over its output, a design-by-contract style check applied to untrusted code. Assertions may return a boolean or `{ pass, reason? }` for diagnostic refusal reasons. +- **Predicate composition.** `allAssertions` (conjunction) and `anyAssertion` (disjunction) build compound post-conditions without re-running the untrusted task. - **Deadline enforcement with cooperative cancellation.** An internal timer races the task and aborts the `AbortSignal` it runs under, composed with any caller-owned signal. - **Hard preemption via worker termination.** On the isolate tier, the wall-clock deadline and caller abort both call `worker.terminate()`, reclaiming the OS thread instead of abandoning a hung task. - **Resource isolation and ceilings.** Three independent budgets gate every run: wall-clock time, V8 old-generation heap (`maxOldGenerationSizeMb`), and measured UTF-8 output size (`maxOutputBytes`). Each maps to a distinct result status (`timeout`, `out-of-memory`, `output-too-large`). @@ -32,10 +34,12 @@ The first slice was the contract and the in-process runner that enforces it. The ``` runVerified(task, { timeoutMs, assert, signal?, maxOutputBytes? }) -> RunResult +selfVerify(value, assert) -> { verified: true, value } | { verified: false, value, reason? } ``` -- The value is returned **only** as `{ status: "ok", value, durationMs }`, and only when the task finished before `timeoutMs`, the payload stayed under `maxOutputBytes` when set, and `assert(value)` returned true. -- Every other outcome is an explicit refusal: `timeout`, `assertion-failed` (carries the value for diagnostics, never as trusted), `output-too-large`, `out-of-memory`, or `error`. +- The value is trusted **only** as `{ status: "ok", verified: true, value, durationMs }`, and only when the task finished before `timeoutMs`, the payload stayed under `maxOutputBytes` when set, and `assert(value)` passed. +- Every other outcome is an explicit refusal with `verified: false`: `timeout`, `assertion-failed` (carries the value and optional `reason` for diagnostics, never as trusted), `output-too-large`, `out-of-memory`, or `error`. +- `isVerified(result)` narrows on the literal `verified: true` arm so callers cannot read a trusted value without a type-level proof. - The task is handed an `AbortSignal` that fires on the deadline or on the caller's own signal, so well-behaved async work can stop early. On the worker tier that same abort path also terminates the isolate. The in-process tier cannot preempt code that blocks the event loop with a synchronous spin; that is what the isolate and container tiers are for. This tier defines the contract those tiers implement. @@ -43,7 +47,7 @@ The in-process tier cannot preempt code that blocks the event loop with a synchr ## Usage ```ts -import { runVerified, isVerified } from "airlock"; +import { runVerified, isVerified, allAssertions, selfVerify } from "airlock"; const result = await runVerified( async (signal) => { @@ -53,15 +57,26 @@ const result = await runVerified( { timeoutMs: 2000, maxOutputBytes: 64 * 1024, - assert: (data) => Number.isInteger(data.total) && data.total >= 0, + assert: allAssertions( + (data) => Number.isInteger(data.total), + (data) => + data.total >= 0 + ? { pass: true } + : { pass: false, reason: "total must be non-negative" }, + ), }, ); if (isVerified(result)) { + // result.verified is literally true here console.log("trusted output:", result.value.total); } else { - console.warn("refused:", result.status); + console.warn("refused:", result.status, result.verified); // always false } + +// Pure re-check of an already-produced value (no re-execution): +const check = await selfVerify({ total: 3 }, (d) => d.total > 0); +// check.verified === true ``` To run untrusted **source code** instead of a trusted closure, use `run`. The code executes with no ambient authority, so `process`, `require`, `fetch`, and timers are all undefined inside it. Any capability it needs is passed explicitly through `grant`: @@ -165,3 +180,4 @@ pnpm run build - `src/worker.ts`: `runInWorker(code, opts)` runs untrusted source in a `worker_threads` isolate started with an empty `process.env` and frozen globals, caps the heap with `maxOldGenerationSizeMb` (reported as `out-of-memory`), and hard-kills the thread on the deadline so a sync spin and a never-settling async task are both preempted. An escape-attempt test confirms the constructor walk that reaches the host realm in-process reaches only the credential-free worker realm here. - `src/limits.ts`: shared resource ceilings for every tier. Wall-clock timeout aborts the task signal and, on the worker tier, calls `worker.terminate()` on both deadline and caller abort. Heap cap via V8 `resourceLimits`. Output size caps (`maxOutputBytes`) measure UTF-8 payload with a budgeted walk (cycle-safe, early-exit) and refuse with `output-too-large` before the post-condition runs. - `src/modules.ts`: deny-by-default module loader with an explicit `allowedModules` allowlist. Omitted means no `require`; `[]` or a list injects `createGatedRequire` over the host/worker require. Exact match only (bare and `node:` equivalent), path specifiers always refused, and the gate wins over a grant-supplied `require`. Wired into both `run` and `runInWorker`. +- Self-verification: run a supplied assertion, return `verified: true` only if it passes. `src/verify.ts` owns the pure post-condition phase (`selfVerify`, `allAssertions` / `anyAssertion`, structured `{ pass, reason? }` outcomes). Every `RunResult` arm carries a literal `verified` flag; `isVerified` narrows on `verified: true`. Wired through `runVerified`, `run`, and `runInWorker`. diff --git a/src/contract.ts b/src/contract.ts index ce8e1ba..975cbf5 100644 --- a/src/contract.ts +++ b/src/contract.ts @@ -1,3 +1,5 @@ +import type { AssertionFn } from "./verify.js"; + export type RunStatus = | "ok" | "timeout" @@ -7,22 +9,38 @@ export type RunStatus = | "output-too-large"; /** - * A run only counts as verified when it carries `status: "ok"`. Every other - * variant is a refusal, so a caller cannot read `value` without first proving - * the run passed both the deadline and the post-condition. + * A run only counts as verified when it carries `verified: true`. Every other + * variant is a refusal, so a caller cannot treat a value as trusted without + * first proving the run passed both the deadline and the post-condition. + * The literal `true` / `false` on each arm makes the trust boundary visible + * at the type level, not only as a status string. */ export type RunResult = - | { status: "ok"; value: T; durationMs: number } - | { status: "timeout"; timeoutMs: number } - | { status: "assertion-failed"; value: T } - | { status: "error"; error: unknown } - | { status: "out-of-memory"; maxOldGenerationSizeMb: number } - | { status: "output-too-large"; maxOutputBytes: number; actualBytes: number }; + | { status: "ok"; verified: true; value: T; durationMs: number } + | { status: "timeout"; verified: false; timeoutMs: number } + | { + status: "assertion-failed"; + verified: false; + value: T; + reason?: string; + } + | { status: "error"; verified: false; error: unknown } + | { + status: "out-of-memory"; + verified: false; + maxOldGenerationSizeMb: number; + } + | { + status: "output-too-large"; + verified: false; + maxOutputBytes: number; + actualBytes: number; + }; export type Task = (signal: AbortSignal) => T | Promise; -/** Post-condition. A run's output is trusted only if this returns true. */ -export type Assertion = (value: T) => boolean | Promise; +/** Post-condition. A run's output is trusted only if this returns a pass. */ +export type Assertion = AssertionFn; export interface VerifiedRunOptions { timeoutMs: number; @@ -35,6 +53,6 @@ export interface VerifiedRunOptions { export function isVerified( result: RunResult, -): result is Extract, { status: "ok" }> { - return result.status === "ok"; +): result is Extract, { verified: true }> { + return result.verified === true; } diff --git a/src/index.ts b/src/index.ts index 0b399c5..1535973 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,16 @@ export { runVerified } from "./run.js"; export { isVerified } from "./contract.js"; +export { + selfVerify, + normalizeAssertOutcome, + allAssertions, + anyAssertion, +} from "./verify.js"; +export type { + AssertOutcome, + AssertionFn, + VerifyResult, +} from "./verify.js"; export { run, probeAmbientAuthority, diff --git a/src/run.ts b/src/run.ts index 4ca7e2f..aed9c7c 100644 --- a/src/run.ts +++ b/src/run.ts @@ -1,11 +1,13 @@ import type { RunResult, Task, VerifiedRunOptions } from "./contract.js"; import { checkOutputSize, validateResourceLimits } from "./limits.js"; +import { selfVerify } from "./verify.js"; const DEADLINE = Symbol("deadline"); /** - * The core airlock primitive. Runs `task` under a deadline, then checks the - * supplied post-condition, and hands back the value only when both pass. + * The core airlock primitive. Runs `task` under a deadline, then self-verifies + * the produced value with the supplied post-condition, and hands back + * `verified: true` only when both the deadline and the assertion pass. * * The deadline is enforced by racing an internal timer and aborting the signal * the task receives. That stops async and cooperative work, but a task that @@ -47,7 +49,7 @@ export async function runVerified( try { const outcome = await Promise.race([running, deadline]); if (outcome === DEADLINE) { - return { status: "timeout", timeoutMs }; + return { status: "timeout", verified: false, timeoutMs }; } const value = outcome as T; @@ -56,18 +58,32 @@ export async function runVerified( if (size.exceeded) { return { status: "output-too-large", + verified: false, maxOutputBytes, actualBytes: size.bytes, }; } } - const passed = await assert(value); - return passed - ? { status: "ok", value, durationMs: performance.now() - started } - : { status: "assertion-failed", value }; + const check = await selfVerify(value, assert); + if (check.verified) { + return { + status: "ok", + verified: true, + value: check.value, + durationMs: performance.now() - started, + }; + } + return check.reason !== undefined + ? { + status: "assertion-failed", + verified: false, + value: check.value, + reason: check.reason, + } + : { status: "assertion-failed", verified: false, value: check.value }; } catch (error) { - return { status: "error", error }; + return { status: "error", verified: false, error }; } finally { if (timer !== undefined) clearTimeout(timer); signal?.removeEventListener("abort", relayAbort); diff --git a/src/sandbox.ts b/src/sandbox.ts index d2a54b8..c7f1a2f 100644 --- a/src/sandbox.ts +++ b/src/sandbox.ts @@ -124,7 +124,7 @@ export async function run( try { script = new vm.Script(code, { filename: filename ?? "airlock-sandbox.js" }); } catch (error) { - return { status: "error", error }; + return { status: "error", verified: false, error }; } const result = await runVerified( @@ -142,7 +142,7 @@ export async function run( ); if (result.status === "error" && isSyncTimeout(result.error)) { - return { status: "timeout", timeoutMs }; + return { status: "timeout", verified: false, timeoutMs }; } return result; } diff --git a/src/verify.ts b/src/verify.ts new file mode 100644 index 0000000..8cb5bd1 --- /dev/null +++ b/src/verify.ts @@ -0,0 +1,86 @@ +/** + * Self-verification is a separate phase from execution: take a produced value + * and a caller-supplied post-condition, and tag the value as trusted only when + * the assertion holds. Execution tiers (vm, worker, later Docker) hand results + * here; this module never runs untrusted code itself. + */ + +export type AssertOutcome = + | boolean + | { pass: true } + | { pass: false; reason?: string }; + +export type AssertionFn = ( + value: T, +) => AssertOutcome | Promise; + +export type VerifyResult = + | { verified: true; value: T } + | { verified: false; value: T; reason?: string }; + +export function normalizeAssertOutcome(outcome: AssertOutcome): { + passed: boolean; + reason?: string; +} { + if (typeof outcome === "boolean") { + return { passed: outcome }; + } + if (outcome.pass) { + return { passed: true }; + } + return outcome.reason !== undefined + ? { passed: false, reason: outcome.reason } + : { passed: false }; +} + +/** + * Pure post-condition check. `verified: true` is returned only when the + * assertion resolves to a passing outcome; every other result is untrusted. + */ +export async function selfVerify( + value: T, + assert: AssertionFn, +): Promise> { + const outcome = await assert(value); + const { passed, reason } = normalizeAssertOutcome(outcome); + if (passed) { + return { verified: true, value }; + } + return reason !== undefined + ? { verified: false, value, reason } + : { verified: false, value }; +} + +/** Conjoin assertions: every one must pass for the value to be trusted. */ +export function allAssertions( + ...asserts: readonly AssertionFn[] +): AssertionFn { + return async (value) => { + for (const assert of asserts) { + const { passed, reason } = normalizeAssertOutcome(await assert(value)); + if (!passed) { + return reason !== undefined + ? { pass: false, reason } + : { pass: false }; + } + } + return { pass: true }; + }; +} + +/** Disjoin assertions: one passing check is enough. */ +export function anyAssertion( + ...asserts: readonly AssertionFn[] +): AssertionFn { + return async (value) => { + let lastReason: string | undefined; + for (const assert of asserts) { + const { passed, reason } = normalizeAssertOutcome(await assert(value)); + if (passed) return { pass: true }; + if (reason !== undefined) lastReason = reason; + } + return lastReason !== undefined + ? { pass: false, reason: lastReason } + : { pass: false }; + }; +} diff --git a/src/worker.ts b/src/worker.ts index ea8a109..9296c2e 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -2,6 +2,7 @@ import { Worker } from "node:worker_threads"; import type { Assertion, RunResult } from "./contract.js"; import { checkOutputSize, validateResourceLimits } from "./limits.js"; import { createGatedRequire } from "./modules.js"; +import { selfVerify } from "./verify.js"; /** * Intrinsics whose prototypes a sandbox escape could otherwise repave to attack @@ -179,7 +180,7 @@ export function runInWorker( }); } catch (error) { // A non-cloneable grant (e.g. a function) fails at construction. - return Promise.resolve({ status: "error", error }); + return Promise.resolve({ status: "error", verified: false, error }); } const started = performance.now(); @@ -199,11 +200,11 @@ export function runInWorker( }; const timer = setTimeout(() => { - finish({ status: "timeout", timeoutMs }); + finish({ status: "timeout", verified: false, timeoutMs }); }, timeoutMs); const onAbort = () => { - finish({ status: "error", error: signal?.reason }); + finish({ status: "error", verified: false, error: signal?.reason }); }; if (signal) { if (signal.aborted) return onAbort(); @@ -220,49 +221,72 @@ export function runInWorker( if (size.exceeded) { finish({ status: "output-too-large", + verified: false, maxOutputBytes, actualBytes: size.bytes, }); return; } } - void Promise.resolve(assert(value)).then( - (passed) => + void selfVerify(value, assert).then( + (check) => { + if (check.verified) { + finish({ + status: "ok", + verified: true, + value: check.value, + durationMs: performance.now() - started, + }); + return; + } finish( - passed + check.reason !== undefined ? { - status: "ok", - value, - durationMs: performance.now() - started, + status: "assertion-failed", + verified: false, + value: check.value, + reason: check.reason, } - : { status: "assertion-failed", value }, - ), - (error) => finish({ status: "error", error }), + : { + status: "assertion-failed", + verified: false, + value: check.value, + }, + ); + }, + (error: unknown) => + finish({ status: "error", verified: false, error }), ); return; } if (msg.error.code === SYNC_TIMEOUT_CODE) { - finish({ status: "timeout", timeoutMs }); + finish({ status: "timeout", verified: false, timeoutMs }); return; } - finish({ status: "error", error: reviveError(msg.error) }); + finish({ + status: "error", + verified: false, + error: reviveError(msg.error), + }); }); worker.on("error", (error: Error & { code?: string }) => { if (error.code === OOM_CODE) { finish({ status: "out-of-memory", + verified: false, maxOldGenerationSizeMb: maxOldGenerationSizeMb ?? 0, }); return; } - finish({ status: "error", error }); + finish({ status: "error", verified: false, error }); }); worker.on("exit", (code) => { if (code !== 0) { finish({ status: "error", + verified: false, error: new Error(`worker exited with code ${code}`), }); } diff --git a/test/limits.test.ts b/test/limits.test.ts index d9bc543..594f0dd 100644 --- a/test/limits.test.ts +++ b/test/limits.test.ts @@ -193,7 +193,7 @@ describe("wall-clock timeout terminates the worker on abort", () => { }); const elapsed = performance.now() - started; - expect(result).toEqual({ status: "timeout", timeoutMs: 80 }); + expect(result).toEqual({ status: "timeout", verified: false, timeoutMs: 80 }); // terminate() should reclaim the thread promptly after the timer fires expect(elapsed).toBeLessThan(1500); }); diff --git a/test/run.test.ts b/test/run.test.ts index a870749..5591dc8 100644 --- a/test/run.test.ts +++ b/test/run.test.ts @@ -21,7 +21,7 @@ describe("runVerified", () => { assert: (v) => v === 42, }); - expect(result).toEqual({ status: "assertion-failed", value: 41 }); + expect(result).toEqual({ status: "assertion-failed", verified: false, value: 41 }); expect(isVerified(result)).toBe(false); }); @@ -50,7 +50,7 @@ describe("runVerified", () => { { timeoutMs: 10, assert: () => true }, ); - expect(result).toEqual({ status: "timeout", timeoutMs: 10 }); + expect(result).toEqual({ status: "timeout", verified: false, timeoutMs: 10 }); expect(aborted).toBe(true); }); @@ -63,7 +63,7 @@ describe("runVerified", () => { { timeoutMs: 100, assert: () => true }, ); - expect(result).toEqual({ status: "error", error: boom }); + expect(result).toEqual({ status: "error", verified: false, error: boom }); }); it("treats a throwing assertion as an error", async () => { diff --git a/test/sandbox.test.ts b/test/sandbox.test.ts index 01b6978..2233258 100644 --- a/test/sandbox.test.ts +++ b/test/sandbox.test.ts @@ -25,7 +25,7 @@ describe("run", () => { assert: (v) => v === 42, }); - expect(result).toEqual({ status: "assertion-failed", value: 41 }); + expect(result).toEqual({ status: "assertion-failed", verified: false, value: 41 }); expect(isVerified(result)).toBe(false); }); @@ -65,7 +65,7 @@ describe("run", () => { assert: () => true, }); - expect(result).toEqual({ status: "timeout", timeoutMs: 25 }); + expect(result).toEqual({ status: "timeout", verified: false, timeoutMs: 25 }); }); it("times out on an async task that never settles", async () => { @@ -74,7 +74,7 @@ describe("run", () => { assert: () => true, }); - expect(result).toEqual({ status: "timeout", timeoutMs: 25 }); + expect(result).toEqual({ status: "timeout", verified: false, timeoutMs: 25 }); }); it.each([0, -1, Number.NaN, Number.POSITIVE_INFINITY])( diff --git a/test/verify.test.ts b/test/verify.test.ts new file mode 100644 index 0000000..e3ae6cb --- /dev/null +++ b/test/verify.test.ts @@ -0,0 +1,244 @@ +import { describe, expect, it } from "vitest"; +import { + allAssertions, + anyAssertion, + isVerified, + normalizeAssertOutcome, + run, + runInWorker, + runVerified, + selfVerify, +} from "../src/index.js"; + +describe("selfVerify", () => { + it("returns verified:true only when the assertion passes", async () => { + const pass = await selfVerify(42, (n) => n === 42); + const fail = await selfVerify(41, (n) => n === 42); + + expect(pass).toEqual({ verified: true, value: 42 }); + expect(fail).toEqual({ verified: false, value: 41 }); + }); + + it("accepts structured { pass } outcomes and carries a reason on failure", async () => { + const pass = await selfVerify("x", () => ({ pass: true as const })); + const fail = await selfVerify("x", () => ({ + pass: false as const, + reason: "expected a number", + })); + + expect(pass).toEqual({ verified: true, value: "x" }); + expect(fail).toEqual({ + verified: false, + value: "x", + reason: "expected a number", + }); + }); + + it("supports async assertions and empty-input edge cases", async () => { + const ok = await selfVerify([1, 2, 3], async (xs) => { + await Promise.resolve(); + return xs.length === 3; + }); + const empty = await selfVerify("", (s) => s.length > 0); + expect(ok).toEqual({ verified: true, value: [1, 2, 3] }); + expect(empty).toEqual({ verified: false, value: "" }); + }); + + it("keeps concurrent self-verifies independent", async () => { + const results = await Promise.all([ + selfVerify(1, (n) => n === 1), + selfVerify(2, () => false), + selfVerify(3, async () => ({ pass: true as const })), + ]); + expect(results.map((r) => r.verified)).toEqual([true, false, true]); + }); +}); + +describe("normalizeAssertOutcome", () => { + it("maps boolean and structured forms", () => { + expect(normalizeAssertOutcome(true)).toEqual({ passed: true }); + expect(normalizeAssertOutcome(false)).toEqual({ passed: false }); + expect(normalizeAssertOutcome({ pass: true })).toEqual({ passed: true }); + expect(normalizeAssertOutcome({ pass: false, reason: "nope" })).toEqual({ + passed: false, + reason: "nope", + }); + }); +}); + +describe("assertion composition", () => { + it("allAssertions requires every check and surfaces the first reason", async () => { + const check = allAssertions<{ n: number }>( + (v) => + v.n > 0 + ? { pass: true } + : { pass: false, reason: "must be positive" }, + (v) => + v.n % 2 === 0 + ? { pass: true } + : { pass: false, reason: "must be even" }, + ); + expect(await selfVerify({ n: 4 }, check)).toEqual({ + verified: true, + value: { n: 4 }, + }); + expect(await selfVerify({ n: -1 }, check)).toEqual({ + verified: false, + value: { n: -1 }, + reason: "must be positive", + }); + expect(await selfVerify({ n: 3 }, check)).toEqual({ + verified: false, + value: { n: 3 }, + reason: "must be even", + }); + }); + + it("anyAssertion passes when one check succeeds; empty all/any edges", async () => { + const check = anyAssertion( + (s) => s.startsWith("http"), + (s) => s.startsWith("/"), + ); + expect(await selfVerify("/local", check)).toEqual({ + verified: true, + value: "/local", + }); + expect(await selfVerify("ftp://x", check)).toEqual({ + verified: false, + value: "ftp://x", + }); + expect(await selfVerify(1, anyAssertion())).toEqual({ + verified: false, + value: 1, + }); + expect(await selfVerify(1, allAssertions())).toEqual({ + verified: true, + value: 1, + }); + }); +}); + +describe("runVerified self-verification gate", () => { + it("tags ok with verified:true and refuses with verified:false", async () => { + const ok = await runVerified(() => 7, { + timeoutMs: 100, + assert: (n) => n === 7, + }); + const refused = await runVerified(() => 7, { + timeoutMs: 100, + assert: (n) => n === 0, + }); + + expect(ok).toMatchObject({ status: "ok", verified: true, value: 7 }); + expect(isVerified(ok)).toBe(true); + if (isVerified(ok)) expect(ok.value).toBe(7); + + expect(refused).toEqual({ + status: "assertion-failed", + verified: false, + value: 7, + }); + expect(isVerified(refused)).toBe(false); + }); + + it("propagates assertion reasons and never verifies failure modes", async () => { + const reason = await runVerified(() => ({ total: -1 }), { + timeoutMs: 100, + assert: (v) => + v.total >= 0 + ? { pass: true } + : { pass: false, reason: "total must be non-negative" }, + }); + const timeout = await runVerified(() => new Promise(() => {}), { + timeoutMs: 15, + assert: () => true, + }); + const oversized = await runVerified(() => "x".repeat(100), { + timeoutMs: 100, + assert: () => true, + maxOutputBytes: 8, + }); + + expect(reason).toEqual({ + status: "assertion-failed", + verified: false, + value: { total: -1 }, + reason: "total must be non-negative", + }); + expect(timeout).toEqual({ + status: "timeout", + verified: false, + timeoutMs: 15, + }); + expect(oversized).toMatchObject({ + status: "output-too-large", + verified: false, + maxOutputBytes: 8, + }); + expect(isVerified(timeout)).toBe(false); + expect(isVerified(oversized)).toBe(false); + }); + + it("boundary: zero is trusted only when the assertion allows it", async () => { + const allowZero = await runVerified(() => 0, { + timeoutMs: 100, + assert: (n) => n === 0, + }); + const refuseZero = await runVerified(() => 0, { + timeoutMs: 100, + assert: (n) => n > 0, + }); + expect(allowZero).toMatchObject({ status: "ok", verified: true, value: 0 }); + expect(refuseZero).toEqual({ + status: "assertion-failed", + verified: false, + value: 0, + }); + }); +}); + +describe("sandbox and worker self-verification", () => { + it("run and runInWorker return verified:true only after the post-condition", async () => { + const ok = await run("21 * 2", { + timeoutMs: 100, + assert: (n) => n === 42, + }); + const bad = await run("21 * 2", { + timeoutMs: 100, + assert: (n) => n === 0, + }); + const workerOk = await runInWorker( + "rows.reduce((s, n) => s + n, 0)", + { + timeoutMs: 500, + assert: (total) => total === 6, + grant: { rows: [1, 2, 3] }, + }, + ); + const workerBad = await runInWorker("99", { + timeoutMs: 500, + assert: () => ({ + pass: false as const, + reason: "unexpected constant", + }), + }); + + expect(ok).toMatchObject({ status: "ok", verified: true, value: 42 }); + expect(bad).toEqual({ + status: "assertion-failed", + verified: false, + value: 42, + }); + expect(workerOk).toMatchObject({ + status: "ok", + verified: true, + value: 6, + }); + expect(workerBad).toEqual({ + status: "assertion-failed", + verified: false, + value: 99, + reason: "unexpected constant", + }); + }); +}); diff --git a/test/worker.test.ts b/test/worker.test.ts index a3dfa74..66a4ded 100644 --- a/test/worker.test.ts +++ b/test/worker.test.ts @@ -26,7 +26,7 @@ describe("runInWorker", () => { assert: (v) => v === 42, }); - expect(result).toEqual({ status: "assertion-failed", value: 41 }); + expect(result).toEqual({ status: "assertion-failed", verified: false, value: 41 }); expect(isVerified(result)).toBe(false); }); @@ -77,7 +77,7 @@ describe("runInWorker", () => { assert: () => true, }); - expect(result).toEqual({ status: "timeout", timeoutMs: 100 }); + expect(result).toEqual({ status: "timeout", verified: false, timeoutMs: 100 }); }); it("times out on an async task that never settles", async () => { @@ -86,7 +86,7 @@ describe("runInWorker", () => { assert: () => true, }); - expect(result).toEqual({ status: "timeout", timeoutMs: 100 }); + expect(result).toEqual({ status: "timeout", verified: false, timeoutMs: 100 }); }); it("aborts when the caller's signal fires", async () => { @@ -100,7 +100,7 @@ describe("runInWorker", () => { controller.abort(reason); const result = await pending; - expect(result).toEqual({ status: "error", error: reason }); + expect(result).toEqual({ status: "error", verified: false, error: reason }); }); it.each([0, -1, Number.NaN, Number.POSITIVE_INFINITY])(