diff --git a/.asf.yaml b/.asf.yaml index 6ac3fe6b8b..abb77a54e5 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -69,6 +69,10 @@ github: # reports and no committer can override it. contexts: - test + # Windows recovery is a separate native crash/owner-death boundary. + # The workflow runs on every PR and main push so this context can be + # required without leaving unrelated pull requests pending forever. + - windows_recovery rulesets: - name: Immutable release tags diff --git a/.github/workflows/windows-recovery.yml b/.github/workflows/windows-recovery.yml index 90dc9b1e5c..91de576062 100644 --- a/.github/workflows/windows-recovery.yml +++ b/.github/workflows/windows-recovery.yml @@ -18,10 +18,15 @@ name: Windows recovery on: + pull_request: + branches: [main] + push: + branches: [main] workflow_dispatch: concurrency: group: windows-recovery-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read @@ -47,6 +52,21 @@ jobs: - name: Build test artifacts run: npm.cmd run build:test + - name: Verify managed dependency alternate streams + shell: pwsh + run: | + node.exe --test --test-reporter=tap --test-concurrency=1 ` + --test-name-pattern="NTFS alternate stream" ` + packages/storage/dist/__tests__/managed-dependency-environment.test.js ` + 2>&1 | Tee-Object -FilePath "$env:RUNNER_TEMP/managed-dependency-ads.tap" + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { exit $exitCode } + $output = Get-Content "$env:RUNNER_TEMP/managed-dependency-ads.tap" + if ($output -notcontains '# tests 3' -or $output -notcontains '# pass 3' -or $output -notcontains '# skipped 0') { + Write-Error 'Managed dependency ADS gate did not run exactly three passing Windows tests' + exit 1 + } + - name: Verify Runtime Host Local IPC trust boundary shell: pwsh run: | diff --git a/packages/storage/src/__tests__/fixtures/root-initialization-race.ts b/packages/storage/src/__tests__/fixtures/root-initialization-race.ts index 6446802002..9234c1d756 100644 --- a/packages/storage/src/__tests__/fixtures/root-initialization-race.ts +++ b/packages/storage/src/__tests__/fixtures/root-initialization-race.ts @@ -18,7 +18,6 @@ */ import fs from 'node:fs'; -import { syncBuiltinESMExports } from 'node:module'; import { join } from 'node:path'; const [rootArgument, markerFile] = process.argv.slice(2); @@ -44,8 +43,9 @@ fs.promises.open = (async (path, flags, mode) => { } return originalOpen(path, flags, mode); }) as typeof fs.promises.open; -syncBuiltinESMExports(); +// Import after the interposition: marker-file captures the intrinsic at module +// evaluation, while production code must ignore later global mutations. const { resolveStorageRoot, StorageRootAuthorityError } = await import('../../root-authority.js'); const parentDisconnected = new Promise((resolvePromise) => diff --git a/packages/storage/src/__tests__/managed-dependency-environment.test.ts b/packages/storage/src/__tests__/managed-dependency-environment.test.ts index d154404ecc..a2318cd487 100644 --- a/packages/storage/src/__tests__/managed-dependency-environment.test.ts +++ b/packages/storage/src/__tests__/managed-dependency-environment.test.ts @@ -363,10 +363,81 @@ test('rejects an NTFS alternate stream created inside a dependency artifact', { storageRoot, producer, }); - await assert.rejects(authority.acquire(identity, source), /alternate data stream/u); + await assert.rejects( + authority.acquire(identity, source), + (error: unknown) => + error instanceof Error && + error.message === 'Managed dependency environment contains an alternate data stream', + ); await authority.close(); }); +test('rejects an NTFS alternate stream attached to the published dependency root', { + skip: process.platform !== 'win32', +}, async (t) => { + const storageRoot = await mkdtemp(join(tmpdir(), 'maka-dependency-root-ads-')); + t.after(() => rm(storageRoot, { recursive: true, force: true })); + const producer = { + capability: FIXTURE_PRODUCER_CAPABILITY, + packageManagerName: 'npm' as const, + packageManagerVersion: '11.12.1', + nodeRuntime: fixtureNodeRuntime(), + async provision(input: { outputRoot: string }) { + await writeFile(join(input.outputRoot, 'index.js'), 'trusted\n', 'utf8'); + }, + }; + const source = dependencySourceForName('root-ads'); + const identity = computeManagedDependencyEnvironmentIdentity(source); + const authority = await createManagedDependencyEnvironmentAuthority({ storageRoot, producer }); + const lease = await authority.acquire(identity, source); + await writeFile(`${lease.dependencyRoot}:unhashed`, 'malicious\n', 'utf8'); + await lease.release(); + await authority.close(); + + const reopened = await createManagedDependencyEnvironmentAuthority({ storageRoot, producer }); + await assert.rejects( + reopened.acquire(identity, source), + (error: unknown) => + error instanceof Error && + error.message === 'Managed dependency environment contains an alternate data stream', + ); + await reopened.close(); +}); + +test('rejects an NTFS alternate stream attached to a published nested directory', { + skip: process.platform !== 'win32', +}, async (t) => { + const storageRoot = await mkdtemp(join(tmpdir(), 'maka-dependency-目录-ads-')); + t.after(() => rm(storageRoot, { recursive: true, force: true })); + const producer = { + capability: FIXTURE_PRODUCER_CAPABILITY, + packageManagerName: 'npm' as const, + packageManagerVersion: '11.12.1', + nodeRuntime: fixtureNodeRuntime(), + async provision(input: { outputRoot: string }) { + const packageRoot = join(input.outputRoot, 'fixture-包'); + await mkdir(packageRoot); + await writeFile(join(packageRoot, 'index.js'), 'trusted\n', 'utf8'); + }, + }; + const source = dependencySourceForName('nested-directory-ads'); + const identity = computeManagedDependencyEnvironmentIdentity(source); + const authority = await createManagedDependencyEnvironmentAuthority({ storageRoot, producer }); + const lease = await authority.acquire(identity, source); + await writeFile(`${join(lease.dependencyRoot, 'fixture-包')}:unhashed`, 'malicious\n', 'utf8'); + await lease.release(); + await authority.close(); + + const reopened = await createManagedDependencyEnvironmentAuthority({ storageRoot, producer }); + await assert.rejects( + reopened.acquire(identity, source), + (error: unknown) => + error instanceof Error && + error.message === 'Managed dependency environment contains an alternate data stream', + ); + await reopened.close(); +}); + test('accepts a POSIX package bin symlink whose target remains inside the dependency root', { skip: process.platform === 'win32', }, async (t) => { diff --git a/packages/storage/src/__tests__/marker-file.test.ts b/packages/storage/src/__tests__/marker-file.test.ts index 294c6276da..e842b946fa 100644 --- a/packages/storage/src/__tests__/marker-file.test.ts +++ b/packages/storage/src/__tests__/marker-file.test.ts @@ -18,6 +18,7 @@ */ import assert from 'node:assert/strict'; +import fs from 'node:fs'; import { mkdtemp, open, readdir, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -29,6 +30,33 @@ import { type MarkerFileHandle, } from '../marker-file.js'; +test('keeps the open primitive captured at module initialization', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-marker-file-captured-open-')); + const markerFile = '.marker.json'; + const originalOpen = fs.promises.open; + let intercepted = false; + fs.promises.open = (async (path, flags, mode) => { + if (typeof path === 'string' && path.startsWith(join(root, `${markerFile}.`))) { + intercepted = true; + } + return originalOpen(path, flags, mode); + }) as typeof fs.promises.open; + try { + await publishMarkerFile({ + root, + markerFile, + contents: '{"schemaVersion":1}\n', + maxBytes: 1_024, + publication: 'create', + invalidFile: () => new Error('invalid marker'), + }); + assert.equal(intercepted, false); + } finally { + fs.promises.open = originalOpen; + await rm(root, { recursive: true, force: true }); + } +}); + for (const publication of ['create', 'replace'] as const) { for (const failurePhase of ['write', 'sync', 'close'] as const) { test(`${publication} removes its temporary marker after a ${failurePhase} failure`, async () => { diff --git a/packages/storage/src/managed-dependency-environment.ts b/packages/storage/src/managed-dependency-environment.ts index 85065dde3c..46aa197777 100644 --- a/packages/storage/src/managed-dependency-environment.ts +++ b/packages/storage/src/managed-dependency-environment.ts @@ -17,7 +17,7 @@ * under the License. */ -import { execFile } from 'node:child_process'; +import { spawn } from 'node:child_process'; import { createHash, randomUUID } from 'node:crypto'; import { createReadStream } from 'node:fs'; import { createRequire } from 'node:module'; @@ -36,7 +36,16 @@ import { stat, utimes, } from 'node:fs/promises'; -import { dirname, isAbsolute, join, normalize, posix, relative, resolve } from 'node:path'; +import { + dirname, + isAbsolute, + join, + normalize, + posix, + relative, + resolve, + toNamespacedPath, +} from 'node:path'; import type { DatabaseSync } from 'node:sqlite'; import { tryLock, unlock } from 'fs-native-extensions'; @@ -57,6 +66,106 @@ const MANAGED_DEPENDENCY_PRODUCER_POLICY_V1 = Object.freeze({ lifecycleScripts: 'disabled' as const, }); const activeAuthorityOwners = new Map(); +const WINDOWS_STREAM_QUERY_TIMEOUT_MS = 30_000; +const WINDOWS_STREAM_QUERY_MAX_OUTPUT_BYTES = 1024 * 1024; +const WINDOWS_STREAM_QUERY_SCRIPT = String.raw` +$ErrorActionPreference = 'Stop' +$OutputEncoding = [Console]::OutputEncoding = [Text.UTF8Encoding]::new($false) +Add-Type -TypeDefinition @' +using System; +using System.ComponentModel; +using System.Runtime.InteropServices; + +public static class MakaWindowsStreamQuery +{ + private const int ErrorHandleEof = 38; + private static readonly IntPtr InvalidHandleValue = new IntPtr(-1); + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] + private struct FindStreamData + { + public long StreamSize; + + [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 296)] + public string StreamName; + } + + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + private static extern IntPtr FindFirstStreamW( + string fileName, + int infoLevel, + out FindStreamData findStreamData, + int flags); + + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + private static extern bool FindNextStreamW( + IntPtr findStream, + out FindStreamData findStreamData); + + [DllImport("kernel32.dll", SetLastError = true)] + private static extern bool FindClose(IntPtr findHandle); + + public static bool HasAlternateDataStream(string path) + { + FindStreamData data; + IntPtr handle = FindFirstStreamW(path, 0, out data, 0); + if (handle == InvalidHandleValue) + { + int error = Marshal.GetLastWin32Error(); + if (error == ErrorHandleEof) + { + return false; + } + throw new Win32Exception(error); + } + + try + { + do + { + if (!String.Equals(data.StreamName, "::$DATA", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + while (FindNextStreamW(handle, out data)); + + int error = Marshal.GetLastWin32Error(); + if (error != ErrorHandleEof) + { + throw new Win32Exception(error); + } + return false; + } + finally + { + if (!FindClose(handle)) + { + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + } + } +} +'@ +$reader = [IO.StreamReader]::new( + [Console]::OpenStandardInput(), + [Text.UTF8Encoding]::new($false, $true), + $true +) +try { + $raw = $reader.ReadToEnd() +} finally { + $reader.Dispose() +} +$paths = if ([string]::IsNullOrWhiteSpace($raw)) { @() } else { @($raw | ConvertFrom-Json) } +$alternate = [System.Collections.Generic.List[string]]::new() +foreach ($path in $paths) { + if ([MakaWindowsStreamQuery]::HasAlternateDataStream([string]$path)) { + $alternate.Add([string]$path) + } +} +[Console]::Out.Write((ConvertTo-Json -InputObject @($alternate.ToArray()) -Compress)) +`; const RECEIPT_KEYS = [ 'protocolVersion', 'environmentId', @@ -983,20 +1092,22 @@ async function hashDirectory( /** * Reject NTFS alternate data streams under a dependency tree. * - * Previously this shellled out to a recursive PowerShell `Get-ChildItem -Recurse` + * Previously this shelled out to a recursive PowerShell `Get-ChildItem -Recurse` * + `Get-Item -Stream *` walk. On GitHub-hosted Windows runners that path * routinely hung until the 30s `execFile` timeout, which misreported every * timeout as "contains an alternate data stream" and burned ~5 minutes across * managed-dependency crash recovery tests. Walk the tree in Node (no reparse - * follow) and query streams per file with `fsutil`, which is bounded and - * does not recurse through junctions. + * follow), then send the bounded object list through stdin to one non-recursive + * Windows PowerShell stream query. `fsutil file queryStreams` is not a supported + * command on the Windows builds used by developers or hosted runners. */ async function assertNoWindowsAlternateStreams(root: string): Promise { const systemRoot = process.env.SystemRoot ?? process.env.WINDIR; if (!systemRoot) { throw new Error('Cannot verify Windows alternate streams without SystemRoot'); } - const fsutil = join(systemRoot, 'System32', 'fsutil.exe'); + const powershell = join(systemRoot, 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe'); + const paths = [toNamespacedPath(root)]; const stack = [root]; while (stack.length > 0) { const directory = stack.pop()!; @@ -1010,67 +1121,98 @@ async function assertNoWindowsAlternateStreams(root: string): Promise { throw new Error('Managed dependency environment contains a Windows reparse point'); } if (entry.isDirectory()) { + paths.push(toNamespacedPath(absolutePath)); stack.push(absolutePath); continue; } if (!entry.isFile()) continue; - await assertWindowsFileHasOnlyDefaultStream(fsutil, absolutePath); + // Windows PowerShell 5.1 cannot open long provider paths unless the + // caller supplies the Win32 namespaced spelling. + paths.push(toNamespacedPath(absolutePath)); } } + const alternate = await queryWindowsAlternateStreams(powershell, paths); + if (alternate.length > 0) { + throw new Error('Managed dependency environment contains an alternate data stream'); + } } -async function assertWindowsFileHasOnlyDefaultStream( - fsutil: string, - filePath: string, -): Promise { - const stdout = await new Promise((resolvePromise, rejectPromise) => { - execFile( - fsutil, - ['file', 'queryStreams', filePath], - { - windowsHide: true, - timeout: 15_000, - maxBuffer: 1024 * 1024, - }, - (error, out) => { - if (error) { - const execError = error as Error & { killed?: boolean; code?: string | number | null }; - const timedOut = - execError.killed === true || - execError.code === 'ETIMEDOUT' || - /ETIMEDOUT|timeout/i.test(execError.message); - rejectPromise( - new Error( - timedOut - ? `Timed out querying alternate data streams for ${filePath}` - : `Unable to query alternate data streams for ${filePath}`, - { cause: error }, - ), - ); - return; - } - resolvePromise(typeof out === 'string' ? out : String(out ?? '')); - }, +function queryWindowsAlternateStreams( + powershell: string, + paths: readonly string[], +): Promise { + if (paths.length === 0) return Promise.resolve([]); + return new Promise((resolvePromise, rejectPromise) => { + const child = spawn( + powershell, + ['-NoLogo', '-NoProfile', '-NonInteractive', '-Command', WINDOWS_STREAM_QUERY_SCRIPT], + { windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'] }, ); + let stdout = ''; + let stderr = ''; + let timedOut = false; + let overflow = false; + let settled = false; + const timer = setTimeout(() => { + timedOut = true; + child.kill(); + }, WINDOWS_STREAM_QUERY_TIMEOUT_MS); + const append = (current: string, chunk: Buffer): string => { + const next = current + chunk.toString('utf8'); + if (Buffer.byteLength(next, 'utf8') <= WINDOWS_STREAM_QUERY_MAX_OUTPUT_BYTES) return next; + overflow = true; + child.kill(); + return current; + }; + child.stdout.on('data', (chunk: Buffer) => { + stdout = append(stdout, chunk); + }); + child.stderr.on('data', (chunk: Buffer) => { + stderr = append(stderr, chunk); + }); + child.stdin.on('error', () => {}); + child.once('error', (error) => { + clearTimeout(timer); + if (settled) return; + settled = true; + rejectPromise( + new Error('Unable to start the Windows alternate-stream query', { cause: error }), + ); + }); + child.once('close', (code) => { + clearTimeout(timer); + if (settled) return; + settled = true; + if (timedOut) { + rejectPromise(new Error('Timed out querying Windows alternate data streams')); + return; + } + if (overflow) { + rejectPromise(new Error('Windows alternate-stream query output exceeded its limit')); + return; + } + if (code !== 0) { + rejectPromise( + new Error( + `Unable to query Windows alternate data streams: ${stderr.trim() || `exit ${code}`}`, + ), + ); + return; + } + try { + const value: unknown = JSON.parse(stdout || '[]'); + if (!Array.isArray(value) || !value.every((path) => typeof path === 'string')) { + throw new Error('query returned an invalid result'); + } + resolvePromise(value); + } catch (error) { + rejectPromise( + new Error('Windows alternate-stream query returned invalid JSON', { cause: error }), + ); + } + }); + child.stdin.end(JSON.stringify(paths)); }); - for (const rawLine of stdout.split(/\r?\n/u)) { - const line = rawLine.trim(); - if (!line) continue; - // Verbose form: "Name : :$DATA" or "Name : :unhashed:$DATA" - const nameMatch = /^Name\s*:\s*(.+)$/iu.exec(line); - if (nameMatch) { - if (isDefaultWindowsDataStreamName(nameMatch[1]!.trim())) continue; - throw new Error('Managed dependency environment contains an alternate data stream'); - } - // Compact form used by some fsutil builds: ":$DATA" / "Zone.Identifier:$DATA" - if (/:\$DATA\b/iu.test(line) && !isDefaultWindowsDataStreamName(line.split(/\s+/u)[0]!)) { - throw new Error('Managed dependency environment contains an alternate data stream'); - } - } -} - -function isDefaultWindowsDataStreamName(name: string): boolean { - return name === ':$DATA' || name === '::$DATA'; } function isPathWithin(candidate: string, root: string): boolean { diff --git a/packages/storage/src/marker-file.ts b/packages/storage/src/marker-file.ts index dc9239481a..e7a66da035 100644 --- a/packages/storage/src/marker-file.ts +++ b/packages/storage/src/marker-file.ts @@ -18,8 +18,8 @@ */ import { randomUUID } from 'node:crypto'; -import { constants as fsConstants, type BigIntStats } from 'node:fs'; -import { link, lstat, open, rename, unlink } from 'node:fs/promises'; +import fs, { constants as fsConstants, type BigIntStats } from 'node:fs'; +import { link, lstat, rename, unlink } from 'node:fs/promises'; import { join } from 'node:path'; export interface MarkerFileHandle { @@ -35,8 +35,12 @@ export interface MarkerFileDependencies { randomUUID(): string; } +const openMarkerFile = fs.promises.open.bind(fs.promises); const defaultDependencies: MarkerFileDependencies = { - open: async (path, flags, mode) => open(path, flags, mode), + // Capture once so later-loaded code cannot replace the marker authority's + // filesystem primitive. Race fixtures interpose before dynamically importing + // this module and are captured at the same boundary. + open: openMarkerFile, randomUUID, }; diff --git a/scripts/ci-test-plan.test.mjs b/scripts/ci-test-plan.test.mjs index 3783160592..803be9a0d6 100644 --- a/scripts/ci-test-plan.test.mjs +++ b/scripts/ci-test-plan.test.mjs @@ -346,10 +346,21 @@ test('pull request triggers stay on an explicit allowlist', () => { 'gitoxide-helper-admission.yml', 'release-windows-check.yml', 'runtime-host-owner-platform.yml', + 'windows-recovery.yml', 'windows-sandbox-w0.yml', ]); }); +test('Windows recovery publishes one stable PR and main check for ruleset enforcement', () => { + const workflow = readWorkflow('windows-recovery.yml'); + + assert.match(workflow, /\n {2}pull_request:\n {4}branches: \[main\]/u); + assert.match(workflow, /\n {2}push:\n {4}branches: \[main\]/u); + assert.match(workflow, /\n {2}workflow_dispatch:/u); + assert.match(workflow, /\n {4}name: windows_recovery/u); + assert.match(workflow, /cancel-in-progress: \$\{\{ github\.event_name == 'pull_request' \}\}/u); +}); + test('the sandbox lane pairs its path filter with a nightly run', () => { const workflow = readWorkflow('windows-sandbox-w0.yml'); @@ -438,6 +449,20 @@ test('specialized platform workflows stay reachable without pull requests', () = assert.match(baseline, /\n schedule:/u); }); +test('Windows recovery executes the exact managed dependency ADS regressions', () => { + const recovery = readWorkflow('windows-recovery.yml'); + + assert.match(recovery, /name: Verify managed dependency alternate streams/u); + assert.match(recovery, /--test-name-pattern="NTFS alternate stream"/u); + assert.match( + recovery, + /packages\/storage\/dist\/__tests__\/managed-dependency-environment\.test\.js/u, + ); + assert.match(recovery, /# tests 3/u); + assert.match(recovery, /# pass 3/u); + assert.match(recovery, /# skipped 0/u); +}); + test('workflows never persist the job credential into the checkout', () => { for (const name of readdirSync(WORKFLOW_DIR)) { for (const step of checkoutSteps(name)) { diff --git a/scripts/product-release.test.mjs b/scripts/product-release.test.mjs index d203ab4aa4..0a3754e071 100644 --- a/scripts/product-release.test.mjs +++ b/scripts/product-release.test.mjs @@ -724,6 +724,10 @@ test('one product workflow gates one draft release on every required artifact', test('repository control plane admits only reviewed immutable release tags', async () => { const config = parseYaml(await readFile(new URL('../.asf.yaml', import.meta.url), 'utf8')); + assert.deepEqual(config.github.protected_branches.main.required_status_checks.contexts, [ + 'test', + 'windows_recovery', + ]); const environments = config.github.environments; for (const [name, tagPattern] of [ ['release', 'v*-incubating-rc*'],