diff --git a/packages/diffs/moon.yml b/packages/diffs/moon.yml index 2673c84d8..b2d72d286 100644 --- a/packages/diffs/moon.yml +++ b/packages/diffs/moon.yml @@ -21,6 +21,18 @@ tasks: cache: false runInCI: 'always' + benchmark-editor-piece-table: + command: 'bun scripts/benchmarkEditorPieceTable.ts' + options: + cache: false + runInCI: 'always' + + benchmark-editor-tokenizer: + command: 'bun scripts/benchmarkEditorTokenizer.ts' + options: + cache: false + runInCI: 'always' + # Browser E2E for shadow-DOM rendering and contentEditable edit mode, which # the jsdom unit suite cannot exercise. Mirrors the trees E2E setup: a Vite # fixture server serves the built dist and Playwright drives real Chromium. diff --git a/packages/diffs/scripts/benchmarkEditorPieceTable.ts b/packages/diffs/scripts/benchmarkEditorPieceTable.ts new file mode 100644 index 000000000..f615283ea --- /dev/null +++ b/packages/diffs/scripts/benchmarkEditorPieceTable.ts @@ -0,0 +1,194 @@ +import { PieceTable } from '../src/editor/pieceTable'; + +const line = 'function benchmark(value: number) { return value + 1; }\n'; +const lineCount = 12_000; +const text = line.repeat(lineCount); +const editCount = 2_000; +const scatteredOffsets = Array.from({ length: editCount }, (_, i) => { + const lineIndex = (i * 7919) % lineCount; + return lineIndex * line.length + 9 + (i % 20); +}); +const batchEdits = Array.from({ length: editCount }, (_, i) => { + const lineIndex = Math.floor(((i + 1) * lineCount) / (editCount + 1)); + const start = lineIndex * line.length + 20; + return { start, end: start + 1, text: i % 2 === 0 ? 'x' : 'y' }; +}); +const lookupOffsets = Array.from( + { length: 20_000 }, + (_, i) => (i * 104729) % text.length +); +const sliceOffsets = Array.from( + { length: 5_000 }, + (_, i) => (i * 48611) % (text.length - 80) +); + +const createFragmentedTable = () => { + const table = new PieceTable(text); + for (let i = 0; i < scatteredOffsets.length; i++) { + const start = scatteredOffsets[i]; + table.applyEdits([ + { start, end: start + 1, text: i % 2 === 0 ? 'q' : 'z' }, + ]); + } + return table; +}; + +const scenarios: { + name: string; + operations: number; + prepare: () => () => number; +}[] = [ + { + name: 'construct/large-document', + operations: 1, + prepare: () => () => new PieceTable(text).lineCount, + }, + { + name: 'edit/sequential-typing', + operations: 5_000, + prepare: () => { + const table = new PieceTable(''); + return () => { + for (let i = 0; i < 5_000; i++) { + table.insert('x', i); + } + return table.getText().length; + }; + }, + }, + { + name: 'edit/scattered-inserts', + operations: scatteredOffsets.length, + prepare: () => { + const table = new PieceTable(text); + return () => { + for (const offset of scatteredOffsets) { + table.insert('x', offset); + } + return table.lineCount; + }; + }, + }, + { + name: 'edit/append-after-fragmentation', + operations: 2_000, + prepare: () => { + const table = createFragmentedTable(); + const end = text.length; + return () => { + for (let i = 0; i < 2_000; i++) { + table.insert('x', end + i); + } + return table.lineCount; + }; + }, + }, + { + name: 'edit/batch-replacements', + operations: batchEdits.length, + prepare: () => { + const table = new PieceTable(text); + return () => { + table.applyEdits(batchEdits); + return table.lineCount; + }; + }, + }, + { + name: 'read/fragmented-position-round-trips', + operations: lookupOffsets.length, + prepare: () => { + const table = createFragmentedTable(); + return () => { + let checksum = 0; + for (const offset of lookupOffsets) { + const position = table.positionAt(offset); + checksum += table.offsetAt(position); + } + return checksum; + }; + }, + }, + { + name: 'read/fragmented-slices', + operations: sliceOffsets.length, + prepare: () => { + const table = createFragmentedTable(); + return () => { + let checksum = 0; + for (const start of sliceOffsets) { + checksum += table.getTextSlice(start, start + 80).length; + } + return checksum; + }; + }, + }, + { + name: 'search/fragmented-whole-word', + operations: lineCount, + prepare: () => { + const table = createFragmentedTable(); + return () => + table.search({ + text: 'function', + replaceText: '', + caseSensitive: true, + wholeWord: true, + regex: false, + }).length; + }, + }, +]; + +const results: { + name: string; + operations: number; + p50Ms: number; + p95Ms: number; + minMs: number; + maxMs: number; + opsPerSecond: number; + samplesMs: number[]; +}[] = []; +let checksum = 0; +for (const scenario of scenarios) { + for (let i = 0; i < 2; i++) { + checksum += scenario.prepare()(); + } + + const samplesMs: number[] = []; + for (let i = 0; i < 9; i++) { + const run = scenario.prepare(); + const start = performance.now(); + checksum += run(); + samplesMs.push(performance.now() - start); + } + samplesMs.sort((a, b) => a - b); + const p50Ms = samplesMs[Math.floor(samplesMs.length / 2)]; + const p95Ms = samplesMs[Math.ceil(samplesMs.length * 0.95) - 1]; + results.push({ + name: scenario.name, + operations: scenario.operations, + p50Ms, + p95Ms, + minMs: samplesMs[0], + maxMs: samplesMs[samplesMs.length - 1], + opsPerSecond: scenario.operations / (p50Ms / 1000), + samplesMs, + }); +} + +if (checksum <= 0) { + throw new Error('PieceTable benchmark produced an invalid checksum'); +} +console.table( + results.map(({ name, p50Ms, p95Ms, minMs, maxMs, opsPerSecond }) => ({ + name, + p50Ms: p50Ms.toFixed(3), + p95Ms: p95Ms.toFixed(3), + minMs: minMs.toFixed(3), + maxMs: maxMs.toFixed(3), + opsPerSecond: Math.round(opsPerSecond), + })) +); +console.log(`PIECE_TABLE_BENCHMARK_RESULTS=${JSON.stringify(results)}`); diff --git a/packages/diffs/scripts/benchmarkEditorTokenizer.ts b/packages/diffs/scripts/benchmarkEditorTokenizer.ts new file mode 100644 index 000000000..e407e6738 --- /dev/null +++ b/packages/diffs/scripts/benchmarkEditorTokenizer.ts @@ -0,0 +1,766 @@ +import { type IGrammar, type StateStack } from 'shiki/textmate'; + +import { TextDocument } from '../src/editor/textDocument'; +import { EditorTokenizer } from '../src/editor/tokenizer'; +import type { DiffsHighlighter, RenderRange } from '../src/types'; + +interface BenchmarkConfig { + lines: number; + runs: number; + warmupRuns: number; + outputJson: boolean; +} + +interface BenchmarkSample { + elapsedMs: number; + operations: Record; +} + +interface BenchmarkCase { + name: string; + description: string; + run: () => BenchmarkSample; +} + +interface BenchmarkSummary { + name: string; + description: string; + runs: number; + meanMs: number; + medianMs: number; + p95Ms: number; + minMs: number; + maxMs: number; + operations: Record; +} + +interface MockCounters { + grammarCalls: number; + setThemeCalls: number; +} + +const DEFAULT_CONFIG: BenchmarkConfig = { + lines: 100_000, + runs: 10, + warmupRuns: 3, + outputJson: false, +}; + +const TOKEN_DATA = new Uint32Array([0, 0]); +const messageListeners = new Set(); +const postedMessages: unknown[] = []; +let mutationCallback: ((records: MutationRecord[]) => void) | undefined; + +Reflect.set(globalThis, 'window', globalThis); +Reflect.set(globalThis, 'matchMedia', () => ({ + addEventListener() {}, + addListener() {}, + dispatchEvent() { + return false; + }, + matches: true, + media: '(prefers-color-scheme: dark)', + onchange: null, + removeEventListener() {}, + removeListener() {}, +})); +Reflect.set(globalThis, 'document', { + body: {}, + documentElement: {}, +}); +Reflect.set(globalThis, 'getComputedStyle', () => ({ + colorScheme: 'dark', +})); +Reflect.set( + globalThis, + 'MutationObserver', + class { + constructor(callback: MutationCallback) { + mutationCallback = (records) => + callback(records, this as unknown as MutationObserver); + } + + observe() {} + + disconnect() {} + + takeRecords() { + return []; + } + } +); +Reflect.set( + globalThis, + 'addEventListener', + (type: string, listener: EventListenerOrEventListenerObject) => { + if (type === 'message') { + messageListeners.add(listener); + } + } +); +Reflect.set( + globalThis, + 'removeEventListener', + (type: string, listener: EventListenerOrEventListenerObject) => { + if (type === 'message') { + messageListeners.delete(listener); + } + } +); +Reflect.set(globalThis, 'postMessage', (message: unknown) => { + postedMessages.push(message); +}); + +function parseInteger( + value: string, + flagName: string, + allowZero = false +): number { + const parsed = Number.parseInt(value, 10); + if (!Number.isFinite(parsed) || parsed < (allowZero ? 0 : 1)) { + throw new Error( + `Invalid ${flagName} value "${value}". Expected ${allowZero ? 'a non-negative' : 'a positive'} integer.` + ); + } + return parsed; +} + +function parseArgs(argv: string[]): BenchmarkConfig { + const config = { ...DEFAULT_CONFIG }; + for (let index = 0; index < argv.length; index++) { + const rawArg = argv[index]; + if (rawArg === '--help' || rawArg === '-h') { + console.log('Usage: moonx diffs:benchmark-editor-tokenizer -- [options]'); + console.log(''); + console.log('Options:'); + console.log( + ` --lines Document lines (default: ${DEFAULT_CONFIG.lines})` + ); + console.log( + ` --runs Measured runs per case (default: ${DEFAULT_CONFIG.runs})` + ); + console.log( + ` --warmup-runs Warmup runs per case (default: ${DEFAULT_CONFIG.warmupRuns})` + ); + console.log(' --json Emit machine-readable JSON'); + console.log(' -h, --help Show this help output'); + process.exit(0); + } + if (rawArg === '--json') { + config.outputJson = true; + continue; + } + + const [flag, inlineValue] = rawArg.split('=', 2); + const value = inlineValue ?? argv[index + 1]; + if (flag === '--lines' || flag === '--runs' || flag === '--warmup-runs') { + if (value === undefined) { + throw new Error(`Missing value for ${flag}`); + } + if (inlineValue === undefined) { + index++; + } + const parsed = parseInteger(value, flag, flag === '--warmup-runs'); + if (flag === '--lines') { + config.lines = parsed; + } else if (flag === '--runs') { + config.runs = parsed; + } else { + config.warmupRuns = parsed; + } + continue; + } + throw new Error(`Unknown argument: ${rawArg}`); + } + if (config.lines < 2) { + throw new Error('--lines must be at least 2'); + } + return config; +} + +function percentile(sortedValues: number[], rank: number): number { + const index = (sortedValues.length - 1) * rank; + const lowerIndex = Math.floor(index); + const upperIndex = Math.ceil(index); + const lower = sortedValues[lowerIndex] ?? 0; + const upper = sortedValues[upperIndex] ?? lower; + return lower + (upper - lower) * (index - lowerIndex); +} + +function createTokenizer( + textDocument: TextDocument, + counters: MockCounters, + options: { + matchBrackets?: boolean; + systemTheme?: boolean; + onThemeChange?: () => void; + } = {} +): EditorTokenizer { + const grammar = { + tokenizeLine2(_lineText: string, ruleStack: StateStack) { + counters.grammarCalls++; + return { + tokens: TOKEN_DATA, + ruleStack, + stoppedEarly: false, + }; + }, + } as unknown as IGrammar; + const highlighter = { + getLanguage: () => grammar, + getLoadedLanguages: () => ['typescript'], + getTheme: () => ({ colors: {} }), + setTheme: (themeName: string) => { + counters.setThemeCalls++; + return { + colorMap: [''], + theme: { name: themeName, type: 'dark' }, + }; + }, + } as unknown as DiffsHighlighter; + return new EditorTokenizer({ + highlighter, + textDocument, + codeOptions: + options.systemTheme === true + ? { + theme: { dark: 'dark-theme', light: 'light-theme' }, + themeType: 'system', + } + : { theme: 'dark-theme', themeType: 'dark' }, + matchBrackets: options.matchBrackets, + setStyle() {}, + onDeferTokenize() {}, + onThemeChange: options.onThemeChange, + }); +} + +function primeTokenizer( + tokenizer: EditorTokenizer, + textDocument: TextDocument +): void { + const lineCount = textDocument.lineCount; + tokenizer.tokenize( + { + startLine: 0, + startCharacter: 0, + endCharacter: 0, + endLine: lineCount - 1, + endedAtDocumentEnd: false, + previousLineCount: lineCount, + lineCount, + lineDelta: 0, + changedLineRanges: [[0, lineCount - 1]], + }, + { + startingLine: 0, + totalLines: lineCount, + bufferBefore: 0, + bufferAfter: 0, + } + ); +} + +function resetMessages(): void { + if (messageListeners.size > 0) { + throw new Error('A benchmark case leaked a background message listener'); + } + postedMessages.length = 0; +} + +function dispatchMessage(data: unknown): void { + const event = { data } as MessageEvent; + for (const listener of [...messageListeners]) { + if (typeof listener === 'function') { + listener(event); + } else { + listener.handleEvent(event); + } + } +} + +function drainMessages(counters: MockCounters, maxMessages: number): number { + const originalPerformanceNow = performance.now; + let messageIndex = 0; + Object.defineProperty(performance, 'now', { + configurable: true, + value: () => counters.grammarCalls / 1000, + }); + try { + while (messageIndex < postedMessages.length) { + if (messageIndex >= maxMessages) { + throw new Error('Background tokenizer did not settle'); + } + dispatchMessage(postedMessages[messageIndex]); + messageIndex++; + } + } finally { + Object.defineProperty(performance, 'now', { + configurable: true, + value: originalPerformanceNow, + }); + } + return messageIndex; +} + +function collectGarbage(): void { + if (typeof Bun !== 'undefined') { + Bun.gc(true); + } +} + +function createBenchmarkCases( + config: BenchmarkConfig, + sourceText: string +): BenchmarkCase[] { + const fullRange: RenderRange = { + startingLine: 0, + totalLines: config.lines, + bufferBefore: 0, + bufferAfter: 0, + }; + + return [ + { + name: 'unbounded-structural-edit', + description: + 'Insert a newline at line 0 after fully populating tokenizer state.', + run() { + resetMessages(); + const counters = { grammarCalls: 0, setThemeCalls: 0 }; + const textDocument = new TextDocument( + 'structural.ts', + sourceText, + 'typescript' + ); + const tokenizer = createTokenizer(textDocument, counters, { + matchBrackets: false, + }); + primeTokenizer(tokenizer, textDocument); + const change = textDocument.applyEdits([ + { + range: { + start: { line: 0, character: 0 }, + end: { line: 0, character: 0 }, + }, + newText: 'inserted\n', + }, + ]); + if (change === undefined) { + throw new Error( + 'Expected the structural edit to change the document' + ); + } + counters.grammarCalls = 0; + postedMessages.length = 0; + collectGarbage(); + const started = performance.now(); + const dirtyLines = tokenizer.tokenize(change); + const elapsedMs = performance.now() - started; + const operations = { + grammarCalls: counters.grammarCalls, + dirtyLines: dirtyLines.size, + backgroundMessages: postedMessages.length, + }; + tokenizer.cleanUp(); + return { elapsedMs, operations }; + }, + }, + { + name: 'deep-state-prebuild', + description: + 'Start grammar-state preparation for a viewport at the document tail.', + run() { + resetMessages(); + const counters = { grammarCalls: 0, setThemeCalls: 0 }; + const textDocument = new TextDocument( + 'prebuild.ts', + sourceText, + 'typescript' + ); + const tokenizer = createTokenizer(textDocument, counters, { + matchBrackets: false, + }); + collectGarbage(); + const started = performance.now(); + tokenizer.prebuildStateStack({ + ...fullRange, + startingLine: config.lines - 1, + totalLines: 1, + }); + const elapsedMs = performance.now() - started; + const synchronousGrammarCalls = counters.grammarCalls; + const backgroundMessages = drainMessages( + counters, + textDocument.lineCount + 1 + ); + const operations = { + synchronousGrammarCalls, + backgroundMessages, + totalGrammarCalls: counters.grammarCalls, + }; + tokenizer.cleanUp(); + return { elapsedMs, operations }; + }, + }, + { + name: 'bracket-cache-invalidation', + description: + 'Edit one character at line 0 after caching bracket ranges for every line.', + run() { + resetMessages(); + const counters = { grammarCalls: 0, setThemeCalls: 0 }; + const textDocument = new TextDocument( + 'brackets.ts', + sourceText, + 'typescript' + ); + const tokenizer = createTokenizer(textDocument, counters, { + matchBrackets: true, + }); + primeTokenizer(tokenizer, textDocument); + const change = textDocument.applyEdits([ + { + range: { + start: { line: 0, character: 0 }, + end: { line: 0, character: 1 }, + }, + newText: 'L', + }, + ]); + if (change === undefined) { + throw new Error('Expected the same-line edit to change the document'); + } + counters.grammarCalls = 0; + collectGarbage(); + const started = performance.now(); + const dirtyLines = tokenizer.tokenize(change, { + ...fullRange, + totalLines: 1, + }); + const elapsedMs = performance.now() - started; + const operations = { + grammarCalls: counters.grammarCalls, + dirtyLines: dirtyLines.size, + }; + tokenizer.cleanUp(); + return { elapsedMs, operations }; + }, + }, + { + name: 'comparison-state-remap-no-baseline', + description: + 'Informational only: remap and converge many structural edits; no permanent pre-fix baseline was captured.', + run() { + resetMessages(); + const counters = { grammarCalls: 0, setThemeCalls: 0 }; + const textDocument = new TextDocument( + 'remap.ts', + sourceText, + 'typescript' + ); + const tokenizer = createTokenizer(textDocument, counters, { + matchBrackets: false, + }); + primeTokenizer(tokenizer, textDocument); + const editCount = Math.min( + 128, + config.lines, + Math.max(2, Math.floor(config.lines / 1000)) + ); + const edits = Array.from({ length: editCount }, (_, index) => { + const line = Math.floor( + ((index + 1) * config.lines) / (editCount + 1) + ); + return { + range: { + start: { line, character: 0 }, + end: { line, character: 0 }, + }, + newText: 'inserted\n', + }; + }); + const change = textDocument.applyEdits(edits); + if (change === undefined) { + throw new Error('Expected structural edits to change the document'); + } + counters.grammarCalls = 0; + postedMessages.length = 0; + collectGarbage(); + const started = performance.now(); + tokenizer.tokenize(change, { + startingLine: change.startLine, + totalLines: 1, + bufferBefore: 0, + bufferAfter: 0, + }); + const elapsedMs = performance.now() - started; + const synchronousGrammarCalls = counters.grammarCalls; + const backgroundMessages = drainMessages( + counters, + textDocument.lineCount + 1 + ); + const operations = { + edits: editCount, + changedLineChanges: change.changedLineChanges?.length ?? 0, + synchronousGrammarCalls, + backgroundMessages, + totalGrammarCalls: counters.grammarCalls, + }; + tokenizer.cleanUp(); + return { elapsedMs, operations }; + }, + }, + { + name: 'no-op-theme-mutation', + description: + 'Notify an unchanged system-theme tokenizer of a root class mutation.', + run() { + resetMessages(); + mutationCallback = undefined; + let themeChangeCallbacks = 0; + const counters = { grammarCalls: 0, setThemeCalls: 0 }; + const textDocument = new TextDocument( + 'theme.ts', + 'line 0\nline 1', + 'typescript' + ); + const tokenizer = createTokenizer(textDocument, counters, { + matchBrackets: false, + systemTheme: true, + onThemeChange: () => { + themeChangeCallbacks++; + }, + }); + const callback = mutationCallback; + if (callback === undefined) { + throw new Error('Expected a system-theme MutationObserver'); + } + counters.setThemeCalls = 0; + collectGarbage(); + const started = performance.now(); + callback([ + { + attributeName: 'class', + type: 'attributes', + } as MutationRecord, + ]); + const elapsedMs = performance.now() - started; + const operations = { + backgroundRestarts: postedMessages.length, + setThemeCalls: counters.setThemeCalls, + themeChangeCallbacks, + }; + tokenizer.cleanUp(); + mutationCallback = undefined; + return { elapsedMs, operations }; + }, + }, + { + name: 'multi-instance-message-collision', + description: + 'Dispatch one tokenizer job message while two tokenizer jobs are active.', + run() { + resetMessages(); + const counters = [ + { grammarCalls: 0, setThemeCalls: 0 }, + { grammarCalls: 0, setThemeCalls: 0 }, + ]; + const tokenizers = counters.map((counter, index) => { + const textDocument = new TextDocument( + `collision-${index}.ts`, + 'line 0\nline 1', + 'typescript' + ); + const tokenizer = createTokenizer(textDocument, counter, { + matchBrackets: false, + }); + tokenizer.tokenize( + { + startLine: 0, + startCharacter: 0, + endCharacter: 0, + endLine: 1, + endedAtDocumentEnd: false, + previousLineCount: 2, + lineCount: 2, + lineDelta: 0, + changedLineRanges: [[0, 1]], + }, + { + startingLine: 0, + totalLines: 1, + bufferBefore: 0, + bufferAfter: 0, + } + ); + return tokenizer; + }); + const firstMessage = postedMessages[0]; + if (firstMessage === undefined || messageListeners.size !== 2) { + throw new Error('Expected two active tokenizer background jobs'); + } + counters[0].grammarCalls = 0; + counters[1].grammarCalls = 0; + collectGarbage(); + const started = performance.now(); + dispatchMessage(firstMessage); + const elapsedMs = performance.now() - started; + const operations = { + sourceGrammarCalls: counters[0].grammarCalls, + foreignGrammarCalls: counters[1].grammarCalls, + triggeredInstances: + Number(counters[0].grammarCalls > 0) + + Number(counters[1].grammarCalls > 0), + }; + for (const tokenizer of tokenizers) { + tokenizer.cleanUp(); + } + return { elapsedMs, operations }; + }, + }, + ]; +} + +function summarize( + benchmarkCase: BenchmarkCase, + samples: BenchmarkSample[] +): BenchmarkSummary { + const elapsed = samples + .map((sample) => sample.elapsedMs) + .sort((left, right) => left - right); + const operations = samples[0]?.operations ?? {}; + const expectedOperations = JSON.stringify(operations); + for (const sample of samples) { + if (JSON.stringify(sample.operations) !== expectedOperations) { + throw new Error( + `Non-deterministic operation counts in ${benchmarkCase.name}` + ); + } + } + const total = elapsed.reduce((sum, value) => sum + value, 0); + return { + name: benchmarkCase.name, + description: benchmarkCase.description, + runs: elapsed.length, + meanMs: total / elapsed.length, + medianMs: percentile(elapsed, 0.5), + p95Ms: percentile(elapsed, 0.95), + minMs: elapsed[0] ?? 0, + maxMs: elapsed[elapsed.length - 1] ?? 0, + operations, + }; +} + +function printTable(summaries: BenchmarkSummary[]): void { + const rows = summaries.map((summary) => ({ + scenario: summary.name, + median: summary.medianMs.toFixed(3), + p95: summary.p95Ms.toFixed(3), + min: summary.minMs.toFixed(3), + max: summary.maxMs.toFixed(3), + operations: Object.entries(summary.operations) + .map(([key, value]) => `${key}=${value}`) + .join(','), + })); + const headers = [ + 'scenario', + 'median', + 'p95', + 'min', + 'max', + 'operations', + ] as const; + const widths = headers.map((header) => + rows.reduce( + (width, row) => Math.max(width, row[header].length), + header.length + ) + ); + const formatRow = (row: Record<(typeof headers)[number], string>) => + headers + .map((header, index) => row[header].padEnd(widths[index])) + .join(' ') + .trimEnd(); + console.log(formatRow(Object.fromEntries(headers.map((key) => [key, key])))); + console.log(widths.map((width) => '-'.repeat(width)).join(' ')); + for (const row of rows) { + console.log(formatRow(row)); + } +} + +function main(): void { + const config = parseArgs(process.argv.slice(2)); + const sourceText = Array.from( + { length: config.lines }, + (_, index) => `const value${index} = ${index};` + ).join('\n'); + const benchmarkCases = createBenchmarkCases(config, sourceText); + const samples = benchmarkCases.map(() => [] as BenchmarkSample[]); + let checksum = 0; + + for (let run = 0; run < config.warmupRuns; run++) { + for (let offset = 0; offset < benchmarkCases.length; offset++) { + benchmarkCases[(run + offset) % benchmarkCases.length].run(); + } + } + for (let run = 0; run < config.runs; run++) { + for (let offset = 0; offset < benchmarkCases.length; offset++) { + const caseIndex = (run + offset) % benchmarkCases.length; + const sample = benchmarkCases[caseIndex].run(); + samples[caseIndex].push(sample); + checksum += Object.values(sample.operations).reduce( + (sum, value) => sum + value, + 0 + ); + } + } + + const summaries = benchmarkCases.map((benchmarkCase, index) => + summarize(benchmarkCase, samples[index]) + ); + const result = { + benchmark: 'EditorTokenizer', + config, + measurement: { + setupExcluded: true, + forcedGcBeforeTiming: true, + prebuildDebounceExcluded: true, + backgroundDrainExcluded: true, + backgroundDrainClock: '1ms per 1000 mock grammar calls', + grammar: 'deterministic one-token mock', + }, + checksum, + summaries, + }; + if (config.outputJson) { + console.log(JSON.stringify(result, null, 2)); + return; + } + + console.log('EditorTokenizer benchmark'); + console.log( + `lines=${config.lines} runsPerCase=${config.runs} warmupRunsPerCase=${config.warmupRuns}` + ); + console.log( + 'Setup, forced GC, and the prebuild debounce delay are excluded from timings.' + ); + console.log(''); + printTable(summaries); + console.log(''); + for (const summary of summaries) { + console.log(`${summary.name}: ${summary.description}`); + } + console.log(`checksum=${checksum}`); +} + +const originalSetTimeout = globalThis.setTimeout; +Reflect.set(globalThis, 'setTimeout', ((callback: () => void) => { + callback(); + return 0; +}) as unknown as typeof setTimeout); +try { + main(); +} finally { + Reflect.set(globalThis, 'setTimeout', originalSetTimeout); +} diff --git a/packages/diffs/src/editor/tokenizer.ts b/packages/diffs/src/editor/tokenizer.ts index d72189a8d..10e72ca7c 100644 --- a/packages/diffs/src/editor/tokenizer.ts +++ b/packages/diffs/src/editor/tokenizer.ts @@ -17,6 +17,7 @@ import type { TextDocument, TextDocumentChange } from './textDocument'; import { addEventListener, debounce, h } from './utils'; const TOKENIZE_TIME_LIMIT = 500; +let nextTokenizerId = 0; export interface EditorTokenizerProps { highlighter: DiffsHighlighter; @@ -59,13 +60,20 @@ export class EditorTokenizer { // state #stateStack: StateStack[] = [INITIAL]; // cached state stack by line index #comparisonStateStack: StateStack[] = []; + #comparisonStateStackStart = 0; + #comparisonLineChanges: NonNullable< + TextDocumentChange['changedLineChanges'] + > = []; #lastLine: number = -1; #isStopped: boolean = true; #isPaused: boolean = false; #backgroundJobId: number = 0; + #tokenizerId = ++nextTokenizerId; + #backgroundPrebuildEndLine = -1; + #pendingPrebuildEndLine = -1; #backgroundChangedLineRanges: readonly [number, number][] | undefined; #backgroundChangedRangeIndex: number = 0; - #bracketIgnoredRanges: Map = new Map(); + #bracketIgnoredRanges: ([number, number][] | null | undefined)[] = []; #isMessageListenerAttached: boolean = false; #prebuildStateStack = debounce(async (renderRange?: RenderRange) => { @@ -92,23 +100,29 @@ export class EditorTokenizer { ); } this.#ensureActiveTheme(); - this.#buildStateStack(endLine); + this.#scheduleStatePrebuild(endLine); }, 500); #onMessage = ({ data }: MessageEvent) => { if (typeof data !== 'object' || data === null) { return; } - const { type, jobId } = data as { + const { type, tokenizerId, jobId } = data as { type?: unknown; + tokenizerId?: unknown; jobId?: unknown; }; if ( type === 'tokenize' && + tokenizerId === this.#tokenizerId && typeof jobId === 'number' && jobId === this.#backgroundJobId ) { - this.#backgroundTokenize(jobId); + if (this.#backgroundPrebuildEndLine >= 0) { + this.#backgroundPrebuild(jobId); + } else { + this.#backgroundTokenize(jobId); + } } }; @@ -130,13 +144,13 @@ export class EditorTokenizer { if (this.#grammar === undefined) { return null; } - if (!this.#bracketIgnoredRanges.has(lineIndex)) { + if (this.#bracketIgnoredRanges[lineIndex] === undefined) { this.#buildStateStack(lineIndex); const state = this.#stateStack[lineIndex] ?? INITIAL; const result = this.#tokenizeLineAt(lineIndex, state); this.#stateStack[lineIndex + 1] = result.state; } - return this.#bracketIgnoredRanges.get(lineIndex) ?? null; + return this.#bracketIgnoredRanges[lineIndex] ?? null; } constructor({ @@ -213,10 +227,15 @@ export class EditorTokenizer { // By default, diffs components support dual themes, but the tokenizer only renders // the preferred theme. When the theme type is changed, the tokenizer will re-tokenize the document. #emitThemeChange(themeName: string, themeType: 'light' | 'dark') { + if (themeName === this.#themeName && themeType === this.#themeType) { + return; + } this.#setTheme(themeName, themeType); this.stopBackgroundTokenize(); this.#stateStack = [INITIAL]; this.#comparisonStateStack = []; + this.#comparisonStateStackStart = 0; + this.#comparisonLineChanges = []; if (this.#grammar !== undefined && this.#textDocument.lineCount > 0) { this.#scheduleBackgroundTokenize(0); } @@ -364,11 +383,10 @@ export class EditorTokenizer { if (this.#matchBrackets) { // Clear ignored token ranges for lines invalidated by the edit. - for (const line of this.#bracketIgnoredRanges.keys()) { - if (line >= change.startLine) { - this.#bracketIgnoredRanges.delete(line); - } - } + this.#bracketIgnoredRanges.length = Math.min( + this.#bracketIgnoredRanges.length, + change.startLine + ); } const { lineCount } = this.#textDocument; @@ -386,7 +404,10 @@ export class EditorTokenizer { change.lineDelta > 0 && dirtyStart < renderRangeEndLine && change.endLine >= renderRangeEndLine; - const canReuseCachedStates = change.lineDelta === 0; + const canReuseCachedStates = + change.lineDelta === 0 && + (change.changedLineChanges?.every(([, , lineDelta]) => lineDelta === 0) ?? + true); const canCacheTokenizedStates = canReuseCachedStates || renderRange === undefined || @@ -394,6 +415,8 @@ export class EditorTokenizer { const changedLineRanges: readonly [number, number][] = change.changedLineRanges ?? [[dirtyStart, change.endLine]]; this.#comparisonStateStack = []; + this.#comparisonStateStackStart = 0; + this.#comparisonLineChanges = []; let offscreenSyncEnd = -1; if (dirtyStart < viewStart) { for (const [rangeStart, rangeEnd] of changedLineRanges) { @@ -547,15 +570,19 @@ export class EditorTokenizer { } stopBackgroundTokenize(): void { + this.#pendingPrebuildEndLine = -1; if (this.#isStopped) { return; } this.#isStopped = true; this.#isPaused = false; this.#lastLine = -1; + this.#backgroundPrebuildEndLine = -1; this.#backgroundChangedLineRanges = undefined; this.#backgroundChangedRangeIndex = 0; this.#comparisonStateStack = []; + this.#comparisonStateStackStart = 0; + this.#comparisonLineChanges = []; this.#detachMessageListener(); } @@ -621,7 +648,11 @@ export class EditorTokenizer { #postTokenizeMessage(jobId: number): void { // use `postMessage` instead of `setTimeout(fn, 0)` to avoid 4ms delay - globalThis.postMessage({ type: 'tokenize', jobId }); + globalThis.postMessage({ + type: 'tokenize', + tokenizerId: this.#tokenizerId, + jobId, + }); } #scheduleBackgroundTokenize( @@ -647,12 +678,52 @@ export class EditorTokenizer { this.#isStopped = false; this.#isPaused = false; this.#lastLine = startLine; + if (this.#backgroundPrebuildEndLine >= 0) { + this.#pendingPrebuildEndLine = Math.max( + this.#pendingPrebuildEndLine, + this.#backgroundPrebuildEndLine + ); + } + this.#backgroundPrebuildEndLine = -1; this.#backgroundChangedLineRanges = changedLineRanges; this.#backgroundChangedRangeIndex = changedRangeIndex; this.#attachMessageListener(); this.#postTokenizeMessage(jobId); } + #scheduleStatePrebuild(endLine: number): void { + if (this.#grammar === undefined || this.#stateStack.length > endLine) { + return; + } + // Extend an active prebuild, or retain the target until the foreground + // edit job reconverges and releases the state cache. + if (!this.#isStopped) { + if (this.#backgroundPrebuildEndLine >= 0) { + this.#backgroundPrebuildEndLine = Math.max( + this.#backgroundPrebuildEndLine, + endLine + ); + } else { + this.#pendingPrebuildEndLine = Math.max( + this.#pendingPrebuildEndLine, + endLine + ); + } + return; + } + + const jobId = ++this.#backgroundJobId; + this.#isStopped = false; + this.#isPaused = false; + this.#lastLine = this.#stateStack.length - 1; + this.#backgroundPrebuildEndLine = endLine; + this.#pendingPrebuildEndLine = -1; + this.#backgroundChangedLineRanges = undefined; + this.#backgroundChangedRangeIndex = 0; + this.#attachMessageListener(); + this.#postTokenizeMessage(jobId); + } + #tokenizeLineAt( line: number, state: StateStack @@ -693,88 +764,70 @@ export class EditorTokenizer { ranges: [number, number][] | null ): void { if (this.#matchBrackets) { - this.#bracketIgnoredRanges.set(line, ranges); + this.#bracketIgnoredRanges[line] = ranges; } } - // Preserve old end states as comparison-only sentinels. They let background - // tokenization stop when grammar state reconverges without letting foreground - // tokenization seed from stale pre-edit states. + // Preserve old end states as comparison-only sentinels, copying whichever + // side of the edit is smaller. Index shifts are resolved lazily on lookup. #shiftComparisonStateStack(change: TextDocumentChange): void { const lineChanges = change.changedLineChanges ?? ([[change.startLine, change.endLine, change.lineDelta]] as const); - const comparisonStateStack = this.#stateStack.slice(); + const comparisonStart = change.startLine + 1; + const comparisonLength = this.#stateStack.length - comparisonStart; + if (comparisonStart <= comparisonLength) { + this.#comparisonStateStack = this.#stateStack; + this.#comparisonStateStackStart = 0; + this.#stateStack = this.#stateStack.slice(0, comparisonStart); + } else { + this.#comparisonStateStackStart = comparisonStart; + this.#comparisonStateStack = this.#stateStack.slice(comparisonStart); + this.#stateStack.length = Math.min( + this.#stateStack.length, + comparisonStart + ); + } + this.#comparisonLineChanges = lineChanges; + } - for (const [startLine, endLine, lineDelta] of lineChanges) { + #getPreviousEndState(line: number): StateStack | undefined { + let previousLine = line; + for ( + let index = this.#comparisonLineChanges.length - 1; + index >= 0; + index-- + ) { + const [startLine, endLine, lineDelta] = + this.#comparisonLineChanges[index]; if (lineDelta === 0) { continue; } - - const insertedLineSpan = endLine - startLine; - const oldLineSpan = insertedLineSpan - lineDelta; - const sourceStart = startLine + oldLineSpan + 1; - const targetStart = startLine + insertedLineSpan + 1; - const originalLength = comparisonStateStack.length; - - if (lineDelta > 0) { - for (let line = originalLength - 1; line >= sourceStart; line--) { - const targetLine = line + lineDelta; - if (line in comparisonStateStack) { - comparisonStateStack[targetLine] = comparisonStateStack[line]; - } else { - Reflect.deleteProperty(comparisonStateStack, targetLine); - } - } - } else { - for (let line = sourceStart; line < originalLength; line++) { - const targetLine = line + lineDelta; - if (line in comparisonStateStack) { - comparisonStateStack[targetLine] = comparisonStateStack[line]; - } else { - Reflect.deleteProperty(comparisonStateStack, targetLine); - } - } - comparisonStateStack.length = Math.max( - Math.min(originalLength, startLine + 1), - originalLength + lineDelta - ); - } - - for (let line = startLine + 1; line < targetStart; line++) { - Reflect.deleteProperty(comparisonStateStack, line); + if (previousLine > endLine) { + previousLine -= lineDelta; + } else if (previousLine > startLine) { + return this.#stateStack[line]; } } - - this.#stateStack.length = Math.min( - this.#stateStack.length, - change.startLine + 1 + return ( + this.#comparisonStateStack[ + previousLine - this.#comparisonStateStackStart + ] ?? this.#stateStack[line] ); - comparisonStateStack.length = Math.min( - comparisonStateStack.length, - this.#textDocument.lineCount + 1 - ); - for (let line = 0; line <= change.startLine; line++) { - Reflect.deleteProperty(comparisonStateStack, line); - } - this.#comparisonStateStack = comparisonStateStack; } - #getPreviousEndState(line: number): StateStack | undefined { - return this.#comparisonStateStack[line] ?? this.#stateStack[line]; - } - - #buildStateStack(endAt: number) { + #buildStateStack(endAt: number, timeBudget?: number): boolean { const boundedEndAt = Math.min( Math.max(0, endAt), this.#textDocument.lineCount ); if (this.#stateStack.length > boundedEndAt || this.#grammar === undefined) { - return; + return true; } + const startedAt = timeBudget === undefined ? 0 : performance.now(); let line = this.#stateStack.length - 1; let state = this.#stateStack[line] ?? INITIAL; - for (; line < boundedEndAt; line++) { + while (line < boundedEndAt) { this.#stateStack[line] = state; const lineText = this.#textDocument.getLineText(line); if ( @@ -788,15 +841,49 @@ export class EditorTokenizer { lineText, state, TOKENIZE_TIME_LIMIT, - this.#matchBrackets + this.#matchBrackets, + false ); this.#cacheBracketIgnoredRanges(line, result.bracketIgnoredRanges); state = result.ruleStack; } else { this.#cacheBracketIgnoredRanges(line, null); } + line++; + this.#stateStack[line] = state; + if ( + timeBudget !== undefined && + performance.now() - startedAt > timeBudget + ) { + break; + } + } + return line >= boundedEndAt; + } + + #backgroundPrebuild(jobId: number): void { + if ( + this.#isStopped || + this.#isPaused || + this.#grammar === undefined || + jobId !== this.#backgroundJobId + ) { + return; + } + + this.#ensureActiveTheme(); + // State prebuilds intentionally omit rendered tokens and yield between + // short chunks so a deep viewport does not monopolize the main thread. + const complete = this.#buildStateStack(this.#backgroundPrebuildEndLine, 1); + if (this.#isStopped || this.#isPaused || jobId !== this.#backgroundJobId) { + return; } - this.#stateStack[line] = state; + if (complete) { + this.stopBackgroundTokenize(); + return; + } + this.#lastLine = this.#stateStack.length - 1; + this.#postTokenizeMessage(jobId); } #backgroundTokenize(jobId: number) { @@ -888,7 +975,11 @@ export class EditorTokenizer { } if (settled || line >= totalLines) { + const pendingPrebuildEndLine = this.#pendingPrebuildEndLine; this.stopBackgroundTokenize(); + if (pendingPrebuildEndLine >= 0) { + this.#scheduleStatePrebuild(pendingPrebuildEndLine); + } return; } @@ -904,7 +995,8 @@ function tokenizeLine( lineText: string, stateStack: StateStack, timeLimit?: number, - collectBracketIgnoredRanges = true + collectBracketIgnoredRanges = true, + resolveTokens = true ): { ruleStack: StateStack; resolvedTokens: Array; @@ -920,6 +1012,13 @@ function tokenizeLine( const tokensLength = rawTokens.length / 2; const resolvedTokens: Array = []; const bracketIgnoredRanges: [number, number][] = []; + if (!resolveTokens && !collectBracketIgnoredRanges) { + return { + ruleStack: result.ruleStack, + resolvedTokens, + bracketIgnoredRanges, + }; + } for (let j = 0; j < tokensLength; j++) { const offset = rawTokens[2 * j]; const nextOffset = @@ -929,9 +1028,14 @@ function tokenizeLine( continue; } const metadata = rawTokens[2 * j + 1]; - const fg = EncodedTokenMetadata.getForeground(metadata); - const tokenText = lineText.slice(offset, nextOffset); - resolvedTokens.push([offset, colorMap[fg], tokenText]); + if (resolveTokens) { + const fg = EncodedTokenMetadata.getForeground(metadata); + resolvedTokens.push([ + offset, + colorMap[fg], + lineText.slice(offset, nextOffset), + ]); + } if ( collectBracketIgnoredRanges && EncodedTokenMetadata.getTokenType(metadata) > 0 diff --git a/packages/diffs/test/editorPieceTable.test.ts b/packages/diffs/test/editorPieceTable.test.ts index 8ce089fb0..d072bea86 100644 --- a/packages/diffs/test/editorPieceTable.test.ts +++ b/packages/diffs/test/editorPieceTable.test.ts @@ -1218,200 +1218,3 @@ describe('very long single line', () => { expect(d.getText()).toBe(str); }); }); - -test.skipIf(process.env.PIECE_TABLE_BENCHMARK !== '1')( - 'benchmarks representative PieceTable workloads', - () => { - const line = 'function benchmark(value: number) { return value + 1; }\n'; - const lineCount = 12_000; - const text = line.repeat(lineCount); - const editCount = 2_000; - const scatteredOffsets = Array.from({ length: editCount }, (_, i) => { - const lineIndex = (i * 7919) % lineCount; - return lineIndex * line.length + 9 + (i % 20); - }); - const batchEdits = Array.from({ length: editCount }, (_, i) => { - const lineIndex = Math.floor(((i + 1) * lineCount) / (editCount + 1)); - const start = lineIndex * line.length + 20; - return { start, end: start + 1, text: i % 2 === 0 ? 'x' : 'y' }; - }); - const lookupOffsets = Array.from( - { length: 20_000 }, - (_, i) => (i * 104729) % text.length - ); - const sliceOffsets = Array.from( - { length: 5_000 }, - (_, i) => (i * 48611) % (text.length - 80) - ); - - const createFragmentedTable = () => { - const table = new PieceTable(text); - for (let i = 0; i < scatteredOffsets.length; i++) { - const start = scatteredOffsets[i]; - table.applyEdits([ - { start, end: start + 1, text: i % 2 === 0 ? 'q' : 'z' }, - ]); - } - return table; - }; - - const scenarios: { - name: string; - operations: number; - prepare: () => () => number; - }[] = [ - { - name: 'construct/large-document', - operations: 1, - prepare: () => () => new PieceTable(text).lineCount, - }, - { - name: 'edit/sequential-typing', - operations: 5_000, - prepare: () => { - const table = new PieceTable(''); - return () => { - for (let i = 0; i < 5_000; i++) { - table.insert('x', i); - } - return table.getText().length; - }; - }, - }, - { - name: 'edit/scattered-inserts', - operations: scatteredOffsets.length, - prepare: () => { - const table = new PieceTable(text); - return () => { - for (const offset of scatteredOffsets) { - table.insert('x', offset); - } - return table.lineCount; - }; - }, - }, - { - name: 'edit/append-after-fragmentation', - operations: 2_000, - prepare: () => { - const table = createFragmentedTable(); - const end = text.length; - return () => { - for (let i = 0; i < 2_000; i++) { - table.insert('x', end + i); - } - return table.lineCount; - }; - }, - }, - { - name: 'edit/batch-replacements', - operations: batchEdits.length, - prepare: () => { - const table = new PieceTable(text); - return () => { - table.applyEdits(batchEdits); - return table.lineCount; - }; - }, - }, - { - name: 'read/fragmented-position-round-trips', - operations: lookupOffsets.length, - prepare: () => { - const table = createFragmentedTable(); - return () => { - let checksum = 0; - for (const offset of lookupOffsets) { - const position = table.positionAt(offset); - checksum += table.offsetAt(position); - } - return checksum; - }; - }, - }, - { - name: 'read/fragmented-slices', - operations: sliceOffsets.length, - prepare: () => { - const table = createFragmentedTable(); - return () => { - let checksum = 0; - for (const start of sliceOffsets) { - checksum += table.getTextSlice(start, start + 80).length; - } - return checksum; - }; - }, - }, - { - name: 'search/fragmented-whole-word', - operations: lineCount, - prepare: () => { - const table = createFragmentedTable(); - return () => - table.search({ - text: 'function', - replaceText: '', - caseSensitive: true, - wholeWord: true, - regex: false, - }).length; - }, - }, - ]; - - const results: { - name: string; - operations: number; - p50Ms: number; - p95Ms: number; - minMs: number; - maxMs: number; - opsPerSecond: number; - samplesMs: number[]; - }[] = []; - let checksum = 0; - for (const scenario of scenarios) { - for (let i = 0; i < 2; i++) { - checksum += scenario.prepare()(); - } - - const samplesMs: number[] = []; - for (let i = 0; i < 9; i++) { - const run = scenario.prepare(); - const start = performance.now(); - checksum += run(); - samplesMs.push(performance.now() - start); - } - samplesMs.sort((a, b) => a - b); - const p50Ms = samplesMs[Math.floor(samplesMs.length / 2)]; - const p95Ms = samplesMs[Math.ceil(samplesMs.length * 0.95) - 1]; - results.push({ - name: scenario.name, - operations: scenario.operations, - p50Ms, - p95Ms, - minMs: samplesMs[0], - maxMs: samplesMs[samplesMs.length - 1], - opsPerSecond: scenario.operations / (p50Ms / 1000), - samplesMs, - }); - } - - expect(checksum).toBeGreaterThan(0); - console.table( - results.map(({ name, p50Ms, p95Ms, minMs, maxMs, opsPerSecond }) => ({ - name, - p50Ms: p50Ms.toFixed(3), - p95Ms: p95Ms.toFixed(3), - minMs: minMs.toFixed(3), - maxMs: maxMs.toFixed(3), - opsPerSecond: Math.round(opsPerSecond), - })) - ); - console.log(`PIECE_TABLE_BENCHMARK_RESULTS=${JSON.stringify(results)}`); - }, - { timeout: 120_000 } -); diff --git a/packages/diffs/test/editorTokenizer.test.ts b/packages/diffs/test/editorTokenizer.test.ts index 76bbfb5a3..c53a6c665 100644 --- a/packages/diffs/test/editorTokenizer.test.ts +++ b/packages/diffs/test/editorTokenizer.test.ts @@ -975,6 +975,266 @@ describe('EditorTokenizer', () => { } }); + test('isolates matching background job ids between tokenizer instances', () => { + const originalAddEventListener = globalThis.addEventListener; + const originalRemoveEventListener = globalThis.removeEventListener; + const originalPostMessage = globalThis.postMessage; + const messageListeners = new Set(); + const postedMessages: unknown[] = []; + const tokenizers: EditorTokenizer[] = []; + + globalThis.addEventListener = (( + type: string, + listener: EventListenerOrEventListenerObject + ) => { + if (type === 'message' && typeof listener === 'function') { + messageListeners.add(listener); + } + }) as typeof globalThis.addEventListener; + globalThis.removeEventListener = (( + type: string, + listener: EventListenerOrEventListenerObject + ) => { + if (type === 'message' && typeof listener === 'function') { + messageListeners.delete(listener); + } + }) as typeof globalThis.removeEventListener; + globalThis.postMessage = ((message: unknown) => { + postedMessages.push(message); + }) as typeof globalThis.postMessage; + + try { + const tokenizeLineCounts = [0, 0]; + for (let index = 0; index < tokenizeLineCounts.length; index++) { + const grammar = { + tokenizeLine2(lineText: string, ruleStack: StateStack) { + tokenizeLineCounts[index]++; + return { + tokens: new Uint32Array([0, 0]), + ruleStack, + stoppedEarly: false, + lineText, + }; + }, + } as unknown as IGrammar; + const textDocument = new TextDocument( + `test-${index}.ts`, + ['line 0', 'line 1', 'line 2'].join('\n'), + 'typescript' + ); + const tokenizer = new EditorTokenizer({ + highlighter: createTestHighlighter({ + getLanguage: () => grammar, + }), + textDocument, + codeOptions: { theme: 'test-theme', themeType: 'dark' }, + setStyle: noopSetStyle, + onDeferTokenize: () => {}, + }); + tokenizers.push(tokenizer); + tokenizer.tokenize( + { + startLine: 0, + startCharacter: 0, + endCharacter: 0, + endLine: 0, + endedAtDocumentEnd: false, + previousLineCount: textDocument.lineCount, + lineCount: textDocument.lineCount, + lineDelta: 0, + changedLineRanges: [[0, 0]], + }, + { + startingLine: 0, + totalLines: 1, + bufferBefore: 0, + bufferAfter: 0, + } + ); + } + + expect(postedMessages).toHaveLength(2); + expect((postedMessages[0] as { jobId: number }).jobId).toBe( + (postedMessages[1] as { jobId: number }).jobId + ); + tokenizeLineCounts.fill(0); + + const event = { data: postedMessages[0] } as MessageEvent; + for (const listener of [...messageListeners]) { + listener(event); + } + + expect(tokenizeLineCounts[0]).toBeGreaterThan(0); + expect(tokenizeLineCounts[1]).toBe(0); + } finally { + tokenizers.forEach((tokenizer) => tokenizer.cleanUp()); + globalThis.addEventListener = originalAddEventListener; + globalThis.removeEventListener = originalRemoveEventListener; + globalThis.postMessage = originalPostMessage; + } + }); + + test('queues state prebuilds and resumes them after foreground work', () => { + const originalAddEventListener = globalThis.addEventListener; + const originalRemoveEventListener = globalThis.removeEventListener; + const originalPostMessage = globalThis.postMessage; + const originalSetTimeout = globalThis.setTimeout; + const originalPerformanceNow = performance.now; + const messageListeners = new Set(); + const postedMessages: unknown[] = []; + let tokenizeLineCount = 0; + + globalThis.addEventListener = (( + type: string, + listener: EventListenerOrEventListenerObject + ) => { + if (type === 'message' && typeof listener === 'function') { + messageListeners.add(listener); + } + }) as typeof globalThis.addEventListener; + globalThis.removeEventListener = (( + type: string, + listener: EventListenerOrEventListenerObject + ) => { + if (type === 'message' && typeof listener === 'function') { + messageListeners.delete(listener); + } + }) as typeof globalThis.removeEventListener; + globalThis.postMessage = ((message: unknown) => { + postedMessages.push(message); + }) as typeof globalThis.postMessage; + globalThis.setTimeout = ((callback: () => void) => { + callback(); + return 0; + }) as unknown as typeof globalThis.setTimeout; + let now = 0; + Object.defineProperty(performance, 'now', { + configurable: true, + value: () => (now += 2), + }); + + const states = new Map(); + const grammar = { + tokenizeLine2(lineText: string) { + let nextState = states.get(lineText); + if (nextState === undefined) { + nextState = { + equals(other: StateStack | null) { + return other === nextState; + }, + } as unknown as StateStack; + states.set(lineText, nextState); + } + tokenizeLineCount++; + return { + tokens: new Uint32Array([0, 0]), + ruleStack: nextState, + stoppedEarly: false, + lineText, + }; + }, + } as unknown as IGrammar; + const textDocument = new TextDocument( + 'test.ts', + Array.from({ length: 100 }, (_, line) => `line ${line}`).join('\n'), + 'typescript' + ); + const tokenizer = new EditorTokenizer({ + highlighter: createTestHighlighter({ + getLanguage: () => grammar, + }), + textDocument, + codeOptions: { theme: 'test-theme', themeType: 'dark' }, + setStyle: noopSetStyle, + onDeferTokenize: () => {}, + }); + + try { + tokenizer.prebuildStateStack({ + startingLine: 1, + totalLines: 1, + bufferBefore: 0, + bufferAfter: 0, + }); + tokenizer.prebuildStateStack({ + startingLine: 3, + totalLines: 1, + bufferBefore: 0, + bufferAfter: 0, + }); + + expect(tokenizeLineCount).toBe(0); + expect(messageListeners.size).toBe(1); + + let messageIndex = 0; + while (messageIndex < postedMessages.length) { + const event = { data: postedMessages[messageIndex++] } as MessageEvent; + for (const listener of [...messageListeners]) { + listener(event); + } + } + + expect(messageIndex).toBe(4); + expect(tokenizeLineCount).toBe(4); + expect(messageListeners.size).toBe(0); + + tokenizer.getStringCommentRegexpRangesInLine(3); + expect(tokenizeLineCount).toBe(4); + + const change = textDocument.applyEdits([ + { + range: { + start: { line: 0, character: 0 }, + end: { line: 0, character: 6 }, + }, + newText: 'LINE 0', + }, + ])!; + tokenizeLineCount = 0; + postedMessages.length = 0; + tokenizer.tokenize(change, { + startingLine: 0, + totalLines: 1, + bufferBefore: 0, + bufferAfter: 0, + }); + expect(postedMessages).toHaveLength(1); + + tokenizer.prebuildStateStack({ + startingLine: 99, + totalLines: 1, + bufferBefore: 0, + bufferAfter: 0, + }); + expect(postedMessages).toHaveLength(1); + + messageIndex = 0; + while (messageIndex < postedMessages.length) { + const event = { data: postedMessages[messageIndex++] } as MessageEvent; + for (const listener of [...messageListeners]) { + listener(event); + } + } + + expect(tokenizeLineCount).toBeGreaterThan(2); + expect(messageListeners.size).toBe(0); + + const completedTokenizeLineCount = tokenizeLineCount; + tokenizer.getStringCommentRegexpRangesInLine(99); + expect(tokenizeLineCount).toBe(completedTokenizeLineCount); + } finally { + tokenizer.cleanUp(); + globalThis.addEventListener = originalAddEventListener; + globalThis.removeEventListener = originalRemoveEventListener; + globalThis.postMessage = originalPostMessage; + globalThis.setTimeout = originalSetTimeout; + Object.defineProperty(performance, 'now', { + configurable: true, + value: originalPerformanceNow, + }); + } + }); + test('settles zero-line edits before the viewport without rebuilding to the viewport', () => { let tokenizeLineCount = 0; const grammar = { @@ -1381,6 +1641,199 @@ describe('EditorTokenizer', () => { expect([...dirtyLines.keys()]).toEqual([0, 750]); }); + test('maps mixed line-count changes and completes an EOF insertion', () => { + const originalAddEventListener = globalThis.addEventListener; + const originalRemoveEventListener = globalThis.removeEventListener; + const originalPostMessage = globalThis.postMessage; + const messageListeners = new Set(); + const postedMessages: unknown[] = []; + const states = new Map(); + let tokenizeLineCount = 0; + + globalThis.addEventListener = (( + type: string, + listener: EventListenerOrEventListenerObject + ) => { + if (type === 'message' && typeof listener === 'function') { + messageListeners.add(listener); + } + }) as typeof globalThis.addEventListener; + globalThis.removeEventListener = (( + type: string, + listener: EventListenerOrEventListenerObject + ) => { + if (type === 'message' && typeof listener === 'function') { + messageListeners.delete(listener); + } + }) as typeof globalThis.removeEventListener; + globalThis.postMessage = ((message: unknown) => { + postedMessages.push(message); + }) as typeof globalThis.postMessage; + + const grammar = { + tokenizeLine2(lineText: string) { + let nextState = states.get(lineText); + if (nextState === undefined) { + nextState = { + equals(other: StateStack | null) { + return other === nextState; + }, + } as unknown as StateStack; + states.set(lineText, nextState); + } + tokenizeLineCount++; + return { + tokens: new Uint32Array([0, 0]), + ruleStack: nextState, + stoppedEarly: false, + lineText, + }; + }, + } as unknown as IGrammar; + const textDocument = new TextDocument( + 'test.ts', + Array.from({ length: 12 }, (_, index) => `line ${index}`).join('\n'), + 'typescript' + ); + const tokenizer = new EditorTokenizer({ + highlighter: createTestHighlighter({ + getLanguage: () => grammar, + }), + textDocument, + codeOptions: { theme: 'test-theme', themeType: 'dark' }, + matchBrackets: false, + setStyle: noopSetStyle, + onDeferTokenize: () => {}, + }); + + try { + tokenizer.tokenize( + { + startLine: 0, + startCharacter: 0, + endCharacter: 0, + endLine: textDocument.lineCount - 1, + endedAtDocumentEnd: false, + previousLineCount: textDocument.lineCount, + lineCount: textDocument.lineCount, + lineDelta: 0, + changedLineRanges: [[0, textDocument.lineCount - 1]], + }, + { + startingLine: 0, + totalLines: textDocument.lineCount, + bufferBefore: 0, + bufferAfter: 0, + } + ); + + const change = textDocument.applyEdits([ + { + range: { + start: { line: 1, character: 6 }, + end: { line: 1, character: 6 }, + }, + newText: '\na', + }, + { + range: { + start: { line: 6, character: 6 }, + end: { line: 7, character: 0 }, + }, + newText: '', + }, + ])!; + expect(change.lineDelta).toBe(0); + expect(change.changedLineChanges).toEqual([ + [1, 2, 1, 6, 6, false], + [7, 7, -1, 6, 0, false], + ]); + + tokenizeLineCount = 0; + postedMessages.length = 0; + const dirtyLines = tokenizer.tokenize(change, { + startingLine: 1, + totalLines: 2, + bufferBefore: 0, + bufferAfter: 0, + }); + expect([...dirtyLines.keys()]).toEqual([1, 2]); + expect(tokenizeLineCount).toBe(2); + + tokenizeLineCount = 0; + let messageIndex = 0; + while (messageIndex < postedMessages.length) { + const event = { data: postedMessages[messageIndex++] } as MessageEvent; + for (const listener of [...messageListeners]) { + listener(event); + } + } + + // State reconverges on the first unchanged line after the deletion, so + // the untouched document tail is not tokenized. + expect(tokenizeLineCount).toBe(6); + expect(messageListeners.size).toBe(0); + + const lineCount = textDocument.lineCount; + tokenizer.tokenize( + { + startLine: 9, + startCharacter: 0, + endCharacter: 0, + endLine: lineCount - 1, + endedAtDocumentEnd: false, + previousLineCount: lineCount, + lineCount, + lineDelta: 0, + changedLineRanges: [[9, lineCount - 1]], + }, + { + startingLine: 9, + totalLines: lineCount - 9, + bufferBefore: 0, + bufferAfter: 0, + } + ); + + const eofChange = textDocument.applyEdits([ + { + range: { + start: { line: lineCount - 1, character: 7 }, + end: { line: lineCount - 1, character: 7 }, + }, + newText: '\ntail', + }, + ])!; + expect(eofChange.changedLineChanges).toEqual([[11, 12, 1, 7, 7, true]]); + + tokenizeLineCount = 0; + postedMessages.length = 0; + tokenizer.tokenize(eofChange, { + startingLine: 11, + totalLines: 1, + bufferBefore: 0, + bufferAfter: 0, + }); + expect(tokenizeLineCount).toBe(1); + + tokenizeLineCount = 0; + messageIndex = 0; + while (messageIndex < postedMessages.length) { + const event = { data: postedMessages[messageIndex++] } as MessageEvent; + for (const listener of [...messageListeners]) { + listener(event); + } + } + expect(tokenizeLineCount).toBe(1); + expect(messageListeners.size).toBe(0); + } finally { + tokenizer.cleanUp(); + globalThis.addEventListener = originalAddEventListener; + globalThis.removeEventListener = originalRemoveEventListener; + globalThis.postMessage = originalPostMessage; + } + }); + test('pins a dual-theme surface to an explicit themeType instead of following the page', () => { const originalMatchMedia = globalThis.window.matchMedia; let mediaListenerCount = 0; @@ -1538,6 +1991,148 @@ describe('EditorTokenizer', () => { } }); + test('ignores system-theme mutations until the resolved theme changes', () => { + const originalPostMessage = globalThis.postMessage; + const originalGetComputedStyle = Reflect.get( + globalThis, + 'getComputedStyle' + ); + const originalDocument = Reflect.get(globalThis, 'document'); + const originalMutationObserver = Reflect.get( + globalThis, + 'MutationObserver' + ); + const postedMessages: unknown[] = []; + let colorScheme: 'light' | 'dark' = 'dark'; + let observerCallback: MutationCallback | undefined; + let themeChangeCount = 0; + let tokenizer: EditorTokenizer | undefined; + const documentStub = { + body: {}, + documentElement: {}, + }; + + globalThis.postMessage = ((message: unknown) => { + postedMessages.push(message); + }) as typeof globalThis.postMessage; + Reflect.set(globalThis, 'document', documentStub); + Reflect.set( + globalThis, + 'getComputedStyle', + (() => + ({ + colorScheme, + }) as CSSStyleDeclaration) as typeof getComputedStyle + ); + Reflect.set( + globalThis, + 'MutationObserver', + class { + constructor(callback: MutationCallback) { + observerCallback = callback; + } + observe() {} + disconnect() {} + takeRecords() { + return []; + } + } + ); + + try { + const grammar = { + tokenizeLine2(lineText: string, ruleStack: StateStack) { + return { + tokens: new Uint32Array([0, 0]), + ruleStack, + stoppedEarly: false, + lineText, + }; + }, + } as unknown as IGrammar; + const textDocument = new TextDocument( + 'test.ts', + ['line 0', 'line 1'].join('\n'), + 'typescript' + ); + tokenizer = new EditorTokenizer({ + highlighter: createTestHighlighter({ + getLanguage: () => grammar, + }), + textDocument, + codeOptions: { + theme: { light: 'light-theme', dark: 'dark-theme' }, + themeType: 'system', + }, + setStyle: noopSetStyle, + onDeferTokenize: () => {}, + onThemeChange: () => { + themeChangeCount++; + }, + }); + + const observer = {} as MutationObserver; + observerCallback?.( + [ + { + attributeName: 'class', + target: documentStub.documentElement, + type: 'attributes', + } as unknown as MutationRecord, + ], + observer + ); + observerCallback?.( + [ + { + attributeName: 'data-layout', + target: documentStub.body, + type: 'attributes', + } as unknown as MutationRecord, + ], + observer + ); + + expect(tokenizer.themeType).toBe('dark'); + expect(themeChangeCount).toBe(0); + expect(postedMessages).toHaveLength(0); + + colorScheme = 'light'; + observerCallback?.( + [ + { + attributeName: 'data-theme', + target: documentStub.body, + type: 'attributes', + } as unknown as MutationRecord, + ], + observer + ); + + expect(tokenizer.themeType).toBe('light'); + expect(themeChangeCount).toBe(1); + expect(postedMessages).toHaveLength(1); + } finally { + tokenizer?.cleanUp(); + globalThis.postMessage = originalPostMessage; + if (originalGetComputedStyle === undefined) { + Reflect.deleteProperty(globalThis, 'getComputedStyle'); + } else { + Reflect.set(globalThis, 'getComputedStyle', originalGetComputedStyle); + } + if (originalDocument === undefined) { + Reflect.deleteProperty(globalThis, 'document'); + } else { + Reflect.set(globalThis, 'document', originalDocument); + } + if (originalMutationObserver === undefined) { + Reflect.deleteProperty(globalThis, 'MutationObserver'); + } else { + Reflect.set(globalThis, 'MutationObserver', originalMutationObserver); + } + } + }); + // Dual-theme SSR (`themes: {dark,light}`) leaves the shared highlighter on the // last theme it applied (usually light). The tokenizer caches a single-theme // colorMap from construction; without re-activating that theme before an edit,