From 79081eda5f49aecc12bd7c04f30fe69f3137d01e Mon Sep 17 00:00:00 2001 From: Je Xia Date: Mon, 27 Jul 2026 19:51:53 +0800 Subject: [PATCH 1/3] perf(diffs): Reduce editor tokenizer blocking Large files could synchronously build grammar state, remap cached states, and scan bracket caches during edits, delaying editor updates. Time-slice state prebuilds, remap comparison states lazily, and truncate the bracket cache in constant time. Scope background messages per tokenizer and ignore unchanged theme notifications. Add repeatable benchmarks and regression coverage for these paths. --- packages/diffs/moon.yml | 6 + .../diffs/scripts/benchmarkEditorTokenizer.ts | 766 ++++++++++++++++++ packages/diffs/src/editor/tokenizer.ts | 240 ++++-- packages/diffs/test/editorTokenizer.test.ts | 542 +++++++++++++ 4 files changed, 1477 insertions(+), 77 deletions(-) create mode 100644 packages/diffs/scripts/benchmarkEditorTokenizer.ts diff --git a/packages/diffs/moon.yml b/packages/diffs/moon.yml index 2673c84d8..9a8b4c412 100644 --- a/packages/diffs/moon.yml +++ b/packages/diffs/moon.yml @@ -21,6 +21,12 @@ tasks: cache: false runInCI: 'always' + benchmark-editor-tokenizer: + command: 'bun scripts/benchmarkEditorTokenizer.ts' + options: + cache: false + runInCI: 'always' + # Browser E2E for shadow-DOM rendering and contentEditable edit mode, which # the jsdom unit suite cannot exercise. Mirrors the trees E2E setup: a Vite # fixture server serves the built dist and Playwright drives real Chromium. diff --git a/packages/diffs/scripts/benchmarkEditorTokenizer.ts b/packages/diffs/scripts/benchmarkEditorTokenizer.ts new file mode 100644 index 000000000..e407e6738 --- /dev/null +++ b/packages/diffs/scripts/benchmarkEditorTokenizer.ts @@ -0,0 +1,766 @@ +import { type IGrammar, type StateStack } from 'shiki/textmate'; + +import { TextDocument } from '../src/editor/textDocument'; +import { EditorTokenizer } from '../src/editor/tokenizer'; +import type { DiffsHighlighter, RenderRange } from '../src/types'; + +interface BenchmarkConfig { + lines: number; + runs: number; + warmupRuns: number; + outputJson: boolean; +} + +interface BenchmarkSample { + elapsedMs: number; + operations: Record; +} + +interface BenchmarkCase { + name: string; + description: string; + run: () => BenchmarkSample; +} + +interface BenchmarkSummary { + name: string; + description: string; + runs: number; + meanMs: number; + medianMs: number; + p95Ms: number; + minMs: number; + maxMs: number; + operations: Record; +} + +interface MockCounters { + grammarCalls: number; + setThemeCalls: number; +} + +const DEFAULT_CONFIG: BenchmarkConfig = { + lines: 100_000, + runs: 10, + warmupRuns: 3, + outputJson: false, +}; + +const TOKEN_DATA = new Uint32Array([0, 0]); +const messageListeners = new Set(); +const postedMessages: unknown[] = []; +let mutationCallback: ((records: MutationRecord[]) => void) | undefined; + +Reflect.set(globalThis, 'window', globalThis); +Reflect.set(globalThis, 'matchMedia', () => ({ + addEventListener() {}, + addListener() {}, + dispatchEvent() { + return false; + }, + matches: true, + media: '(prefers-color-scheme: dark)', + onchange: null, + removeEventListener() {}, + removeListener() {}, +})); +Reflect.set(globalThis, 'document', { + body: {}, + documentElement: {}, +}); +Reflect.set(globalThis, 'getComputedStyle', () => ({ + colorScheme: 'dark', +})); +Reflect.set( + globalThis, + 'MutationObserver', + class { + constructor(callback: MutationCallback) { + mutationCallback = (records) => + callback(records, this as unknown as MutationObserver); + } + + observe() {} + + disconnect() {} + + takeRecords() { + return []; + } + } +); +Reflect.set( + globalThis, + 'addEventListener', + (type: string, listener: EventListenerOrEventListenerObject) => { + if (type === 'message') { + messageListeners.add(listener); + } + } +); +Reflect.set( + globalThis, + 'removeEventListener', + (type: string, listener: EventListenerOrEventListenerObject) => { + if (type === 'message') { + messageListeners.delete(listener); + } + } +); +Reflect.set(globalThis, 'postMessage', (message: unknown) => { + postedMessages.push(message); +}); + +function parseInteger( + value: string, + flagName: string, + allowZero = false +): number { + const parsed = Number.parseInt(value, 10); + if (!Number.isFinite(parsed) || parsed < (allowZero ? 0 : 1)) { + throw new Error( + `Invalid ${flagName} value "${value}". Expected ${allowZero ? 'a non-negative' : 'a positive'} integer.` + ); + } + return parsed; +} + +function parseArgs(argv: string[]): BenchmarkConfig { + const config = { ...DEFAULT_CONFIG }; + for (let index = 0; index < argv.length; index++) { + const rawArg = argv[index]; + if (rawArg === '--help' || rawArg === '-h') { + console.log('Usage: moonx diffs:benchmark-editor-tokenizer -- [options]'); + console.log(''); + console.log('Options:'); + console.log( + ` --lines Document lines (default: ${DEFAULT_CONFIG.lines})` + ); + console.log( + ` --runs Measured runs per case (default: ${DEFAULT_CONFIG.runs})` + ); + console.log( + ` --warmup-runs Warmup runs per case (default: ${DEFAULT_CONFIG.warmupRuns})` + ); + console.log(' --json Emit machine-readable JSON'); + console.log(' -h, --help Show this help output'); + process.exit(0); + } + if (rawArg === '--json') { + config.outputJson = true; + continue; + } + + const [flag, inlineValue] = rawArg.split('=', 2); + const value = inlineValue ?? argv[index + 1]; + if (flag === '--lines' || flag === '--runs' || flag === '--warmup-runs') { + if (value === undefined) { + throw new Error(`Missing value for ${flag}`); + } + if (inlineValue === undefined) { + index++; + } + const parsed = parseInteger(value, flag, flag === '--warmup-runs'); + if (flag === '--lines') { + config.lines = parsed; + } else if (flag === '--runs') { + config.runs = parsed; + } else { + config.warmupRuns = parsed; + } + continue; + } + throw new Error(`Unknown argument: ${rawArg}`); + } + if (config.lines < 2) { + throw new Error('--lines must be at least 2'); + } + return config; +} + +function percentile(sortedValues: number[], rank: number): number { + const index = (sortedValues.length - 1) * rank; + const lowerIndex = Math.floor(index); + const upperIndex = Math.ceil(index); + const lower = sortedValues[lowerIndex] ?? 0; + const upper = sortedValues[upperIndex] ?? lower; + return lower + (upper - lower) * (index - lowerIndex); +} + +function createTokenizer( + textDocument: TextDocument, + counters: MockCounters, + options: { + matchBrackets?: boolean; + systemTheme?: boolean; + onThemeChange?: () => void; + } = {} +): EditorTokenizer { + const grammar = { + tokenizeLine2(_lineText: string, ruleStack: StateStack) { + counters.grammarCalls++; + return { + tokens: TOKEN_DATA, + ruleStack, + stoppedEarly: false, + }; + }, + } as unknown as IGrammar; + const highlighter = { + getLanguage: () => grammar, + getLoadedLanguages: () => ['typescript'], + getTheme: () => ({ colors: {} }), + setTheme: (themeName: string) => { + counters.setThemeCalls++; + return { + colorMap: [''], + theme: { name: themeName, type: 'dark' }, + }; + }, + } as unknown as DiffsHighlighter; + return new EditorTokenizer({ + highlighter, + textDocument, + codeOptions: + options.systemTheme === true + ? { + theme: { dark: 'dark-theme', light: 'light-theme' }, + themeType: 'system', + } + : { theme: 'dark-theme', themeType: 'dark' }, + matchBrackets: options.matchBrackets, + setStyle() {}, + onDeferTokenize() {}, + onThemeChange: options.onThemeChange, + }); +} + +function primeTokenizer( + tokenizer: EditorTokenizer, + textDocument: TextDocument +): void { + const lineCount = textDocument.lineCount; + tokenizer.tokenize( + { + startLine: 0, + startCharacter: 0, + endCharacter: 0, + endLine: lineCount - 1, + endedAtDocumentEnd: false, + previousLineCount: lineCount, + lineCount, + lineDelta: 0, + changedLineRanges: [[0, lineCount - 1]], + }, + { + startingLine: 0, + totalLines: lineCount, + bufferBefore: 0, + bufferAfter: 0, + } + ); +} + +function resetMessages(): void { + if (messageListeners.size > 0) { + throw new Error('A benchmark case leaked a background message listener'); + } + postedMessages.length = 0; +} + +function dispatchMessage(data: unknown): void { + const event = { data } as MessageEvent; + for (const listener of [...messageListeners]) { + if (typeof listener === 'function') { + listener(event); + } else { + listener.handleEvent(event); + } + } +} + +function drainMessages(counters: MockCounters, maxMessages: number): number { + const originalPerformanceNow = performance.now; + let messageIndex = 0; + Object.defineProperty(performance, 'now', { + configurable: true, + value: () => counters.grammarCalls / 1000, + }); + try { + while (messageIndex < postedMessages.length) { + if (messageIndex >= maxMessages) { + throw new Error('Background tokenizer did not settle'); + } + dispatchMessage(postedMessages[messageIndex]); + messageIndex++; + } + } finally { + Object.defineProperty(performance, 'now', { + configurable: true, + value: originalPerformanceNow, + }); + } + return messageIndex; +} + +function collectGarbage(): void { + if (typeof Bun !== 'undefined') { + Bun.gc(true); + } +} + +function createBenchmarkCases( + config: BenchmarkConfig, + sourceText: string +): BenchmarkCase[] { + const fullRange: RenderRange = { + startingLine: 0, + totalLines: config.lines, + bufferBefore: 0, + bufferAfter: 0, + }; + + return [ + { + name: 'unbounded-structural-edit', + description: + 'Insert a newline at line 0 after fully populating tokenizer state.', + run() { + resetMessages(); + const counters = { grammarCalls: 0, setThemeCalls: 0 }; + const textDocument = new TextDocument( + 'structural.ts', + sourceText, + 'typescript' + ); + const tokenizer = createTokenizer(textDocument, counters, { + matchBrackets: false, + }); + primeTokenizer(tokenizer, textDocument); + const change = textDocument.applyEdits([ + { + range: { + start: { line: 0, character: 0 }, + end: { line: 0, character: 0 }, + }, + newText: 'inserted\n', + }, + ]); + if (change === undefined) { + throw new Error( + 'Expected the structural edit to change the document' + ); + } + counters.grammarCalls = 0; + postedMessages.length = 0; + collectGarbage(); + const started = performance.now(); + const dirtyLines = tokenizer.tokenize(change); + const elapsedMs = performance.now() - started; + const operations = { + grammarCalls: counters.grammarCalls, + dirtyLines: dirtyLines.size, + backgroundMessages: postedMessages.length, + }; + tokenizer.cleanUp(); + return { elapsedMs, operations }; + }, + }, + { + name: 'deep-state-prebuild', + description: + 'Start grammar-state preparation for a viewport at the document tail.', + run() { + resetMessages(); + const counters = { grammarCalls: 0, setThemeCalls: 0 }; + const textDocument = new TextDocument( + 'prebuild.ts', + sourceText, + 'typescript' + ); + const tokenizer = createTokenizer(textDocument, counters, { + matchBrackets: false, + }); + collectGarbage(); + const started = performance.now(); + tokenizer.prebuildStateStack({ + ...fullRange, + startingLine: config.lines - 1, + totalLines: 1, + }); + const elapsedMs = performance.now() - started; + const synchronousGrammarCalls = counters.grammarCalls; + const backgroundMessages = drainMessages( + counters, + textDocument.lineCount + 1 + ); + const operations = { + synchronousGrammarCalls, + backgroundMessages, + totalGrammarCalls: counters.grammarCalls, + }; + tokenizer.cleanUp(); + return { elapsedMs, operations }; + }, + }, + { + name: 'bracket-cache-invalidation', + description: + 'Edit one character at line 0 after caching bracket ranges for every line.', + run() { + resetMessages(); + const counters = { grammarCalls: 0, setThemeCalls: 0 }; + const textDocument = new TextDocument( + 'brackets.ts', + sourceText, + 'typescript' + ); + const tokenizer = createTokenizer(textDocument, counters, { + matchBrackets: true, + }); + primeTokenizer(tokenizer, textDocument); + const change = textDocument.applyEdits([ + { + range: { + start: { line: 0, character: 0 }, + end: { line: 0, character: 1 }, + }, + newText: 'L', + }, + ]); + if (change === undefined) { + throw new Error('Expected the same-line edit to change the document'); + } + counters.grammarCalls = 0; + collectGarbage(); + const started = performance.now(); + const dirtyLines = tokenizer.tokenize(change, { + ...fullRange, + totalLines: 1, + }); + const elapsedMs = performance.now() - started; + const operations = { + grammarCalls: counters.grammarCalls, + dirtyLines: dirtyLines.size, + }; + tokenizer.cleanUp(); + return { elapsedMs, operations }; + }, + }, + { + name: 'comparison-state-remap-no-baseline', + description: + 'Informational only: remap and converge many structural edits; no permanent pre-fix baseline was captured.', + run() { + resetMessages(); + const counters = { grammarCalls: 0, setThemeCalls: 0 }; + const textDocument = new TextDocument( + 'remap.ts', + sourceText, + 'typescript' + ); + const tokenizer = createTokenizer(textDocument, counters, { + matchBrackets: false, + }); + primeTokenizer(tokenizer, textDocument); + const editCount = Math.min( + 128, + config.lines, + Math.max(2, Math.floor(config.lines / 1000)) + ); + const edits = Array.from({ length: editCount }, (_, index) => { + const line = Math.floor( + ((index + 1) * config.lines) / (editCount + 1) + ); + return { + range: { + start: { line, character: 0 }, + end: { line, character: 0 }, + }, + newText: 'inserted\n', + }; + }); + const change = textDocument.applyEdits(edits); + if (change === undefined) { + throw new Error('Expected structural edits to change the document'); + } + counters.grammarCalls = 0; + postedMessages.length = 0; + collectGarbage(); + const started = performance.now(); + tokenizer.tokenize(change, { + startingLine: change.startLine, + totalLines: 1, + bufferBefore: 0, + bufferAfter: 0, + }); + const elapsedMs = performance.now() - started; + const synchronousGrammarCalls = counters.grammarCalls; + const backgroundMessages = drainMessages( + counters, + textDocument.lineCount + 1 + ); + const operations = { + edits: editCount, + changedLineChanges: change.changedLineChanges?.length ?? 0, + synchronousGrammarCalls, + backgroundMessages, + totalGrammarCalls: counters.grammarCalls, + }; + tokenizer.cleanUp(); + return { elapsedMs, operations }; + }, + }, + { + name: 'no-op-theme-mutation', + description: + 'Notify an unchanged system-theme tokenizer of a root class mutation.', + run() { + resetMessages(); + mutationCallback = undefined; + let themeChangeCallbacks = 0; + const counters = { grammarCalls: 0, setThemeCalls: 0 }; + const textDocument = new TextDocument( + 'theme.ts', + 'line 0\nline 1', + 'typescript' + ); + const tokenizer = createTokenizer(textDocument, counters, { + matchBrackets: false, + systemTheme: true, + onThemeChange: () => { + themeChangeCallbacks++; + }, + }); + const callback = mutationCallback; + if (callback === undefined) { + throw new Error('Expected a system-theme MutationObserver'); + } + counters.setThemeCalls = 0; + collectGarbage(); + const started = performance.now(); + callback([ + { + attributeName: 'class', + type: 'attributes', + } as MutationRecord, + ]); + const elapsedMs = performance.now() - started; + const operations = { + backgroundRestarts: postedMessages.length, + setThemeCalls: counters.setThemeCalls, + themeChangeCallbacks, + }; + tokenizer.cleanUp(); + mutationCallback = undefined; + return { elapsedMs, operations }; + }, + }, + { + name: 'multi-instance-message-collision', + description: + 'Dispatch one tokenizer job message while two tokenizer jobs are active.', + run() { + resetMessages(); + const counters = [ + { grammarCalls: 0, setThemeCalls: 0 }, + { grammarCalls: 0, setThemeCalls: 0 }, + ]; + const tokenizers = counters.map((counter, index) => { + const textDocument = new TextDocument( + `collision-${index}.ts`, + 'line 0\nline 1', + 'typescript' + ); + const tokenizer = createTokenizer(textDocument, counter, { + matchBrackets: false, + }); + tokenizer.tokenize( + { + startLine: 0, + startCharacter: 0, + endCharacter: 0, + endLine: 1, + endedAtDocumentEnd: false, + previousLineCount: 2, + lineCount: 2, + lineDelta: 0, + changedLineRanges: [[0, 1]], + }, + { + startingLine: 0, + totalLines: 1, + bufferBefore: 0, + bufferAfter: 0, + } + ); + return tokenizer; + }); + const firstMessage = postedMessages[0]; + if (firstMessage === undefined || messageListeners.size !== 2) { + throw new Error('Expected two active tokenizer background jobs'); + } + counters[0].grammarCalls = 0; + counters[1].grammarCalls = 0; + collectGarbage(); + const started = performance.now(); + dispatchMessage(firstMessage); + const elapsedMs = performance.now() - started; + const operations = { + sourceGrammarCalls: counters[0].grammarCalls, + foreignGrammarCalls: counters[1].grammarCalls, + triggeredInstances: + Number(counters[0].grammarCalls > 0) + + Number(counters[1].grammarCalls > 0), + }; + for (const tokenizer of tokenizers) { + tokenizer.cleanUp(); + } + return { elapsedMs, operations }; + }, + }, + ]; +} + +function summarize( + benchmarkCase: BenchmarkCase, + samples: BenchmarkSample[] +): BenchmarkSummary { + const elapsed = samples + .map((sample) => sample.elapsedMs) + .sort((left, right) => left - right); + const operations = samples[0]?.operations ?? {}; + const expectedOperations = JSON.stringify(operations); + for (const sample of samples) { + if (JSON.stringify(sample.operations) !== expectedOperations) { + throw new Error( + `Non-deterministic operation counts in ${benchmarkCase.name}` + ); + } + } + const total = elapsed.reduce((sum, value) => sum + value, 0); + return { + name: benchmarkCase.name, + description: benchmarkCase.description, + runs: elapsed.length, + meanMs: total / elapsed.length, + medianMs: percentile(elapsed, 0.5), + p95Ms: percentile(elapsed, 0.95), + minMs: elapsed[0] ?? 0, + maxMs: elapsed[elapsed.length - 1] ?? 0, + operations, + }; +} + +function printTable(summaries: BenchmarkSummary[]): void { + const rows = summaries.map((summary) => ({ + scenario: summary.name, + median: summary.medianMs.toFixed(3), + p95: summary.p95Ms.toFixed(3), + min: summary.minMs.toFixed(3), + max: summary.maxMs.toFixed(3), + operations: Object.entries(summary.operations) + .map(([key, value]) => `${key}=${value}`) + .join(','), + })); + const headers = [ + 'scenario', + 'median', + 'p95', + 'min', + 'max', + 'operations', + ] as const; + const widths = headers.map((header) => + rows.reduce( + (width, row) => Math.max(width, row[header].length), + header.length + ) + ); + const formatRow = (row: Record<(typeof headers)[number], string>) => + headers + .map((header, index) => row[header].padEnd(widths[index])) + .join(' ') + .trimEnd(); + console.log(formatRow(Object.fromEntries(headers.map((key) => [key, key])))); + console.log(widths.map((width) => '-'.repeat(width)).join(' ')); + for (const row of rows) { + console.log(formatRow(row)); + } +} + +function main(): void { + const config = parseArgs(process.argv.slice(2)); + const sourceText = Array.from( + { length: config.lines }, + (_, index) => `const value${index} = ${index};` + ).join('\n'); + const benchmarkCases = createBenchmarkCases(config, sourceText); + const samples = benchmarkCases.map(() => [] as BenchmarkSample[]); + let checksum = 0; + + for (let run = 0; run < config.warmupRuns; run++) { + for (let offset = 0; offset < benchmarkCases.length; offset++) { + benchmarkCases[(run + offset) % benchmarkCases.length].run(); + } + } + for (let run = 0; run < config.runs; run++) { + for (let offset = 0; offset < benchmarkCases.length; offset++) { + const caseIndex = (run + offset) % benchmarkCases.length; + const sample = benchmarkCases[caseIndex].run(); + samples[caseIndex].push(sample); + checksum += Object.values(sample.operations).reduce( + (sum, value) => sum + value, + 0 + ); + } + } + + const summaries = benchmarkCases.map((benchmarkCase, index) => + summarize(benchmarkCase, samples[index]) + ); + const result = { + benchmark: 'EditorTokenizer', + config, + measurement: { + setupExcluded: true, + forcedGcBeforeTiming: true, + prebuildDebounceExcluded: true, + backgroundDrainExcluded: true, + backgroundDrainClock: '1ms per 1000 mock grammar calls', + grammar: 'deterministic one-token mock', + }, + checksum, + summaries, + }; + if (config.outputJson) { + console.log(JSON.stringify(result, null, 2)); + return; + } + + console.log('EditorTokenizer benchmark'); + console.log( + `lines=${config.lines} runsPerCase=${config.runs} warmupRunsPerCase=${config.warmupRuns}` + ); + console.log( + 'Setup, forced GC, and the prebuild debounce delay are excluded from timings.' + ); + console.log(''); + printTable(summaries); + console.log(''); + for (const summary of summaries) { + console.log(`${summary.name}: ${summary.description}`); + } + console.log(`checksum=${checksum}`); +} + +const originalSetTimeout = globalThis.setTimeout; +Reflect.set(globalThis, 'setTimeout', ((callback: () => void) => { + callback(); + return 0; +}) as unknown as typeof setTimeout); +try { + main(); +} finally { + Reflect.set(globalThis, 'setTimeout', originalSetTimeout); +} diff --git a/packages/diffs/src/editor/tokenizer.ts b/packages/diffs/src/editor/tokenizer.ts index d72189a8d..3a84f664e 100644 --- a/packages/diffs/src/editor/tokenizer.ts +++ b/packages/diffs/src/editor/tokenizer.ts @@ -17,6 +17,7 @@ import type { TextDocument, TextDocumentChange } from './textDocument'; import { addEventListener, debounce, h } from './utils'; const TOKENIZE_TIME_LIMIT = 500; +let nextTokenizerId = 0; export interface EditorTokenizerProps { highlighter: DiffsHighlighter; @@ -59,13 +60,19 @@ export class EditorTokenizer { // state #stateStack: StateStack[] = [INITIAL]; // cached state stack by line index #comparisonStateStack: StateStack[] = []; + #comparisonStateStackStart = 0; + #comparisonLineChanges: NonNullable< + TextDocumentChange['changedLineChanges'] + > = []; #lastLine: number = -1; #isStopped: boolean = true; #isPaused: boolean = false; #backgroundJobId: number = 0; + #tokenizerId = ++nextTokenizerId; + #backgroundPrebuildEndLine = -1; #backgroundChangedLineRanges: readonly [number, number][] | undefined; #backgroundChangedRangeIndex: number = 0; - #bracketIgnoredRanges: Map = new Map(); + #bracketIgnoredRanges: ([number, number][] | null | undefined)[] = []; #isMessageListenerAttached: boolean = false; #prebuildStateStack = debounce(async (renderRange?: RenderRange) => { @@ -92,23 +99,29 @@ export class EditorTokenizer { ); } this.#ensureActiveTheme(); - this.#buildStateStack(endLine); + this.#scheduleStatePrebuild(endLine); }, 500); #onMessage = ({ data }: MessageEvent) => { if (typeof data !== 'object' || data === null) { return; } - const { type, jobId } = data as { + const { type, tokenizerId, jobId } = data as { type?: unknown; + tokenizerId?: unknown; jobId?: unknown; }; if ( type === 'tokenize' && + tokenizerId === this.#tokenizerId && typeof jobId === 'number' && jobId === this.#backgroundJobId ) { - this.#backgroundTokenize(jobId); + if (this.#backgroundPrebuildEndLine >= 0) { + this.#backgroundPrebuild(jobId); + } else { + this.#backgroundTokenize(jobId); + } } }; @@ -130,13 +143,13 @@ export class EditorTokenizer { if (this.#grammar === undefined) { return null; } - if (!this.#bracketIgnoredRanges.has(lineIndex)) { + if (this.#bracketIgnoredRanges[lineIndex] === undefined) { this.#buildStateStack(lineIndex); const state = this.#stateStack[lineIndex] ?? INITIAL; const result = this.#tokenizeLineAt(lineIndex, state); this.#stateStack[lineIndex + 1] = result.state; } - return this.#bracketIgnoredRanges.get(lineIndex) ?? null; + return this.#bracketIgnoredRanges[lineIndex] ?? null; } constructor({ @@ -213,10 +226,15 @@ export class EditorTokenizer { // By default, diffs components support dual themes, but the tokenizer only renders // the preferred theme. When the theme type is changed, the tokenizer will re-tokenize the document. #emitThemeChange(themeName: string, themeType: 'light' | 'dark') { + if (themeName === this.#themeName && themeType === this.#themeType) { + return; + } this.#setTheme(themeName, themeType); this.stopBackgroundTokenize(); this.#stateStack = [INITIAL]; this.#comparisonStateStack = []; + this.#comparisonStateStackStart = 0; + this.#comparisonLineChanges = []; if (this.#grammar !== undefined && this.#textDocument.lineCount > 0) { this.#scheduleBackgroundTokenize(0); } @@ -364,11 +382,10 @@ export class EditorTokenizer { if (this.#matchBrackets) { // Clear ignored token ranges for lines invalidated by the edit. - for (const line of this.#bracketIgnoredRanges.keys()) { - if (line >= change.startLine) { - this.#bracketIgnoredRanges.delete(line); - } - } + this.#bracketIgnoredRanges.length = Math.min( + this.#bracketIgnoredRanges.length, + change.startLine + ); } const { lineCount } = this.#textDocument; @@ -386,7 +403,10 @@ export class EditorTokenizer { change.lineDelta > 0 && dirtyStart < renderRangeEndLine && change.endLine >= renderRangeEndLine; - const canReuseCachedStates = change.lineDelta === 0; + const canReuseCachedStates = + change.lineDelta === 0 && + (change.changedLineChanges?.every(([, , lineDelta]) => lineDelta === 0) ?? + true); const canCacheTokenizedStates = canReuseCachedStates || renderRange === undefined || @@ -394,6 +414,8 @@ export class EditorTokenizer { const changedLineRanges: readonly [number, number][] = change.changedLineRanges ?? [[dirtyStart, change.endLine]]; this.#comparisonStateStack = []; + this.#comparisonStateStackStart = 0; + this.#comparisonLineChanges = []; let offscreenSyncEnd = -1; if (dirtyStart < viewStart) { for (const [rangeStart, rangeEnd] of changedLineRanges) { @@ -553,9 +575,12 @@ export class EditorTokenizer { this.#isStopped = true; this.#isPaused = false; this.#lastLine = -1; + this.#backgroundPrebuildEndLine = -1; this.#backgroundChangedLineRanges = undefined; this.#backgroundChangedRangeIndex = 0; this.#comparisonStateStack = []; + this.#comparisonStateStackStart = 0; + this.#comparisonLineChanges = []; this.#detachMessageListener(); } @@ -621,7 +646,11 @@ export class EditorTokenizer { #postTokenizeMessage(jobId: number): void { // use `postMessage` instead of `setTimeout(fn, 0)` to avoid 4ms delay - globalThis.postMessage({ type: 'tokenize', jobId }); + globalThis.postMessage({ + type: 'tokenize', + tokenizerId: this.#tokenizerId, + jobId, + }); } #scheduleBackgroundTokenize( @@ -647,12 +676,40 @@ export class EditorTokenizer { this.#isStopped = false; this.#isPaused = false; this.#lastLine = startLine; + this.#backgroundPrebuildEndLine = -1; this.#backgroundChangedLineRanges = changedLineRanges; this.#backgroundChangedRangeIndex = changedRangeIndex; this.#attachMessageListener(); this.#postTokenizeMessage(jobId); } + #scheduleStatePrebuild(endLine: number): void { + if (this.#grammar === undefined || this.#stateStack.length > endLine) { + return; + } + // Keep an active state-only job and extend it when a later viewport needs + // more state. A foreground edit job remains authoritative. + if (!this.#isStopped) { + if (this.#backgroundPrebuildEndLine >= 0) { + this.#backgroundPrebuildEndLine = Math.max( + this.#backgroundPrebuildEndLine, + endLine + ); + } + return; + } + + const jobId = ++this.#backgroundJobId; + this.#isStopped = false; + this.#isPaused = false; + this.#lastLine = this.#stateStack.length - 1; + this.#backgroundPrebuildEndLine = endLine; + this.#backgroundChangedLineRanges = undefined; + this.#backgroundChangedRangeIndex = 0; + this.#attachMessageListener(); + this.#postTokenizeMessage(jobId); + } + #tokenizeLineAt( line: number, state: StateStack @@ -693,88 +750,70 @@ export class EditorTokenizer { ranges: [number, number][] | null ): void { if (this.#matchBrackets) { - this.#bracketIgnoredRanges.set(line, ranges); + this.#bracketIgnoredRanges[line] = ranges; } } - // Preserve old end states as comparison-only sentinels. They let background - // tokenization stop when grammar state reconverges without letting foreground - // tokenization seed from stale pre-edit states. + // Preserve old end states as comparison-only sentinels, copying whichever + // side of the edit is smaller. Index shifts are resolved lazily on lookup. #shiftComparisonStateStack(change: TextDocumentChange): void { const lineChanges = change.changedLineChanges ?? ([[change.startLine, change.endLine, change.lineDelta]] as const); - const comparisonStateStack = this.#stateStack.slice(); + const comparisonStart = change.startLine + 1; + const comparisonLength = this.#stateStack.length - comparisonStart; + if (comparisonStart <= comparisonLength) { + this.#comparisonStateStack = this.#stateStack; + this.#comparisonStateStackStart = 0; + this.#stateStack = this.#stateStack.slice(0, comparisonStart); + } else { + this.#comparisonStateStackStart = comparisonStart; + this.#comparisonStateStack = this.#stateStack.slice(comparisonStart); + this.#stateStack.length = Math.min( + this.#stateStack.length, + comparisonStart + ); + } + this.#comparisonLineChanges = lineChanges; + } - for (const [startLine, endLine, lineDelta] of lineChanges) { + #getPreviousEndState(line: number): StateStack | undefined { + let previousLine = line; + for ( + let index = this.#comparisonLineChanges.length - 1; + index >= 0; + index-- + ) { + const [startLine, endLine, lineDelta] = + this.#comparisonLineChanges[index]; if (lineDelta === 0) { continue; } - - const insertedLineSpan = endLine - startLine; - const oldLineSpan = insertedLineSpan - lineDelta; - const sourceStart = startLine + oldLineSpan + 1; - const targetStart = startLine + insertedLineSpan + 1; - const originalLength = comparisonStateStack.length; - - if (lineDelta > 0) { - for (let line = originalLength - 1; line >= sourceStart; line--) { - const targetLine = line + lineDelta; - if (line in comparisonStateStack) { - comparisonStateStack[targetLine] = comparisonStateStack[line]; - } else { - Reflect.deleteProperty(comparisonStateStack, targetLine); - } - } - } else { - for (let line = sourceStart; line < originalLength; line++) { - const targetLine = line + lineDelta; - if (line in comparisonStateStack) { - comparisonStateStack[targetLine] = comparisonStateStack[line]; - } else { - Reflect.deleteProperty(comparisonStateStack, targetLine); - } - } - comparisonStateStack.length = Math.max( - Math.min(originalLength, startLine + 1), - originalLength + lineDelta - ); - } - - for (let line = startLine + 1; line < targetStart; line++) { - Reflect.deleteProperty(comparisonStateStack, line); + if (previousLine > endLine) { + previousLine -= lineDelta; + } else if (previousLine > startLine) { + return this.#stateStack[line]; } } - - this.#stateStack.length = Math.min( - this.#stateStack.length, - change.startLine + 1 - ); - comparisonStateStack.length = Math.min( - comparisonStateStack.length, - this.#textDocument.lineCount + 1 + return ( + this.#comparisonStateStack[ + previousLine - this.#comparisonStateStackStart + ] ?? this.#stateStack[line] ); - for (let line = 0; line <= change.startLine; line++) { - Reflect.deleteProperty(comparisonStateStack, line); - } - this.#comparisonStateStack = comparisonStateStack; - } - - #getPreviousEndState(line: number): StateStack | undefined { - return this.#comparisonStateStack[line] ?? this.#stateStack[line]; } - #buildStateStack(endAt: number) { + #buildStateStack(endAt: number, timeBudget?: number): boolean { const boundedEndAt = Math.min( Math.max(0, endAt), this.#textDocument.lineCount ); if (this.#stateStack.length > boundedEndAt || this.#grammar === undefined) { - return; + return true; } + const startedAt = timeBudget === undefined ? 0 : performance.now(); let line = this.#stateStack.length - 1; let state = this.#stateStack[line] ?? INITIAL; - for (; line < boundedEndAt; line++) { + while (line < boundedEndAt) { this.#stateStack[line] = state; const lineText = this.#textDocument.getLineText(line); if ( @@ -788,15 +827,49 @@ export class EditorTokenizer { lineText, state, TOKENIZE_TIME_LIMIT, - this.#matchBrackets + this.#matchBrackets, + false ); this.#cacheBracketIgnoredRanges(line, result.bracketIgnoredRanges); state = result.ruleStack; } else { this.#cacheBracketIgnoredRanges(line, null); } + line++; + this.#stateStack[line] = state; + if ( + timeBudget !== undefined && + performance.now() - startedAt > timeBudget + ) { + break; + } + } + return line >= boundedEndAt; + } + + #backgroundPrebuild(jobId: number): void { + if ( + this.#isStopped || + this.#isPaused || + this.#grammar === undefined || + jobId !== this.#backgroundJobId + ) { + return; } - this.#stateStack[line] = state; + + this.#ensureActiveTheme(); + // State prebuilds intentionally omit rendered tokens and yield between + // short chunks so a deep viewport does not monopolize the main thread. + const complete = this.#buildStateStack(this.#backgroundPrebuildEndLine, 1); + if (this.#isStopped || this.#isPaused || jobId !== this.#backgroundJobId) { + return; + } + if (complete) { + this.stopBackgroundTokenize(); + return; + } + this.#lastLine = this.#stateStack.length - 1; + this.#postTokenizeMessage(jobId); } #backgroundTokenize(jobId: number) { @@ -904,7 +977,8 @@ function tokenizeLine( lineText: string, stateStack: StateStack, timeLimit?: number, - collectBracketIgnoredRanges = true + collectBracketIgnoredRanges = true, + resolveTokens = true ): { ruleStack: StateStack; resolvedTokens: Array; @@ -920,6 +994,13 @@ function tokenizeLine( const tokensLength = rawTokens.length / 2; const resolvedTokens: Array = []; const bracketIgnoredRanges: [number, number][] = []; + if (!resolveTokens && !collectBracketIgnoredRanges) { + return { + ruleStack: result.ruleStack, + resolvedTokens, + bracketIgnoredRanges, + }; + } for (let j = 0; j < tokensLength; j++) { const offset = rawTokens[2 * j]; const nextOffset = @@ -929,9 +1010,14 @@ function tokenizeLine( continue; } const metadata = rawTokens[2 * j + 1]; - const fg = EncodedTokenMetadata.getForeground(metadata); - const tokenText = lineText.slice(offset, nextOffset); - resolvedTokens.push([offset, colorMap[fg], tokenText]); + if (resolveTokens) { + const fg = EncodedTokenMetadata.getForeground(metadata); + resolvedTokens.push([ + offset, + colorMap[fg], + lineText.slice(offset, nextOffset), + ]); + } if ( collectBracketIgnoredRanges && EncodedTokenMetadata.getTokenType(metadata) > 0 diff --git a/packages/diffs/test/editorTokenizer.test.ts b/packages/diffs/test/editorTokenizer.test.ts index 76bbfb5a3..0f293587a 100644 --- a/packages/diffs/test/editorTokenizer.test.ts +++ b/packages/diffs/test/editorTokenizer.test.ts @@ -975,6 +975,213 @@ describe('EditorTokenizer', () => { } }); + test('isolates matching background job ids between tokenizer instances', () => { + const originalAddEventListener = globalThis.addEventListener; + const originalRemoveEventListener = globalThis.removeEventListener; + const originalPostMessage = globalThis.postMessage; + const messageListeners = new Set(); + const postedMessages: unknown[] = []; + const tokenizers: EditorTokenizer[] = []; + + globalThis.addEventListener = (( + type: string, + listener: EventListenerOrEventListenerObject + ) => { + if (type === 'message' && typeof listener === 'function') { + messageListeners.add(listener); + } + }) as typeof globalThis.addEventListener; + globalThis.removeEventListener = (( + type: string, + listener: EventListenerOrEventListenerObject + ) => { + if (type === 'message' && typeof listener === 'function') { + messageListeners.delete(listener); + } + }) as typeof globalThis.removeEventListener; + globalThis.postMessage = ((message: unknown) => { + postedMessages.push(message); + }) as typeof globalThis.postMessage; + + try { + const tokenizeLineCounts = [0, 0]; + for (let index = 0; index < tokenizeLineCounts.length; index++) { + const grammar = { + tokenizeLine2(lineText: string, ruleStack: StateStack) { + tokenizeLineCounts[index]++; + return { + tokens: new Uint32Array([0, 0]), + ruleStack, + stoppedEarly: false, + lineText, + }; + }, + } as unknown as IGrammar; + const textDocument = new TextDocument( + `test-${index}.ts`, + ['line 0', 'line 1', 'line 2'].join('\n'), + 'typescript' + ); + const tokenizer = new EditorTokenizer({ + highlighter: createTestHighlighter({ + getLanguage: () => grammar, + }), + textDocument, + codeOptions: { theme: 'test-theme', themeType: 'dark' }, + setStyle: noopSetStyle, + onDeferTokenize: () => {}, + }); + tokenizers.push(tokenizer); + tokenizer.tokenize( + { + startLine: 0, + startCharacter: 0, + endCharacter: 0, + endLine: 0, + endedAtDocumentEnd: false, + previousLineCount: textDocument.lineCount, + lineCount: textDocument.lineCount, + lineDelta: 0, + changedLineRanges: [[0, 0]], + }, + { + startingLine: 0, + totalLines: 1, + bufferBefore: 0, + bufferAfter: 0, + } + ); + } + + expect(postedMessages).toHaveLength(2); + expect((postedMessages[0] as { jobId: number }).jobId).toBe( + (postedMessages[1] as { jobId: number }).jobId + ); + tokenizeLineCounts.fill(0); + + const event = { data: postedMessages[0] } as MessageEvent; + for (const listener of [...messageListeners]) { + listener(event); + } + + expect(tokenizeLineCounts[0]).toBeGreaterThan(0); + expect(tokenizeLineCounts[1]).toBe(0); + } finally { + tokenizers.forEach((tokenizer) => tokenizer.cleanUp()); + globalThis.addEventListener = originalAddEventListener; + globalThis.removeEventListener = originalRemoveEventListener; + globalThis.postMessage = originalPostMessage; + } + }); + + test('prebuilds state only from queued messages and detaches when complete', () => { + const originalAddEventListener = globalThis.addEventListener; + const originalRemoveEventListener = globalThis.removeEventListener; + const originalPostMessage = globalThis.postMessage; + const originalSetTimeout = globalThis.setTimeout; + const originalPerformanceNow = performance.now; + const messageListeners = new Set(); + const postedMessages: unknown[] = []; + let tokenizeLineCount = 0; + + globalThis.addEventListener = (( + type: string, + listener: EventListenerOrEventListenerObject + ) => { + if (type === 'message' && typeof listener === 'function') { + messageListeners.add(listener); + } + }) as typeof globalThis.addEventListener; + globalThis.removeEventListener = (( + type: string, + listener: EventListenerOrEventListenerObject + ) => { + if (type === 'message' && typeof listener === 'function') { + messageListeners.delete(listener); + } + }) as typeof globalThis.removeEventListener; + globalThis.postMessage = ((message: unknown) => { + postedMessages.push(message); + }) as typeof globalThis.postMessage; + globalThis.setTimeout = ((callback: () => void) => { + callback(); + return 0; + }) as unknown as typeof globalThis.setTimeout; + let now = 0; + Object.defineProperty(performance, 'now', { + configurable: true, + value: () => (now += 2), + }); + + const grammar = { + tokenizeLine2(lineText: string, ruleStack: StateStack) { + tokenizeLineCount++; + return { + tokens: new Uint32Array([0, 0]), + ruleStack, + stoppedEarly: false, + lineText, + }; + }, + } as unknown as IGrammar; + const tokenizer = new EditorTokenizer({ + highlighter: createTestHighlighter({ + getLanguage: () => grammar, + }), + textDocument: new TextDocument( + 'test.ts', + ['line 0', 'line 1', 'line 2', 'line 3'].join('\n'), + 'typescript' + ), + codeOptions: { theme: 'test-theme', themeType: 'dark' }, + setStyle: noopSetStyle, + onDeferTokenize: () => {}, + }); + + try { + tokenizer.prebuildStateStack({ + startingLine: 1, + totalLines: 1, + bufferBefore: 0, + bufferAfter: 0, + }); + tokenizer.prebuildStateStack({ + startingLine: 3, + totalLines: 1, + bufferBefore: 0, + bufferAfter: 0, + }); + + expect(tokenizeLineCount).toBe(0); + expect(messageListeners.size).toBe(1); + + let messageIndex = 0; + while (messageIndex < postedMessages.length) { + const event = { data: postedMessages[messageIndex++] } as MessageEvent; + for (const listener of [...messageListeners]) { + listener(event); + } + } + + expect(messageIndex).toBe(4); + expect(tokenizeLineCount).toBe(4); + expect(messageListeners.size).toBe(0); + + tokenizer.getStringCommentRegexpRangesInLine(3); + expect(tokenizeLineCount).toBe(4); + } finally { + tokenizer.cleanUp(); + globalThis.addEventListener = originalAddEventListener; + globalThis.removeEventListener = originalRemoveEventListener; + globalThis.postMessage = originalPostMessage; + globalThis.setTimeout = originalSetTimeout; + Object.defineProperty(performance, 'now', { + configurable: true, + value: originalPerformanceNow, + }); + } + }); + test('settles zero-line edits before the viewport without rebuilding to the viewport', () => { let tokenizeLineCount = 0; const grammar = { @@ -1381,6 +1588,199 @@ describe('EditorTokenizer', () => { expect([...dirtyLines.keys()]).toEqual([0, 750]); }); + test('maps mixed line-count changes and completes an EOF insertion', () => { + const originalAddEventListener = globalThis.addEventListener; + const originalRemoveEventListener = globalThis.removeEventListener; + const originalPostMessage = globalThis.postMessage; + const messageListeners = new Set(); + const postedMessages: unknown[] = []; + const states = new Map(); + let tokenizeLineCount = 0; + + globalThis.addEventListener = (( + type: string, + listener: EventListenerOrEventListenerObject + ) => { + if (type === 'message' && typeof listener === 'function') { + messageListeners.add(listener); + } + }) as typeof globalThis.addEventListener; + globalThis.removeEventListener = (( + type: string, + listener: EventListenerOrEventListenerObject + ) => { + if (type === 'message' && typeof listener === 'function') { + messageListeners.delete(listener); + } + }) as typeof globalThis.removeEventListener; + globalThis.postMessage = ((message: unknown) => { + postedMessages.push(message); + }) as typeof globalThis.postMessage; + + const grammar = { + tokenizeLine2(lineText: string) { + let nextState = states.get(lineText); + if (nextState === undefined) { + nextState = { + equals(other: StateStack | null) { + return other === nextState; + }, + } as unknown as StateStack; + states.set(lineText, nextState); + } + tokenizeLineCount++; + return { + tokens: new Uint32Array([0, 0]), + ruleStack: nextState, + stoppedEarly: false, + lineText, + }; + }, + } as unknown as IGrammar; + const textDocument = new TextDocument( + 'test.ts', + Array.from({ length: 12 }, (_, index) => `line ${index}`).join('\n'), + 'typescript' + ); + const tokenizer = new EditorTokenizer({ + highlighter: createTestHighlighter({ + getLanguage: () => grammar, + }), + textDocument, + codeOptions: { theme: 'test-theme', themeType: 'dark' }, + matchBrackets: false, + setStyle: noopSetStyle, + onDeferTokenize: () => {}, + }); + + try { + tokenizer.tokenize( + { + startLine: 0, + startCharacter: 0, + endCharacter: 0, + endLine: textDocument.lineCount - 1, + endedAtDocumentEnd: false, + previousLineCount: textDocument.lineCount, + lineCount: textDocument.lineCount, + lineDelta: 0, + changedLineRanges: [[0, textDocument.lineCount - 1]], + }, + { + startingLine: 0, + totalLines: textDocument.lineCount, + bufferBefore: 0, + bufferAfter: 0, + } + ); + + const change = textDocument.applyEdits([ + { + range: { + start: { line: 1, character: 6 }, + end: { line: 1, character: 6 }, + }, + newText: '\na', + }, + { + range: { + start: { line: 6, character: 6 }, + end: { line: 7, character: 0 }, + }, + newText: '', + }, + ])!; + expect(change.lineDelta).toBe(0); + expect(change.changedLineChanges).toEqual([ + [1, 2, 1, 6, 6, false], + [7, 7, -1, 6, 0, false], + ]); + + tokenizeLineCount = 0; + postedMessages.length = 0; + const dirtyLines = tokenizer.tokenize(change, { + startingLine: 1, + totalLines: 2, + bufferBefore: 0, + bufferAfter: 0, + }); + expect([...dirtyLines.keys()]).toEqual([1, 2]); + expect(tokenizeLineCount).toBe(2); + + tokenizeLineCount = 0; + let messageIndex = 0; + while (messageIndex < postedMessages.length) { + const event = { data: postedMessages[messageIndex++] } as MessageEvent; + for (const listener of [...messageListeners]) { + listener(event); + } + } + + // State reconverges on the first unchanged line after the deletion, so + // the untouched document tail is not tokenized. + expect(tokenizeLineCount).toBe(6); + expect(messageListeners.size).toBe(0); + + const lineCount = textDocument.lineCount; + tokenizer.tokenize( + { + startLine: 9, + startCharacter: 0, + endCharacter: 0, + endLine: lineCount - 1, + endedAtDocumentEnd: false, + previousLineCount: lineCount, + lineCount, + lineDelta: 0, + changedLineRanges: [[9, lineCount - 1]], + }, + { + startingLine: 9, + totalLines: lineCount - 9, + bufferBefore: 0, + bufferAfter: 0, + } + ); + + const eofChange = textDocument.applyEdits([ + { + range: { + start: { line: lineCount - 1, character: 7 }, + end: { line: lineCount - 1, character: 7 }, + }, + newText: '\ntail', + }, + ])!; + expect(eofChange.changedLineChanges).toEqual([[11, 12, 1, 7, 7, true]]); + + tokenizeLineCount = 0; + postedMessages.length = 0; + tokenizer.tokenize(eofChange, { + startingLine: 11, + totalLines: 1, + bufferBefore: 0, + bufferAfter: 0, + }); + expect(tokenizeLineCount).toBe(1); + + tokenizeLineCount = 0; + messageIndex = 0; + while (messageIndex < postedMessages.length) { + const event = { data: postedMessages[messageIndex++] } as MessageEvent; + for (const listener of [...messageListeners]) { + listener(event); + } + } + expect(tokenizeLineCount).toBe(1); + expect(messageListeners.size).toBe(0); + } finally { + tokenizer.cleanUp(); + globalThis.addEventListener = originalAddEventListener; + globalThis.removeEventListener = originalRemoveEventListener; + globalThis.postMessage = originalPostMessage; + } + }); + test('pins a dual-theme surface to an explicit themeType instead of following the page', () => { const originalMatchMedia = globalThis.window.matchMedia; let mediaListenerCount = 0; @@ -1538,6 +1938,148 @@ describe('EditorTokenizer', () => { } }); + test('ignores system-theme mutations until the resolved theme changes', () => { + const originalPostMessage = globalThis.postMessage; + const originalGetComputedStyle = Reflect.get( + globalThis, + 'getComputedStyle' + ); + const originalDocument = Reflect.get(globalThis, 'document'); + const originalMutationObserver = Reflect.get( + globalThis, + 'MutationObserver' + ); + const postedMessages: unknown[] = []; + let colorScheme: 'light' | 'dark' = 'dark'; + let observerCallback: MutationCallback | undefined; + let themeChangeCount = 0; + let tokenizer: EditorTokenizer | undefined; + const documentStub = { + body: {}, + documentElement: {}, + }; + + globalThis.postMessage = ((message: unknown) => { + postedMessages.push(message); + }) as typeof globalThis.postMessage; + Reflect.set(globalThis, 'document', documentStub); + Reflect.set( + globalThis, + 'getComputedStyle', + (() => + ({ + colorScheme, + }) as CSSStyleDeclaration) as typeof getComputedStyle + ); + Reflect.set( + globalThis, + 'MutationObserver', + class { + constructor(callback: MutationCallback) { + observerCallback = callback; + } + observe() {} + disconnect() {} + takeRecords() { + return []; + } + } + ); + + try { + const grammar = { + tokenizeLine2(lineText: string, ruleStack: StateStack) { + return { + tokens: new Uint32Array([0, 0]), + ruleStack, + stoppedEarly: false, + lineText, + }; + }, + } as unknown as IGrammar; + const textDocument = new TextDocument( + 'test.ts', + ['line 0', 'line 1'].join('\n'), + 'typescript' + ); + tokenizer = new EditorTokenizer({ + highlighter: createTestHighlighter({ + getLanguage: () => grammar, + }), + textDocument, + codeOptions: { + theme: { light: 'light-theme', dark: 'dark-theme' }, + themeType: 'system', + }, + setStyle: noopSetStyle, + onDeferTokenize: () => {}, + onThemeChange: () => { + themeChangeCount++; + }, + }); + + const observer = {} as MutationObserver; + observerCallback?.( + [ + { + attributeName: 'class', + target: documentStub.documentElement, + type: 'attributes', + } as unknown as MutationRecord, + ], + observer + ); + observerCallback?.( + [ + { + attributeName: 'data-layout', + target: documentStub.body, + type: 'attributes', + } as unknown as MutationRecord, + ], + observer + ); + + expect(tokenizer.themeType).toBe('dark'); + expect(themeChangeCount).toBe(0); + expect(postedMessages).toHaveLength(0); + + colorScheme = 'light'; + observerCallback?.( + [ + { + attributeName: 'data-theme', + target: documentStub.body, + type: 'attributes', + } as unknown as MutationRecord, + ], + observer + ); + + expect(tokenizer.themeType).toBe('light'); + expect(themeChangeCount).toBe(1); + expect(postedMessages).toHaveLength(1); + } finally { + tokenizer?.cleanUp(); + globalThis.postMessage = originalPostMessage; + if (originalGetComputedStyle === undefined) { + Reflect.deleteProperty(globalThis, 'getComputedStyle'); + } else { + Reflect.set(globalThis, 'getComputedStyle', originalGetComputedStyle); + } + if (originalDocument === undefined) { + Reflect.deleteProperty(globalThis, 'document'); + } else { + Reflect.set(globalThis, 'document', originalDocument); + } + if (originalMutationObserver === undefined) { + Reflect.deleteProperty(globalThis, 'MutationObserver'); + } else { + Reflect.set(globalThis, 'MutationObserver', originalMutationObserver); + } + } + }); + // Dual-theme SSR (`themes: {dark,light}`) leaves the shared highlighter on the // last theme it applied (usually light). The tokenizer caches a single-theme // colorMap from construction; without re-activating that theme before an edit, From 835816319b5e09894351b1954a12a80b192cc8e4 Mon Sep 17 00:00:00 2001 From: Je Xia Date: Mon, 27 Jul 2026 20:03:01 +0800 Subject: [PATCH 2/3] move benchmark code in `test/editorPieceTable.test.ts` to `scripts/benchmarkEditorPieceTable.ts` --- packages/diffs/moon.yml | 6 + .../scripts/benchmarkEditorPieceTable.ts | 194 +++++++++++++++++ packages/diffs/test/editorPieceTable.test.ts | 197 ------------------ 3 files changed, 200 insertions(+), 197 deletions(-) create mode 100644 packages/diffs/scripts/benchmarkEditorPieceTable.ts diff --git a/packages/diffs/moon.yml b/packages/diffs/moon.yml index 9a8b4c412..b2d72d286 100644 --- a/packages/diffs/moon.yml +++ b/packages/diffs/moon.yml @@ -21,6 +21,12 @@ tasks: cache: false runInCI: 'always' + benchmark-editor-piece-table: + command: 'bun scripts/benchmarkEditorPieceTable.ts' + options: + cache: false + runInCI: 'always' + benchmark-editor-tokenizer: command: 'bun scripts/benchmarkEditorTokenizer.ts' options: diff --git a/packages/diffs/scripts/benchmarkEditorPieceTable.ts b/packages/diffs/scripts/benchmarkEditorPieceTable.ts new file mode 100644 index 000000000..f615283ea --- /dev/null +++ b/packages/diffs/scripts/benchmarkEditorPieceTable.ts @@ -0,0 +1,194 @@ +import { PieceTable } from '../src/editor/pieceTable'; + +const line = 'function benchmark(value: number) { return value + 1; }\n'; +const lineCount = 12_000; +const text = line.repeat(lineCount); +const editCount = 2_000; +const scatteredOffsets = Array.from({ length: editCount }, (_, i) => { + const lineIndex = (i * 7919) % lineCount; + return lineIndex * line.length + 9 + (i % 20); +}); +const batchEdits = Array.from({ length: editCount }, (_, i) => { + const lineIndex = Math.floor(((i + 1) * lineCount) / (editCount + 1)); + const start = lineIndex * line.length + 20; + return { start, end: start + 1, text: i % 2 === 0 ? 'x' : 'y' }; +}); +const lookupOffsets = Array.from( + { length: 20_000 }, + (_, i) => (i * 104729) % text.length +); +const sliceOffsets = Array.from( + { length: 5_000 }, + (_, i) => (i * 48611) % (text.length - 80) +); + +const createFragmentedTable = () => { + const table = new PieceTable(text); + for (let i = 0; i < scatteredOffsets.length; i++) { + const start = scatteredOffsets[i]; + table.applyEdits([ + { start, end: start + 1, text: i % 2 === 0 ? 'q' : 'z' }, + ]); + } + return table; +}; + +const scenarios: { + name: string; + operations: number; + prepare: () => () => number; +}[] = [ + { + name: 'construct/large-document', + operations: 1, + prepare: () => () => new PieceTable(text).lineCount, + }, + { + name: 'edit/sequential-typing', + operations: 5_000, + prepare: () => { + const table = new PieceTable(''); + return () => { + for (let i = 0; i < 5_000; i++) { + table.insert('x', i); + } + return table.getText().length; + }; + }, + }, + { + name: 'edit/scattered-inserts', + operations: scatteredOffsets.length, + prepare: () => { + const table = new PieceTable(text); + return () => { + for (const offset of scatteredOffsets) { + table.insert('x', offset); + } + return table.lineCount; + }; + }, + }, + { + name: 'edit/append-after-fragmentation', + operations: 2_000, + prepare: () => { + const table = createFragmentedTable(); + const end = text.length; + return () => { + for (let i = 0; i < 2_000; i++) { + table.insert('x', end + i); + } + return table.lineCount; + }; + }, + }, + { + name: 'edit/batch-replacements', + operations: batchEdits.length, + prepare: () => { + const table = new PieceTable(text); + return () => { + table.applyEdits(batchEdits); + return table.lineCount; + }; + }, + }, + { + name: 'read/fragmented-position-round-trips', + operations: lookupOffsets.length, + prepare: () => { + const table = createFragmentedTable(); + return () => { + let checksum = 0; + for (const offset of lookupOffsets) { + const position = table.positionAt(offset); + checksum += table.offsetAt(position); + } + return checksum; + }; + }, + }, + { + name: 'read/fragmented-slices', + operations: sliceOffsets.length, + prepare: () => { + const table = createFragmentedTable(); + return () => { + let checksum = 0; + for (const start of sliceOffsets) { + checksum += table.getTextSlice(start, start + 80).length; + } + return checksum; + }; + }, + }, + { + name: 'search/fragmented-whole-word', + operations: lineCount, + prepare: () => { + const table = createFragmentedTable(); + return () => + table.search({ + text: 'function', + replaceText: '', + caseSensitive: true, + wholeWord: true, + regex: false, + }).length; + }, + }, +]; + +const results: { + name: string; + operations: number; + p50Ms: number; + p95Ms: number; + minMs: number; + maxMs: number; + opsPerSecond: number; + samplesMs: number[]; +}[] = []; +let checksum = 0; +for (const scenario of scenarios) { + for (let i = 0; i < 2; i++) { + checksum += scenario.prepare()(); + } + + const samplesMs: number[] = []; + for (let i = 0; i < 9; i++) { + const run = scenario.prepare(); + const start = performance.now(); + checksum += run(); + samplesMs.push(performance.now() - start); + } + samplesMs.sort((a, b) => a - b); + const p50Ms = samplesMs[Math.floor(samplesMs.length / 2)]; + const p95Ms = samplesMs[Math.ceil(samplesMs.length * 0.95) - 1]; + results.push({ + name: scenario.name, + operations: scenario.operations, + p50Ms, + p95Ms, + minMs: samplesMs[0], + maxMs: samplesMs[samplesMs.length - 1], + opsPerSecond: scenario.operations / (p50Ms / 1000), + samplesMs, + }); +} + +if (checksum <= 0) { + throw new Error('PieceTable benchmark produced an invalid checksum'); +} +console.table( + results.map(({ name, p50Ms, p95Ms, minMs, maxMs, opsPerSecond }) => ({ + name, + p50Ms: p50Ms.toFixed(3), + p95Ms: p95Ms.toFixed(3), + minMs: minMs.toFixed(3), + maxMs: maxMs.toFixed(3), + opsPerSecond: Math.round(opsPerSecond), + })) +); +console.log(`PIECE_TABLE_BENCHMARK_RESULTS=${JSON.stringify(results)}`); diff --git a/packages/diffs/test/editorPieceTable.test.ts b/packages/diffs/test/editorPieceTable.test.ts index 8ce089fb0..d072bea86 100644 --- a/packages/diffs/test/editorPieceTable.test.ts +++ b/packages/diffs/test/editorPieceTable.test.ts @@ -1218,200 +1218,3 @@ describe('very long single line', () => { expect(d.getText()).toBe(str); }); }); - -test.skipIf(process.env.PIECE_TABLE_BENCHMARK !== '1')( - 'benchmarks representative PieceTable workloads', - () => { - const line = 'function benchmark(value: number) { return value + 1; }\n'; - const lineCount = 12_000; - const text = line.repeat(lineCount); - const editCount = 2_000; - const scatteredOffsets = Array.from({ length: editCount }, (_, i) => { - const lineIndex = (i * 7919) % lineCount; - return lineIndex * line.length + 9 + (i % 20); - }); - const batchEdits = Array.from({ length: editCount }, (_, i) => { - const lineIndex = Math.floor(((i + 1) * lineCount) / (editCount + 1)); - const start = lineIndex * line.length + 20; - return { start, end: start + 1, text: i % 2 === 0 ? 'x' : 'y' }; - }); - const lookupOffsets = Array.from( - { length: 20_000 }, - (_, i) => (i * 104729) % text.length - ); - const sliceOffsets = Array.from( - { length: 5_000 }, - (_, i) => (i * 48611) % (text.length - 80) - ); - - const createFragmentedTable = () => { - const table = new PieceTable(text); - for (let i = 0; i < scatteredOffsets.length; i++) { - const start = scatteredOffsets[i]; - table.applyEdits([ - { start, end: start + 1, text: i % 2 === 0 ? 'q' : 'z' }, - ]); - } - return table; - }; - - const scenarios: { - name: string; - operations: number; - prepare: () => () => number; - }[] = [ - { - name: 'construct/large-document', - operations: 1, - prepare: () => () => new PieceTable(text).lineCount, - }, - { - name: 'edit/sequential-typing', - operations: 5_000, - prepare: () => { - const table = new PieceTable(''); - return () => { - for (let i = 0; i < 5_000; i++) { - table.insert('x', i); - } - return table.getText().length; - }; - }, - }, - { - name: 'edit/scattered-inserts', - operations: scatteredOffsets.length, - prepare: () => { - const table = new PieceTable(text); - return () => { - for (const offset of scatteredOffsets) { - table.insert('x', offset); - } - return table.lineCount; - }; - }, - }, - { - name: 'edit/append-after-fragmentation', - operations: 2_000, - prepare: () => { - const table = createFragmentedTable(); - const end = text.length; - return () => { - for (let i = 0; i < 2_000; i++) { - table.insert('x', end + i); - } - return table.lineCount; - }; - }, - }, - { - name: 'edit/batch-replacements', - operations: batchEdits.length, - prepare: () => { - const table = new PieceTable(text); - return () => { - table.applyEdits(batchEdits); - return table.lineCount; - }; - }, - }, - { - name: 'read/fragmented-position-round-trips', - operations: lookupOffsets.length, - prepare: () => { - const table = createFragmentedTable(); - return () => { - let checksum = 0; - for (const offset of lookupOffsets) { - const position = table.positionAt(offset); - checksum += table.offsetAt(position); - } - return checksum; - }; - }, - }, - { - name: 'read/fragmented-slices', - operations: sliceOffsets.length, - prepare: () => { - const table = createFragmentedTable(); - return () => { - let checksum = 0; - for (const start of sliceOffsets) { - checksum += table.getTextSlice(start, start + 80).length; - } - return checksum; - }; - }, - }, - { - name: 'search/fragmented-whole-word', - operations: lineCount, - prepare: () => { - const table = createFragmentedTable(); - return () => - table.search({ - text: 'function', - replaceText: '', - caseSensitive: true, - wholeWord: true, - regex: false, - }).length; - }, - }, - ]; - - const results: { - name: string; - operations: number; - p50Ms: number; - p95Ms: number; - minMs: number; - maxMs: number; - opsPerSecond: number; - samplesMs: number[]; - }[] = []; - let checksum = 0; - for (const scenario of scenarios) { - for (let i = 0; i < 2; i++) { - checksum += scenario.prepare()(); - } - - const samplesMs: number[] = []; - for (let i = 0; i < 9; i++) { - const run = scenario.prepare(); - const start = performance.now(); - checksum += run(); - samplesMs.push(performance.now() - start); - } - samplesMs.sort((a, b) => a - b); - const p50Ms = samplesMs[Math.floor(samplesMs.length / 2)]; - const p95Ms = samplesMs[Math.ceil(samplesMs.length * 0.95) - 1]; - results.push({ - name: scenario.name, - operations: scenario.operations, - p50Ms, - p95Ms, - minMs: samplesMs[0], - maxMs: samplesMs[samplesMs.length - 1], - opsPerSecond: scenario.operations / (p50Ms / 1000), - samplesMs, - }); - } - - expect(checksum).toBeGreaterThan(0); - console.table( - results.map(({ name, p50Ms, p95Ms, minMs, maxMs, opsPerSecond }) => ({ - name, - p50Ms: p50Ms.toFixed(3), - p95Ms: p95Ms.toFixed(3), - minMs: minMs.toFixed(3), - maxMs: maxMs.toFixed(3), - opsPerSecond: Math.round(opsPerSecond), - })) - ); - console.log(`PIECE_TABLE_BENCHMARK_RESULTS=${JSON.stringify(results)}`); - }, - { timeout: 120_000 } -); From ff18cb6a63d596dd2e147b8418332b592ba676f4 Mon Sep 17 00:00:00 2001 From: Je Xia Date: Mon, 27 Jul 2026 20:26:11 +0800 Subject: [PATCH 3/3] fix codex --- packages/diffs/src/editor/tokenizer.ts | 22 ++++++- packages/diffs/test/editorTokenizer.test.ts | 69 ++++++++++++++++++--- 2 files changed, 81 insertions(+), 10 deletions(-) diff --git a/packages/diffs/src/editor/tokenizer.ts b/packages/diffs/src/editor/tokenizer.ts index 3a84f664e..10e72ca7c 100644 --- a/packages/diffs/src/editor/tokenizer.ts +++ b/packages/diffs/src/editor/tokenizer.ts @@ -70,6 +70,7 @@ export class EditorTokenizer { #backgroundJobId: number = 0; #tokenizerId = ++nextTokenizerId; #backgroundPrebuildEndLine = -1; + #pendingPrebuildEndLine = -1; #backgroundChangedLineRanges: readonly [number, number][] | undefined; #backgroundChangedRangeIndex: number = 0; #bracketIgnoredRanges: ([number, number][] | null | undefined)[] = []; @@ -569,6 +570,7 @@ export class EditorTokenizer { } stopBackgroundTokenize(): void { + this.#pendingPrebuildEndLine = -1; if (this.#isStopped) { return; } @@ -676,6 +678,12 @@ export class EditorTokenizer { this.#isStopped = false; this.#isPaused = false; this.#lastLine = startLine; + if (this.#backgroundPrebuildEndLine >= 0) { + this.#pendingPrebuildEndLine = Math.max( + this.#pendingPrebuildEndLine, + this.#backgroundPrebuildEndLine + ); + } this.#backgroundPrebuildEndLine = -1; this.#backgroundChangedLineRanges = changedLineRanges; this.#backgroundChangedRangeIndex = changedRangeIndex; @@ -687,14 +695,19 @@ export class EditorTokenizer { if (this.#grammar === undefined || this.#stateStack.length > endLine) { return; } - // Keep an active state-only job and extend it when a later viewport needs - // more state. A foreground edit job remains authoritative. + // Extend an active prebuild, or retain the target until the foreground + // edit job reconverges and releases the state cache. if (!this.#isStopped) { if (this.#backgroundPrebuildEndLine >= 0) { this.#backgroundPrebuildEndLine = Math.max( this.#backgroundPrebuildEndLine, endLine ); + } else { + this.#pendingPrebuildEndLine = Math.max( + this.#pendingPrebuildEndLine, + endLine + ); } return; } @@ -704,6 +717,7 @@ export class EditorTokenizer { this.#isPaused = false; this.#lastLine = this.#stateStack.length - 1; this.#backgroundPrebuildEndLine = endLine; + this.#pendingPrebuildEndLine = -1; this.#backgroundChangedLineRanges = undefined; this.#backgroundChangedRangeIndex = 0; this.#attachMessageListener(); @@ -961,7 +975,11 @@ export class EditorTokenizer { } if (settled || line >= totalLines) { + const pendingPrebuildEndLine = this.#pendingPrebuildEndLine; this.stopBackgroundTokenize(); + if (pendingPrebuildEndLine >= 0) { + this.#scheduleStatePrebuild(pendingPrebuildEndLine); + } return; } diff --git a/packages/diffs/test/editorTokenizer.test.ts b/packages/diffs/test/editorTokenizer.test.ts index 0f293587a..c53a6c665 100644 --- a/packages/diffs/test/editorTokenizer.test.ts +++ b/packages/diffs/test/editorTokenizer.test.ts @@ -1074,7 +1074,7 @@ describe('EditorTokenizer', () => { } }); - test('prebuilds state only from queued messages and detaches when complete', () => { + test('queues state prebuilds and resumes them after foreground work', () => { const originalAddEventListener = globalThis.addEventListener; const originalRemoveEventListener = globalThis.removeEventListener; const originalPostMessage = globalThis.postMessage; @@ -1113,26 +1113,37 @@ describe('EditorTokenizer', () => { value: () => (now += 2), }); + const states = new Map(); const grammar = { - tokenizeLine2(lineText: string, ruleStack: StateStack) { + tokenizeLine2(lineText: string) { + let nextState = states.get(lineText); + if (nextState === undefined) { + nextState = { + equals(other: StateStack | null) { + return other === nextState; + }, + } as unknown as StateStack; + states.set(lineText, nextState); + } tokenizeLineCount++; return { tokens: new Uint32Array([0, 0]), - ruleStack, + ruleStack: nextState, stoppedEarly: false, lineText, }; }, } as unknown as IGrammar; + const textDocument = new TextDocument( + 'test.ts', + Array.from({ length: 100 }, (_, line) => `line ${line}`).join('\n'), + 'typescript' + ); const tokenizer = new EditorTokenizer({ highlighter: createTestHighlighter({ getLanguage: () => grammar, }), - textDocument: new TextDocument( - 'test.ts', - ['line 0', 'line 1', 'line 2', 'line 3'].join('\n'), - 'typescript' - ), + textDocument, codeOptions: { theme: 'test-theme', themeType: 'dark' }, setStyle: noopSetStyle, onDeferTokenize: () => {}, @@ -1169,6 +1180,48 @@ describe('EditorTokenizer', () => { tokenizer.getStringCommentRegexpRangesInLine(3); expect(tokenizeLineCount).toBe(4); + + const change = textDocument.applyEdits([ + { + range: { + start: { line: 0, character: 0 }, + end: { line: 0, character: 6 }, + }, + newText: 'LINE 0', + }, + ])!; + tokenizeLineCount = 0; + postedMessages.length = 0; + tokenizer.tokenize(change, { + startingLine: 0, + totalLines: 1, + bufferBefore: 0, + bufferAfter: 0, + }); + expect(postedMessages).toHaveLength(1); + + tokenizer.prebuildStateStack({ + startingLine: 99, + totalLines: 1, + bufferBefore: 0, + bufferAfter: 0, + }); + expect(postedMessages).toHaveLength(1); + + messageIndex = 0; + while (messageIndex < postedMessages.length) { + const event = { data: postedMessages[messageIndex++] } as MessageEvent; + for (const listener of [...messageListeners]) { + listener(event); + } + } + + expect(tokenizeLineCount).toBeGreaterThan(2); + expect(messageListeners.size).toBe(0); + + const completedTokenizeLineCount = tokenizeLineCount; + tokenizer.getStringCommentRegexpRangesInLine(99); + expect(tokenizeLineCount).toBe(completedTokenizeLineCount); } finally { tokenizer.cleanUp(); globalThis.addEventListener = originalAddEventListener;