chore(trace): split actions and context options into separate files

packages/isomorphic/trace/traceLoader.ts

@@ -39,12 +39,13 @@ export class TraceLoader {
   async load(backend: TraceLoaderBackend, traceFile?: string, unzipProgress?: (done: number, total: number) => void) {
     this._backend = backend;
 
-    const prefix = traceFile?.match(/(.+)\.trace$/)?.[1];
+    const requestedPrefix = traceFile?.match(/(.+)\.trace$/)?.[1];
     const prefixes: string[] = [];
+    const entryNames = await this._backend.entryNames();
     let hasSource = false;
-    for (const entryName of await this._backend.entryNames()) {
+    for (const entryName of entryNames) {
       const match = entryName.match(/(.+)\.trace$/);
-      if (match && (!prefix || prefix  === match[1]))
+      if (match && (!requestedPrefix || requestedPrefix === match[1]))
         prefixes.push(match[1] || '');
       if (entryName.startsWith('src/') || entryName.includes('src@'))
         hasSource = true;
@@ -54,21 +55,19 @@ export class TraceLoader {
 
     this._snapshotStorage = new SnapshotStorage();
 
-    // 3 * ordinals progress increments below.
-    const total = prefixes.length * 3;
+    const traceFilesByPrefix = new Map(prefixes.map(prefix => [prefix, traceFileNames(entryNames, prefix)]));
+    const total = prefixes.length + [...traceFilesByPrefix.values()].reduce((sum, files) => sum + files.length, 0);
     let done = 0;
     for (const prefix of prefixes) {
       const contextEntry = createEmptyContext();
       contextEntry.hasSource = hasSource;
       const modernizer = new TraceModernizer(contextEntry, this._snapshotStorage);
 
-      const trace = await this._backend.readText(prefix + '.trace') || '';
-      modernizer.appendTrace(trace);
-      unzipProgress?.(++done, total);
-
-      const network = await this._backend.readText(prefix + '.network') || '';
-      modernizer.appendTrace(network);
-      unzipProgress?.(++done, total);
+      for (const traceFileName of traceFilesByPrefix.get(prefix)!) {
+        const trace = await this._backend.readText(traceFileName) || '';
+        modernizer.appendTrace(trace);
+        unzipProgress?.(++done, total);
+      }
 
       const stacks = await this._backend.readText(prefix + '.stacks');
       if (stacks)
@@ -122,6 +121,32 @@ export class TraceLoader {
   }
 }
 
+// The ".meta" file carries the trace version required to read everything else, and actions
+// are referenced by the events in the ".trace" file, hence the order. Files with an unknown
+// extension come last. The ".stacks" file is not a trace stream and is read separately.
+const kTraceFileExtensions = ['.meta', '.actions', '.trace', '.network'];
+const kNonTraceFileExtensions = ['.stacks'];
+
+function traceFileNames(entryNames: string[], prefix: string): string[] {
+  const remaining = new Set<string>();
+  for (const entryName of entryNames) {
+    if (!entryName.startsWith(prefix))
+      continue;
+    // Only take "<prefix>.<extension>" files, e.g. neither "<prefix>-chunk1.trace" nor "<prefix>.trace.zip".
+    const extension = entryName.substring(prefix.length);
+    if (/^\.[a-z]+$/.test(extension) && !kNonTraceFileExtensions.includes(extension))
+      remaining.add(entryName);
+  }
+
+  const result: string[] = [];
+  for (const extension of kTraceFileExtensions) {
+    if (remaining.delete(prefix + extension))
+      result.push(prefix + extension);
+  }
+  result.push(...remaining);
+  return result;
+}
+
 function stripEncodingFromContentType(contentType: string) {
   const charset = contentType.match(/^(.*);\s*charset=.*$/);
   if (charset)

packages/isomorphic/trace/traceModernizer.ts

@@ -149,13 +149,14 @@ export class TraceModernizer {
       }
       case 'input': {
         const existing = this._actionMap.get(event.callId);
-        existing!.point = event.point;
-        existing!.box = event.box;
+        if (!existing)
+          return;
+        existing.point = event.point;
+        existing.box = event.box;
         break;
       }
       case 'log': {
         const existing = this._actionMap.get(event.callId);
-        // We have some corrupted traces out there, tolerate them.
         if (!existing)
           return;
         existing.log.push({

packages/playwright-core/src/server/trace/recorder/tracing.ts

@@ -71,6 +71,8 @@ type RecordingState = {
   traceName: string,
   networkFile: string,
   traceFile: string,
+  actionsFile: string,
+  metaFile: string,
   tracesDir: string,
   chunkOrdinal: number,
   // Blobs referenced by the network stream. The network file is preserved between
@@ -165,6 +167,8 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps
       traceName,
       tracesDir,
       traceFile: path.join(tracesDir, traceName + '.trace'),
+      actionsFile: path.join(tracesDir, traceName + '.actions'),
+      metaFile: path.join(tracesDir, traceName + '.meta'),
       networkFile: path.join(tracesDir, traceName + '.network'),
       chunkOrdinal: 0,
       chunkFiles: new Set(),
@@ -219,7 +223,9 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps
       wallTime: Date.now(),
       monotonicTime: monotonicTime()
     };
-    this._appendTraceEvent(event);
+    this._appendTraceEvent(event); // Write the meta file before anything else.
+    this._fs.writeFile(this._state.traceFile, '', true /* skipIfExists */);
+    this._fs.writeFile(this._state.actionsFile, '', true /* skipIfExists */);
 
     this._context.instrumentation.addListener(this, this._context);
     this._eventListeners.push(
@@ -310,6 +316,8 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps
     const suffix = state.chunkOrdinal ? `-chunk${state.chunkOrdinal}` : ``;
     state.chunkOrdinal++;
     state.traceFile = path.join(state.tracesDir, `${state.traceName}${suffix}.trace`);
+    state.actionsFile = path.join(state.tracesDir, `${state.traceName}${suffix}.actions`);
+    state.metaFile = path.join(state.tracesDir, `${state.traceName}${suffix}.meta`);
   }
 
   private _changeTraceName(state: RecordingState, name: string, preserveNetworkResources: boolean) {
@@ -451,7 +459,9 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps
     const newNetworkFile = path.join(this._state.tracesDir, this._state.traceName + `-pwnetcopy-${this._state.chunkOrdinal}.network`);
 
     const entries: NameValue[] = [];
+    entries.push({ name: 'trace.meta', value: this._state.metaFile });
     entries.push({ name: 'trace.trace', value: this._state.traceFile });
+    entries.push({ name: 'trace.actions', value: this._state.actionsFile });
     entries.push({ name: 'trace.network', value: newNetworkFile });
     for (const file of new Set([...this._state.chunkFiles, ...this._state.crossChunkFiles]))
       entries.push({ name: file, value: path.join(this._state.tracesDir, file) });
@@ -737,10 +747,21 @@ export class Tracing extends SdkObject implements InstrumentationListener, Snaps
   }
 
   private _appendTraceEvent(event: trace.TraceEvent) {
+    if (event.type === 'context-options') {
+      this._appendEventToFile(this._state!.metaFile, event);
+    } else if (event.type === 'before' || event.type === 'after') {
+      this._fs.flushFile(this._state!.traceFile); // Flush pending events upon an action.
+      this._appendEventToFile(this._state!.actionsFile, event);
+    } else {
+      this._appendEventToFile(this._state!.traceFile, event);
+    }
+  }
+
+  private _appendEventToFile(file: string, event: trace.TraceEvent) {
     const visited = visitTraceEvent(event);
     // Do not flush (console) events, they are too noisy, unless we are in ui mode (live).
     const flush = this._state!.options.live || (event.type !== 'event' && event.type !== 'console' && event.type !== 'log');
-    this._fs.appendFile(this._state!.traceFile, JSON.stringify(visited) + '\n', flush);
+    this._fs.appendFile(file, JSON.stringify(visited) + '\n', flush);
   }
 
   private _appendResource(file: string, buffer: Buffer) {

packages/playwright-core/src/tools/backend/tracing.ts

@@ -40,7 +40,8 @@ const tracingStart = defineTool({
       live: true,
     });
     response.addTextResult(`Trace recording started`);
-    response.addFileLink('Action log', `${tracesDir}/${name}.trace`);
+    response.addFileLink('Action log', `${tracesDir}/${name}.actions`);
+    response.addFileLink('Trace', `${tracesDir}/${name}.trace`);
     response.addFileLink('Network log', `${tracesDir}/${name}.network`);
     response.addFileLink('Resources', `${tracesDir}/resources`);
     // eslint-disable-next-line no-restricted-syntax
@@ -70,6 +71,7 @@ const tracingStop = defineTool({
     delete (browserContext.tracing as any)[traceLegendSymbol];
 
     response.addTextResult(`Trace recording stopped.`);
+    response.addFileLink('Action log', `${traceLegend.tracesDir}/${traceLegend.name}.actions`);
     response.addFileLink('Trace', `${traceLegend.tracesDir}/${traceLegend.name}.trace`);
     response.addFileLink('Network log', `${traceLegend.tracesDir}/${traceLegend.name}.network`);
     response.addFileLink('Resources', `${traceLegend.tracesDir}/resources`);

packages/utils/serializedFS.ts

@@ -68,10 +68,10 @@ export class SerializedFS {
     for (const chunk of buffer)
       size += chunk.length;
     if (flush || size >= APPEND_CHUNK_SIZE)
-      this._flushFile(file);
+      this.flushFile(file);
   }
 
-  private _flushFile(file: string) {
+  flushFile(file: string) {
     const buffer = this._buffers.get(file);
     if (buffer === undefined)
       return;
@@ -81,14 +81,14 @@ export class SerializedFS {
   }
 
   copyFile(from: string, to: string) {
-    this._flushFile(from);
+    this.flushFile(from);
     this._buffers.delete(to); // No need to flush the buffer since we'll overwrite anyway.
     this._appendOperation({ op: 'copyFile', from, to });
   }
 
   async sync() {
     for (const file of this._buffers.keys())
-      this._flushFile(file);
+      this.flushFile(file);
     await this._operationsDone;
     if (this._error) {
       const e = this._error;
@@ -99,7 +99,7 @@ export class SerializedFS {
 
   zip(entries: NameValue[], zipFileName: string) {
     for (const file of this._buffers.keys())
-      this._flushFile(file);
+      this.flushFile(file);
 
     // Chain the export operation against write operations,
     // so that files do not change during the export.

tests/config/utils.ts

@@ -113,7 +113,8 @@ export async function parseTraceRaw(file: string): Promise<{ events: any[], reso
 
   const actionMap = new Map<string, ActionTraceEvent>();
   const events: any[] = [];
-  for (const traceFile of [...resources.keys()].filter(name => name.endsWith('.trace'))) {
+  const traceFiles = ['.meta', '.actions', '.trace'].flatMap(extension => [...resources.keys()].filter(name => name.endsWith(extension)));
+  for (const traceFile of traceFiles) {
     for (const line of resources.get(traceFile)!.toString().split('\n')) {
       if (line) {
         const event = JSON.parse(line) as TraceEvent;

tests/library/tracing.spec.ts

@@ -301,6 +301,8 @@ test('should respect tracesDir and name', async ({ browserType, server, mode },
     expect(resourceNames(resources)).toEqual([
       'resources/XXX.css',
       'resources/XXX.html',
+      'trace.actions',
+      'trace.meta',
       'trace.network',
       'trace.stacks',
       'trace.trace',
@@ -314,6 +316,8 @@ test('should respect tracesDir and name', async ({ browserType, server, mode },
       'resources/XXX.css',
       'resources/XXX.html',
       'resources/XXX.html',
+      'trace.actions',
+      'trace.meta',
       'trace.network',
       'trace.stacks',
       'trace.trace',
@@ -840,13 +844,13 @@ test('should not flush console events', async ({ context, page, mode }, testInfo
 
   const dir = path.join(testInfo.project.outputDir, artifactsFolderName(testInfo.workerIndex), 'traces');
 
-  let content: string;
   await expect(async () => {
-    const traceName = fs.readdirSync(dir).find(name => name.endsWith(testId + '.trace'));
-    content = await fs.promises.readFile(path.join(dir, traceName), 'utf8');
-    expect(content).toContain('31415926');
+    const actionsName = fs.readdirSync(dir).find(name => name.endsWith(testId + '.actions'));
+    const actions = await fs.promises.readFile(path.join(dir, actionsName), 'utf8');
+    expect(actions).toContain('31415926');
   }).toPass();
-  expect(content).not.toContain('hello 0');
+  const traceName = fs.readdirSync(dir).find(name => name.endsWith(testId + '.trace'));
+  expect(await fs.promises.readFile(path.join(dir, traceName), 'utf8')).not.toContain('hello 0');
 
   await page.evaluate(() => 42);
 

tests/mcp/tracing.spec.ts

@@ -46,6 +46,8 @@ test('check that trace is saved with browser_start_tracing', async ({ startClien
   expect(files).toEqual([
     'resources',
     'screencast',
+    expect.stringMatching(/trace-\d+\.actions/),
+    expect.stringMatching(/trace-\d+\.meta/),
     expect.stringMatching(/trace-\d+\.network/),
     expect.stringMatching(/trace-\d+\.stacks/),
     expect.stringMatching(/trace-\d+\.trace/),
@@ -80,6 +82,8 @@ test('check that trace is saved with browser_start_tracing (no output dir)', asy
   expect(files).toEqual([
     'resources',
     'screencast',
+    expect.stringMatching(/trace-\d+\.actions/),
+    expect.stringMatching(/trace-\d+\.meta/),
     expect.stringMatching(/trace-\d+\.network/),
     expect.stringMatching(/trace-\d+\.stacks/),
     expect.stringMatching(/trace-\d+\.trace/),

tests/playwright-test/playwright.trace.spec.ts

@@ -212,8 +212,8 @@ test('should not mixup network files between contexts', async ({ runInlineTest,
   expect(result.passed).toBe(1);
   const tracePath = testInfo.outputPath('test-results', 'a-example', 'trace.zip');
   const { resources } = await parseTraceRaw(tracePath);
-  const traceEntries = [...resources].filter(([name]) => name.endsWith('.trace')).map(([name, content]) => ({
-    prefix: name.slice(0, -'.trace'.length),
+  const traceEntries = [...resources].filter(([name]) => name.endsWith('.meta')).map(([name, content]) => ({
+    prefix: name.slice(0, -'.meta'.length),
     contextOptions: JSON.parse(content.toString().split('\n')[0]),
   })).filter(entry => entry.contextOptions.origin === 'library');
   // Each of the 3 browser contexts and 3 api request contexts produces