diff --git a/.github/workflows/generate-stats.yml b/.github/workflows/generate-stats.yml index b929e979..f503563c 100644 --- a/.github/workflows/generate-stats.yml +++ b/.github/workflows/generate-stats.yml @@ -1,5 +1,8 @@ name: Generate Stats +env: + RUNNER_LABEL: 'Depot depot-ubuntu-24.04 GitHub Actions runner (2 CPUs, 8 GB RAM, 100 GB disk, 2 GB disk accelerator)' + on: push: branches: [main] @@ -93,8 +96,6 @@ jobs: - name: Save CI stats run: pnpm --filter @framework-tracker/stats-generator save:ci-stats $GITHUB_WORKSPACE/artifacts - env: - RUNNER_LABEL: ubuntu-latest - name: Collect stats for docs run: pnpm collect:stats diff --git a/.github/workflows/measure-framework.yml b/.github/workflows/measure-framework.yml index 3605274c..4daa1d9e 100644 --- a/.github/workflows/measure-framework.yml +++ b/.github/workflows/measure-framework.yml @@ -1,5 +1,8 @@ name: Measure Framework +env: + RUNNER_LABEL: 'Depot depot-ubuntu-24.04 GitHub Actions runner (2 CPUs, 8 GB RAM, 100 GB disk, 2 GB disk accelerator)' + on: workflow_call: inputs: @@ -169,8 +172,6 @@ jobs: - name: Run SSR request throughput benchmark run: pnpm --filter @framework-tracker/stats-generator run:ssr-request-throughput ${{ matrix.framework.package }} - env: - RUNNER_LABEL: ubuntu-latest - name: Upload SSR request throughput stats uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -249,8 +250,6 @@ jobs: - name: Run SSR load benchmark run: pnpm --filter @framework-tracker/stats-generator run:ssr-load ${{ matrix.framework.package }} - env: - RUNNER_LABEL: ubuntu-latest - name: Upload SSR load stats uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -263,9 +262,6 @@ jobs: measure-client-side-rendered: if: inputs.client-side-rendered-matrix != '[]' runs-on: depot-ubuntu-24.04 - container: - image: mcr.microsoft.com/playwright:v1.49.0-noble - options: --ipc=host strategy: fail-fast: false matrix: @@ -294,12 +290,11 @@ jobs: working-directory: ./packages/${{ matrix.framework.package }} run: pnpm build + - name: Verify runner Chrome + run: google-chrome --version + - name: Run client-side rendered benchmark run: pnpm --filter @framework-tracker/stats-generator run:client-side-rendered ${{ matrix.framework.package }} - env: - PLAYWRIGHT_BROWSERS_PATH: /ms-playwright - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1' - RUNNER_LABEL: ubuntu-latest - name: Upload client-side rendered stats uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -312,9 +307,6 @@ jobs: measure-server-side-rendered: if: inputs.server-side-rendered-matrix != '[]' runs-on: depot-ubuntu-24.04 - container: - image: mcr.microsoft.com/playwright:v1.49.0-noble - options: --ipc=host strategy: fail-fast: false matrix: @@ -343,12 +335,11 @@ jobs: working-directory: ./packages/${{ matrix.framework.package }} run: pnpm build + - name: Verify runner Chrome + run: google-chrome --version + - name: Run server-side rendered benchmark run: pnpm --filter @framework-tracker/stats-generator run:server-side-rendered ${{ matrix.framework.package }} - env: - PLAYWRIGHT_BROWSERS_PATH: /ms-playwright - PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1' - RUNNER_LABEL: ubuntu-latest - name: Upload server-side rendered stats uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/preview-stats.yml b/.github/workflows/preview-stats.yml index 049a39c7..f585a76f 100644 --- a/.github/workflows/preview-stats.yml +++ b/.github/workflows/preview-stats.yml @@ -1,5 +1,8 @@ name: Preview Stats +env: + RUNNER_LABEL: 'Depot depot-ubuntu-24.04 GitHub Actions runner (2 CPUs, 8 GB RAM, 100 GB disk, 2 GB disk accelerator)' + on: workflow_dispatch: @@ -113,8 +116,6 @@ jobs: - name: Save CI stats run: pnpm --filter @framework-tracker/stats-generator save:ci-stats $GITHUB_WORKSPACE/artifacts - env: - RUNNER_LABEL: ubuntu-latest - name: Collect stats for docs run: pnpm collect:stats diff --git a/packages/docs/src/content.config.ts b/packages/docs/src/content.config.ts index 49766643..ec060c52 100644 --- a/packages/docs/src/content.config.ts +++ b/packages/docs/src/content.config.ts @@ -53,6 +53,7 @@ const devtimeSchema = z.object({ .optional(), timingMeasuredAt: z.string(), runner: z.string(), + browserVersion: z.string().optional(), frameworkVersion: z.string().optional(), }) @@ -118,6 +119,7 @@ const runtimeSchema = z.object({ .optional(), timingMeasuredAt: z.string().optional(), runner: z.string().optional(), + browserVersion: z.string().optional(), frameworkVersion: z.string().optional(), }) diff --git a/packages/docs/src/content/docs/methodology.md b/packages/docs/src/content/docs/methodology.md index 27877391..36724604 100644 --- a/packages/docs/src/content/docs/methodology.md +++ b/packages/docs/src/content/docs/methodology.md @@ -13,6 +13,14 @@ starting a framework and the runtime cost of serving and hydrating a comparable app. Timing results are run multiple times and averaged, and generated JSON is published into the docs package. +Benchmarks run on Depot GitHub Actions runners using +[`depot-ubuntu-24.04`](https://depot.dev/docs/github-actions/runner-types), +which Depot documents as an Intel runner with 2 CPUs, 8 GB RAM, 100 GB disk, +and a 2 GB disk accelerator. Browser rendering benchmarks run directly on the +Depot runner host and use the host Chrome installation rather than a job-level +browser container. The generated runtime stats record the Chrome version used +for browser rendering benchmarks. + ## Dev Time Dev Time measurements use the repository's `starter-*` packages. These projects diff --git a/packages/stats-generator/src/clientSideRendered/run-benchmark.ts b/packages/stats-generator/src/clientSideRendered/run-benchmark.ts index bb2faeeb..d5f3f68f 100644 --- a/packages/stats-generator/src/clientSideRendered/run-benchmark.ts +++ b/packages/stats-generator/src/clientSideRendered/run-benchmark.ts @@ -1,4 +1,5 @@ import { existsSync, readdirSync } from 'node:fs' +import { execFileSync } from 'node:child_process' import puppeteer from 'puppeteer-core' import { startFlow } from 'lighthouse' import type { @@ -36,6 +37,16 @@ function findChromium(): string { ) } +function getBrowserVersion(chromiumPath: string): string | undefined { + try { + return execFileSync(chromiumPath, ['--version'], { + encoding: 'utf-8', + }).trim() + } catch { + return undefined + } +} + async function runOnce( url: string, chromiumPath: string, @@ -107,6 +118,7 @@ export async function runBenchmark( runs: number, ): Promise { const chromiumPath = findChromium() + const browserVersion = getBrowserVersion(chromiumPath) const results: ClientSideRenderedRunResult[] = [] for (let i = 0; i < runs; i++) { @@ -124,6 +136,7 @@ export async function runBenchmark( name: packageName, displayName, package: packageName, + browserVersion, clientSideRenderedTests: { firstPaintMs: avg(fp), fcpMs: avg(fcp), diff --git a/packages/stats-generator/src/clientSideRendered/types.ts b/packages/stats-generator/src/clientSideRendered/types.ts index 1b90bec9..be21c54c 100644 --- a/packages/stats-generator/src/clientSideRendered/types.ts +++ b/packages/stats-generator/src/clientSideRendered/types.ts @@ -8,6 +8,7 @@ export interface ClientSideRenderedBenchmarkResult { name: string displayName: string package: string + browserVersion?: string clientSideRenderedTests: { firstPaintMs: number fcpMs: number @@ -19,6 +20,7 @@ export interface ClientSideRenderedBenchmarkResult { export interface ClientSideRenderedStats { timingMeasuredAt: string runner: string + browserVersion?: string frameworkVersion?: string clientSideRenderedTests: { firstPaintMs: number diff --git a/packages/stats-generator/src/run-client-side-rendered-benchmark.ts b/packages/stats-generator/src/run-client-side-rendered-benchmark.ts index 4e45ad30..dd4d4f41 100644 --- a/packages/stats-generator/src/run-client-side-rendered-benchmark.ts +++ b/packages/stats-generator/src/run-client-side-rendered-benchmark.ts @@ -40,6 +40,7 @@ async function main() { ...existingStats, timingMeasuredAt: timestamp, runner, + browserVersion: result.browserVersion, frameworkVersion: frameworkVersion ?? existingStats?.frameworkVersion, clientSideRenderedTests: result.clientSideRenderedTests, } diff --git a/packages/stats-generator/src/run-server-side-rendered-benchmark.ts b/packages/stats-generator/src/run-server-side-rendered-benchmark.ts index d4bfc1af..2c49b1ce 100644 --- a/packages/stats-generator/src/run-server-side-rendered-benchmark.ts +++ b/packages/stats-generator/src/run-server-side-rendered-benchmark.ts @@ -40,6 +40,7 @@ async function main() { ...existingStats, timingMeasuredAt: timestamp, runner, + browserVersion: result.browserVersion, frameworkVersion: frameworkVersion ?? existingStats?.frameworkVersion, serverSideRenderedTests: result.serverSideRenderedTests, } diff --git a/packages/stats-generator/src/save-ci-stats.ts b/packages/stats-generator/src/save-ci-stats.ts index c423516a..2597d04d 100644 --- a/packages/stats-generator/src/save-ci-stats.ts +++ b/packages/stats-generator/src/save-ci-stats.ts @@ -274,6 +274,8 @@ async function main() { console.info(` ✓ Found client-side rendered stats artifact`) stats = { ...stats, + browserVersion: + clientSideRenderedStats.browserVersion ?? stats.browserVersion, clientSideRenderedTests: clientSideRenderedStats.clientSideRenderedTests, } @@ -300,6 +302,8 @@ async function main() { console.info(` ✓ Found server-side rendered stats artifact`) stats = { ...stats, + browserVersion: + serverSideRenderedStats.browserVersion ?? stats.browserVersion, frameworkVersion: serverSideRenderedStats.frameworkVersion, serverSideRenderedTests: serverSideRenderedStats.serverSideRenderedTests, diff --git a/packages/stats-generator/src/schemas.ts b/packages/stats-generator/src/schemas.ts index 6bc1e68e..37eee142 100644 --- a/packages/stats-generator/src/schemas.ts +++ b/packages/stats-generator/src/schemas.ts @@ -43,6 +43,7 @@ export const SSRRequestThroughputStatsSchema = z.object({ frameworkVersion: z.string().optional(), timingMeasuredAt: z.string().optional(), runner: z.string().optional(), + browserVersion: z.string().optional(), }) export const SSRLoadStatsSchema = z.object({ @@ -79,6 +80,7 @@ export const SSRLoadStatsSchema = z.object({ frameworkVersion: z.string().optional(), timingMeasuredAt: z.string().optional(), runner: z.string().optional(), + browserVersion: z.string().optional(), }) export type InstallStats = z.infer diff --git a/packages/stats-generator/src/serverSideRendered/run-benchmark.ts b/packages/stats-generator/src/serverSideRendered/run-benchmark.ts index 9622bc84..d51e2dd0 100644 --- a/packages/stats-generator/src/serverSideRendered/run-benchmark.ts +++ b/packages/stats-generator/src/serverSideRendered/run-benchmark.ts @@ -1,4 +1,5 @@ import { existsSync, readdirSync } from 'node:fs' +import { execFileSync } from 'node:child_process' import puppeteer from 'puppeteer-core' import { startFlow } from 'lighthouse' import type { @@ -36,6 +37,16 @@ function findChromium(): string { ) } +function getBrowserVersion(chromiumPath: string): string | undefined { + try { + return execFileSync(chromiumPath, ['--version'], { + encoding: 'utf-8', + }).trim() + } catch { + return undefined + } +} + async function runOnce( url: string, chromiumPath: string, @@ -109,6 +120,7 @@ export async function runBenchmark( runs: number, ): Promise { const chromiumPath = findChromium() + const browserVersion = getBrowserVersion(chromiumPath) const results: ServerSideRenderedRunResult[] = [] for (let i = 0; i < runs; i++) { @@ -126,6 +138,7 @@ export async function runBenchmark( name: packageName, displayName, package: packageName, + browserVersion, serverSideRenderedTests: { firstPaintMs: avg(fp), fcpMs: avg(fcp), diff --git a/packages/stats-generator/src/serverSideRendered/types.ts b/packages/stats-generator/src/serverSideRendered/types.ts index 303b98c5..55bafb91 100644 --- a/packages/stats-generator/src/serverSideRendered/types.ts +++ b/packages/stats-generator/src/serverSideRendered/types.ts @@ -8,6 +8,7 @@ export interface ServerSideRenderedBenchmarkResult { name: string displayName: string package: string + browserVersion?: string serverSideRenderedTests: { firstPaintMs: number fcpMs: number @@ -19,6 +20,7 @@ export interface ServerSideRenderedBenchmarkResult { export interface ServerSideRenderedStats { timingMeasuredAt: string runner: string + browserVersion?: string frameworkVersion?: string serverSideRenderedTests: { firstPaintMs: number diff --git a/packages/stats-generator/src/types.ts b/packages/stats-generator/src/types.ts index b56b8010..8f870472 100644 --- a/packages/stats-generator/src/types.ts +++ b/packages/stats-generator/src/types.ts @@ -33,6 +33,7 @@ export interface FrameworkConfig { export interface CIStats { timingMeasuredAt?: string runner?: string + browserVersion?: string frameworkVersion?: string // Install stats installTime?: TimeStat