refactor: use vitest for benchmarks (withastro#12605)

* refactor: use `vitest` for benchmarks

* increase timeout

* increase timeout

* Restore correct label

Author: ematipico

.github/workflows/continuous_benchmark.yml

@@ -50,6 +50,7 @@ jobs:
         uses: CodSpeedHQ/action@513a19673a831f139e8717bf45ead67e47f00044 # v3.2.0
         timeout-minutes: 30
         with:
-          run: pnpm benchmark codspeed
+          working-directory: ./benchmark
+          run: pnpm bench
           token: ${{ secrets.CODSPEED_TOKEN }}
 

benchmark/bench/_util.js

@@ -19,3 +19,14 @@ export function calculateStat(numbers) {
 	const max = Math.max(...numbers);
 	return { avg, stdev, max };
 }
+
+export async function makeProject(name) {
+	console.log('Making project:', name);
+	const projectDir = new URL(`../projects/${name}/`, import.meta.url);
+
+	const makeProjectMod = await import(`../make-project/${name}.js`);
+	await makeProjectMod.run(projectDir);
+
+	console.log('Finished making project:', name);
+	return projectDir;
+}

benchmark/bench/codspeed.bench.js

@@ -0,0 +1,48 @@
+import { fileURLToPath } from 'node:url';
+import { exec } from 'tinyexec';
+import { beforeAll, bench, describe } from 'vitest';
+import { astroBin, makeProject } from './_util.js';
+let streamingApp;
+let nonStreamingApp;
+beforeAll(async () => {
+	const render = await makeProject('render-bench');
+	const root = fileURLToPath(render);
+	await exec(astroBin, ['build'], {
+		nodeOptions: {
+			cwd: root,
+			stdio: 'inherit',
+		},
+	});
+	const entry = new URL('./dist/server/entry.mjs', `file://${root}`);
+	const { manifest, createApp } = await import(entry);
+	streamingApp = createApp(manifest, true);
+	nonStreamingApp = createApp(manifest, false);
+}, 900000);
+
+describe('Bench rendering', () => {
+	bench('Rendering: streaming [true], .astro file', async () => {
+		const request = new Request(new URL('http://exmpale.com/astro'));
+		await streamingApp.render(request);
+	});
+	bench('Rendering: streaming [true], .md file', async () => {
+		const request = new Request(new URL('http://exmpale.com/md'));
+		await streamingApp.render(request);
+	});
+	bench('Rendering: streaming [true], .mdx file', async () => {
+		const request = new Request(new URL('http://exmpale.com/mdx'));
+		await streamingApp.render(request);
+	});
+
+	bench('Rendering: streaming [false], .astro file', async () => {
+		const request = new Request(new URL('http://exmpale.com/astro'));
+		await nonStreamingApp.render(request);
+	});
+	bench('Rendering: streaming [false], .md file', async () => {
+		const request = new Request(new URL('http://exmpale.com/md'));
+		await nonStreamingApp.render(request);
+	});
+	bench('Rendering: streaming [false], .mdx file', async () => {
+		const request = new Request(new URL('http://exmpale.com/mdx'));
+		await nonStreamingApp.render(request);
+	});
+});

benchmark/bench/codspeed.js

@@ -1,7 +1,8 @@
 import mri from 'mri';
 import fs from 'node:fs/promises';
 import path from 'node:path';
-import {fileURLToPath, pathToFileURL} from 'node:url';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { makeProject } from './bench/_util.js';
 
 const args = mri(process.argv.slice(2));
 
@@ -14,7 +15,6 @@ Command
   memory          Run build memory and speed test
   render          Run rendering speed test
   server-stress   Run server stress test
-  codspeed        Run codspeed test
   cli-startup     Run CLI startup speed test
 
 Options
@@ -30,7 +30,6 @@ const benchmarks = {
 	render: () => import('./bench/render.js'),
 	'server-stress': () => import('./bench/server-stress.js'),
 	'cli-startup': () => import('./bench/cli-startup.js'),
-	codspeed: () => import('./bench/codspeed.js')
 };
 
 if (commandName && !(commandName in benchmarks)) {
@@ -39,26 +38,12 @@ if (commandName && !(commandName in benchmarks)) {
 }
 
 if (commandName) {
-		if (commandName === 'codspeed') {
-			const render = await makeProject('render-bench');
-			const rootRender = fileURLToPath(render);
-			const bench = benchmarks[commandName];
-			const benchMod = await bench();
-			const payload = {
-				render: {
-					root: rootRender,
-					output: await getOutputFile('render')
-				},
-			};
-			await benchMod.run(payload);
-		} else {
-			// Run single benchmark
-			const bench = benchmarks[commandName];
-			const benchMod = await bench();
-			const projectDir = await makeProject(args.project || benchMod.defaultProject);
-			const outputFile = await getOutputFile(commandName);
-			await benchMod.run(projectDir, outputFile);
-		}
+	// Run single benchmark
+	const bench = benchmarks[commandName];
+	const benchMod = await bench();
+	const projectDir = await makeProject(args.project || benchMod.defaultProject);
+	const outputFile = await getOutputFile(commandName);
+	await benchMod.run(projectDir, outputFile);
 } else {
 	// Run all benchmarks
 	for (const name in benchmarks) {
@@ -70,21 +55,10 @@ if (commandName) {
 	}
 }
 
-export async function makeProject(name) {
-	console.log('Making project:', name);
-	const projectDir = new URL(`./projects/${name}/`, import.meta.url);
-
-	const makeProjectMod = await import(`./make-project/${name}.js`);
-	await makeProjectMod.run(projectDir);
-
-	console.log('Finished making project:', name);
-	return projectDir;
-}
-
 /**
  * @param {string} benchmarkName
  */
-async function getOutputFile(benchmarkName) {
+export async function getOutputFile(benchmarkName) {
 	let file;
 	if (args.output) {
 		file = pathToFileURL(path.resolve(args.output));

benchmark/index.js

@@ -8,11 +8,13 @@ export async function run(projectDir) {
 	await fs.rm(projectDir, { recursive: true, force: true });
 	await fs.mkdir(new URL('./src/pages/blog', projectDir), { recursive: true });
 	await fs.mkdir(new URL('./src/content/blog', projectDir), { recursive: true });
-	await fs.copyFile(new URL('./image.jpg', import.meta.url), new URL('./src/image.jpg', projectDir));
+	await fs.copyFile(
+		new URL('./image.jpg', import.meta.url),
+		new URL('./src/image.jpg', projectDir),
+	);
 
 	const promises = [];
 
-
 	for (let i = 0; i < 10000; i++) {
 		const content = `\
 # Article ${i}
@@ -24,11 +26,10 @@ ${loremIpsumMd}
 
 `;
 		promises.push(
-			fs.writeFile(new URL(`./src/content/blog/article-${i}.md`, projectDir), content, 'utf-8')
+			fs.writeFile(new URL(`./src/content/blog/article-${i}.md`, projectDir), content, 'utf-8'),
 		);
 	}
 
-
 	await fs.writeFile(
 		new URL(`./src/pages/blog/[...slug].astro`, projectDir),
 		`\
@@ -46,7 +47,7 @@ const { Content } = await entry.render();
 <h1>{entry.data.title}</h1>
 <Content />
 `,
-		'utf-8'
+		'utf-8',
 	);
 
 	await Promise.all(promises);
@@ -58,6 +59,6 @@ import { defineConfig } from 'astro/config';
 
 export default defineConfig({
 });`,
-		'utf-8'
+		'utf-8',
 	);
 }

benchmark/make-project/markdown-cc1.js

@@ -23,7 +23,7 @@ ${loremIpsumMd}
 
 `;
 		promises.push(
-			fs.writeFile(new URL(`./data/blog/article-${i}.md`, projectDir), content, 'utf-8')
+			fs.writeFile(new URL(`./data/blog/article-${i}.md`, projectDir), content, 'utf-8'),
 		);
 	}
 
@@ -39,7 +39,7 @@ ${loremIpsumMd}
 		
 		export const collections = { blog }
 
-		`
+		`,
 	);
 
 	await fs.writeFile(
@@ -60,7 +60,7 @@ const { Content } = await render(entry);
 <h1>{entry.data.title}</h1>
 <Content />
 `,
-		'utf-8'
+		'utf-8',
 	);
 
 	await Promise.all(promises);
@@ -71,6 +71,6 @@ const { Content } = await render(entry);
 import { defineConfig } from 'astro/config';
 
 export default defineConfig({});`,
-		'utf-8'
+		'utf-8',
 	);
 }

benchmark/make-project/markdown-cc2.js

@@ -8,11 +8,13 @@ export async function run(projectDir) {
 	await fs.rm(projectDir, { recursive: true, force: true });
 	await fs.mkdir(new URL('./src/pages/blog', projectDir), { recursive: true });
 	await fs.mkdir(new URL('./src/content/blog', projectDir), { recursive: true });
-	await fs.copyFile(new URL('./image.jpg', import.meta.url), new URL('./src/image.jpg', projectDir));
+	await fs.copyFile(
+		new URL('./image.jpg', import.meta.url),
+		new URL('./src/image.jpg', projectDir),
+	);
 
 	const promises = [];
 
-
 	for (let i = 0; i < 10000; i++) {
 		const content = `\
 # Article ${i}
@@ -24,11 +26,10 @@ ${loremIpsumMd}
 
 `;
 		promises.push(
-			fs.writeFile(new URL(`./src/content/blog/article-${i}.mdx`, projectDir), content, 'utf-8')
+			fs.writeFile(new URL(`./src/content/blog/article-${i}.mdx`, projectDir), content, 'utf-8'),
 		);
 	}
 
-
 	await fs.writeFile(
 		new URL(`./src/pages/blog/[...slug].astro`, projectDir),
 		`\
@@ -46,7 +47,7 @@ const { Content } = await entry.render();
 <h1>{entry.data.title}</h1>
 <Content />
 `,
-		'utf-8'
+		'utf-8',
 	);
 
 	await Promise.all(promises);
@@ -61,6 +62,6 @@ import mdx from '@astrojs/mdx';
 export default defineConfig({
   integrations: [mdx()],
 });`,
-		'utf-8'
+		'utf-8',
 	);
 }

benchmark/make-project/mdx-cc1.js

@@ -23,7 +23,7 @@ ${loremIpsumMd}
 
 `;
 		promises.push(
-			fs.writeFile(new URL(`./data/blog/article-${i}.mdx`, projectDir), content, 'utf-8')
+			fs.writeFile(new URL(`./data/blog/article-${i}.mdx`, projectDir), content, 'utf-8'),
 		);
 	}
 
@@ -39,7 +39,7 @@ ${loremIpsumMd}
 		
 		export const collections = { blog }
 
-		`
+		`,
 	);
 
 	await fs.writeFile(
@@ -60,7 +60,7 @@ const { Content } = await render(entry);
 <h1>{entry.data.title}</h1>
 <Content />
 `,
-		'utf-8'
+		'utf-8',
 	);
 
 	await Promise.all(promises);
@@ -75,6 +75,6 @@ import mdx from '@astrojs/mdx';
 export default defineConfig({
   integrations: [mdx()],
 });`,
-		'utf-8'
+		'utf-8',
 	);
 }