[pull] main from remotion-dev:main

.gitignore

@@ -36,3 +36,4 @@ packages/example/build
 packages/example-without-zod/build
 packages/docs/out
 packages/cloudrun/container/ensure-browser.mjs
+packages/skills-evals/.runs

package.json

@@ -85,6 +85,7 @@
 			"!packages/template-audiogram/whisper.cpp",
 			"!packages/template-audiogram/whisper.cpp/**",
 			"!packages/template-vercel/**",
+			"!packages/skills-evals/.runs/**",
 			"!packages/bugs"
 		],
 		"catalog": {

packages/cli/src/list-of-remotion-packages.ts

@@ -65,6 +65,7 @@ export const listOfRemotionPackages = [
 	'@remotion/serverless',
 	'@remotion/serverless-client',
 	'@remotion/skills',
+	'@remotion/skills-evals',
 	'@remotion/studio-server',
 	'@remotion/studio-shared',
 	'@remotion/studio',

packages/create-video/src/list-of-remotion-packages.ts

@@ -65,6 +65,7 @@ export const listOfRemotionPackages = [
 	'@remotion/serverless',
 	'@remotion/serverless-client',
 	'@remotion/skills',
+	'@remotion/skills-evals',
 	'@remotion/studio-server',
 	'@remotion/studio-shared',
 	'@remotion/studio',

packages/skills-evals/README.md

@@ -0,0 +1,5 @@
+# @remotion/skills-evals
+
+## Usage
+
+This is an internal package and has no documentation.

packages/skills-evals/eslint.config.mjs

@@ -0,0 +1,7 @@
+import {remotionFlatConfig} from '@remotion/eslint-config-internal';
+
+const config = remotionFlatConfig({react: false});
+
+export default {
+	...config,
+};

packages/skills-evals/package.json

@@ -0,0 +1,29 @@
+{
+	"repository": {
+		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/skills-evals"
+	},
+	"name": "@remotion/skills-evals",
+	"version": "4.0.461",
+	"private": true,
+	"type": "module",
+	"scripts": {
+		"dev": "bun src/server.tsx",
+		"eval": "bun src/cli.ts",
+		"format": "oxfmt src scenarios.ts",
+		"formatting": "oxfmt src scenarios.ts --check",
+		"lint": "eslint src && tsgo"
+	},
+	"dependencies": {
+		"@remotion/skills": "workspace:*",
+		"react": "catalog:",
+		"react-dom": "catalog:"
+	},
+	"devDependencies": {
+		"@remotion/eslint-config-internal": "workspace:*",
+		"@types/bun": "catalog:",
+		"@types/react": "catalog:",
+		"@types/react-dom": "catalog:",
+		"@typescript/native-preview": "catalog:",
+		"eslint": "catalog:"
+	}
+}

packages/skills-evals/scenarios.ts

@@ -0,0 +1,8 @@
+export type SkillEvalScenario = {
+	id: string;
+	model: string;
+	prompt: string;
+	timeoutMs?: number;
+};
+
+export const scenarios: SkillEvalScenario[] = [];

packages/skills-evals/src/app/comparison-data.ts

@@ -0,0 +1,67 @@
+import {existsSync} from 'node:fs';
+import {readFile} from 'node:fs/promises';
+import {join} from 'node:path';
+import {listFilesRecursively, readJson, sanitizePathPart} from '../files';
+import type {SkillEvalComparison, SkillEvalManifest} from '../manifest';
+import {comparisonsRoot} from './shared';
+
+export type ComparisonWithManifests = {
+	afterManifest: SkillEvalManifest;
+	beforeManifest: SkillEvalManifest;
+	comparison: SkillEvalComparison;
+	skillDiff: string;
+};
+
+export const loadComparisons = async () => {
+	const files = (await listFilesRecursively(comparisonsRoot)).filter((file) =>
+		file.endsWith('/comparison.json'),
+	);
+	const comparisons = await Promise.all(
+		files.map((file) => readJson<SkillEvalComparison>(file)),
+	);
+
+	comparisons.sort((a, b) => b.completedAt.localeCompare(a.completedAt));
+
+	return comparisons;
+};
+
+export const getLatestComparisonByScenario = async () => {
+	const latest = new Map<string, SkillEvalComparison>();
+
+	for (const comparison of await loadComparisons()) {
+		if (!latest.has(comparison.scenarioId)) {
+			latest.set(comparison.scenarioId, comparison);
+		}
+	}
+
+	return latest;
+};
+
+export const loadComparison = async (
+	scenarioId: string,
+	comparisonId: string,
+): Promise<ComparisonWithManifests | null> => {
+	const comparisonPath = join(
+		comparisonsRoot,
+		sanitizePathPart(scenarioId),
+		comparisonId,
+		'comparison.json',
+	);
+
+	if (!existsSync(comparisonPath)) {
+		return null;
+	}
+
+	const comparison = await readJson<SkillEvalComparison>(comparisonPath);
+	const [beforeManifest, afterManifest, skillDiff] = await Promise.all([
+		readJson<SkillEvalManifest>(comparison.before.manifestPath),
+		readJson<SkillEvalManifest>(comparison.after.manifestPath),
+		readFile(comparison.skillDiffPath, 'utf-8'),
+	]);
+
+	return {afterManifest, beforeManifest, comparison, skillDiff};
+};
+
+export const getPreferredArtifact = (manifest: SkillEvalManifest) =>
+	manifest.artifacts.find((artifact) => artifact.type === 'video') ??
+	manifest.artifacts[0];