From 6928bf7dda6de8974d178a1cc7d23cd351d57653 Mon Sep 17 00:00:00 2001 From: Huakun Shen Date: Thu, 25 Jun 2026 01:15:41 +0800 Subject: [PATCH] fix: support Drizzle folder migrations --- .journal/2026-06-25.md | 114 ++++++++++++++++++++++++++++ README.md | 14 ++-- guest-js/migrate.ts | 51 +++++++++++-- package.json | 7 +- pnpm-lock.yaml | 42 +++++++++++ test/migrate.test.ts | 168 +++++++++++++++++++++++++++++++++++++++++ test/tsconfig.json | 7 ++ tsconfig.json | 1 + 8 files changed, 391 insertions(+), 13 deletions(-) create mode 100644 .journal/2026-06-25.md create mode 100644 test/migrate.test.ts create mode 100644 test/tsconfig.json diff --git a/.journal/2026-06-25.md b/.journal/2026-06-25.md new file mode 100644 index 0000000..27c334c --- /dev/null +++ b/.journal/2026-06-25.md @@ -0,0 +1,114 @@ +# Journal - 2026-06-25 + +--- + +## 2026-06-25 01:11:00 - Drizzle Folder V3 Migration Hash Fix + +### Core Decision/Topic + +Fixed `migrate()` so Drizzle v1/RC Folder V3 migrations no longer collide in `__drizzle_migrations.hash` when every SQL file is named `migration.sql`. + +### Problem + +Drizzle v1/RC changed migration output from flat SQL files: + +```text +drizzle/0000_init.sql +drizzle/0001_add_column.sql +``` + +to folder-based migrations: + +```text +drizzle/20260605110000_add_users/migration.sql +``` + +The old parser used only the basename of the path as the migration hash. With Folder V3, every migration basename is `migration.sql`, so the second migration fails with: + +```text +UNIQUE constraint failed: __drizzle_migrations.hash +``` + +Worse, the first failed run can leave `hash = 'migration.sql'` in the ledger. On the next app launch, the old code may treat later migrations as already applied and skip real schema changes. + +### Options Considered + +1. **Use full paths as hashes** + - Pros: Always unique. + - Cons: Can vary by import style (`./`, `/src`, alias paths, Windows separators), making hashes less stable across bundler changes. + +2. **Use Folder V3 parent directory name** (CHOSEN) + - Pros: Matches Drizzle's stable migration identity; keeps old flat filenames unchanged. + - Cons: Needs an explicit parser for folder migrations. + +3. **Only document a workaround** + - Pros: No code change. + - Cons: Leaves users with a sharp migration bug and polluted ledgers. + +### Final Decision & Rationale + +Use the parent folder name plus `.sql` as the stable hash for Folder V3 migrations: + +```text +drizzle/20260605110000_add_users/migration.sql +-> 20260605110000_add_users.sql +``` + +Flat migrations keep their existing hash: + +```text +drizzle/0000_init.sql +-> 0000_init.sql +``` + +This preserves backward compatibility while making Drizzle v1/RC folder migrations unique and deterministic. + +### Key Changes Made + +- `guest-js/migrate.ts` + - Added `parseMigrationPath()` to normalize `/` and `\` separators. + - Detects Folder V3 paths whose basename is `migration.sql`. + - Adds internal `isFolderV3` metadata so legacy recovery only applies to Folder V3 migrations. + - If the ledger already contains old `migration.sql`, treats the first Folder V3 migration as already applied and continues with later migrations. + +- `test/migrate.test.ts` + - Added Bun TypeScript regression tests for: + - Folder V3 migrations getting unique parent-folder hashes. + - Flat migration filenames staying unchanged. + - Legacy `migration.sql` recovery applying later Folder V3 migrations. + +- `package.json`, `pnpm-lock.yaml`, `test/tsconfig.json`, `tsconfig.json` + - Added `@types/bun` for `bun:test` type declarations. + - Added `typecheck:test`. + - Kept Bun types scoped to `test/tsconfig.json`. + - Added root `"types": []` so Bun/Node globals do not leak into browser-facing `guest-js`. + - Changed `pretest` to `rollup -c` instead of nested `pnpm build`. + +- `README.md` + - Updated migration examples to use `./drizzle/**/*.sql`. + - Documented support for both flat migrations and Drizzle v1/RC Folder V3. + - Documented stable migration ids and legacy `migration.sql` recognition. + +### Testing & Verification + +Ran and passed: + +```bash +./node_modules/.bin/tsc --noEmit +./node_modules/.bin/tsc --noEmit -p test/tsconfig.json +./node_modules/.bin/rollup -c +bun test test/migrate.test.ts +npm test +pnpm test +``` + +The original red test reproduced `UNIQUE constraint failed: __drizzle_migrations.hash` before the parser change. + +### Future Considerations + +1. **Legacy recovery assumption**: `migration.sql` is treated as the first Folder V3 migration having already run. This is pragmatic, but databases with unusual partial failure states may still need manual repair. +2. **No ledger rewrite**: The fix does not rewrite existing `migration.sql` rows; it only recognizes them at runtime. +3. **Checksum validation**: The migration runner still tracks only stable ids, not SQL content checksums. +4. **Published artifacts**: `dist-js/` is generated but ignored by git in this repo, so release/publish must still run the build step. + +--- diff --git a/README.md b/README.md index 2f31413..8924a8a 100644 --- a/README.md +++ b/README.md @@ -245,7 +245,7 @@ export default defineConfig({ ```bash npx drizzle-kit generate -# creates drizzle/0000_init.sql, drizzle/0001_add_column.sql, etc. +# creates either flat SQL files or Drizzle v1/RC migration folders ``` **4. Run migrations on startup**: @@ -253,8 +253,9 @@ npx drizzle-kit generate ```typescript import { Database, migrate } from "tauri-plugin-libsql-api"; -// Vite bundles these SQL files into the app at build time -const migrations = import.meta.glob("./drizzle/*.sql", { +// Vite bundles these SQL files into the app at build time. +// The glob supports both flat files and Drizzle v1/RC folder migrations. +const migrations = import.meta.glob("./drizzle/**/*.sql", { eager: true, query: "?raw", import: "default", @@ -271,9 +272,10 @@ const db = drizzle(createDrizzleProxy("sqlite:myapp.db"), { schema }); ### How `migrate()` works - Creates a `__drizzle_migrations` tracking table if it doesn't exist -- Parses migration filenames by their numeric prefix (`0000_`, `0001_`, etc.) +- Parses flat migration filenames and Drizzle v1/RC folder names by their numeric prefix (`0000_`, `0001_`, `20260605110000_`, etc.) - Applies only pending migrations in order -- Records each applied migration by filename +- Records each applied migration by a stable id: the filename for flat migrations, or the parent folder name plus `.sql` for Drizzle v1/RC folder migrations +- Recognizes the legacy `migration.sql` marker left by older versions and continues with later Drizzle v1/RC folder migrations ### Adding schema changes @@ -396,7 +398,7 @@ await db.close(); ```typescript import { migrate } from "tauri-plugin-libsql-api"; -const migrations = import.meta.glob("./drizzle/*.sql", { +const migrations = import.meta.glob("./drizzle/**/*.sql", { eager: true, query: "?raw", import: "default", diff --git a/guest-js/migrate.ts b/guest-js/migrate.ts index b52a469..508b9e1 100644 --- a/guest-js/migrate.ts +++ b/guest-js/migrate.ts @@ -27,19 +27,53 @@ interface ParsedMigration { filename: string sql: string index: number + isFolderV3: boolean +} + +function parseMigrationPath(path: string): Omit | null { + const segments = path.split(/[\\/]+/).filter(Boolean) + const fileName = segments[segments.length - 1] + + if (!fileName) { + return null + } + + if (fileName === 'migration.sql' && segments.length >= 2) { + const folderName = segments[segments.length - 2] + const match = folderName.match(/^(\d+)[_\-].*$/) + + if (match) { + return { + filename: `${folderName}.sql`, + index: parseInt(match[1], 10), + isFolderV3: true, + } + } + } + + const match = fileName.match(/^(\d+)[_\-].*\.sql$/) + if (!match) { + return null + } + + return { + filename: fileName, + index: parseInt(match[1], 10), + isFolderV3: false, + } } function parseMigrations(files: MigrationFiles): ParsedMigration[] { const migrations: ParsedMigration[] = [] for (const [path, sql] of Object.entries(files)) { - // Match drizzle-kit naming: 0000_xxx.sql, 0001_xxx.sql, etc. - const match = path.match(/(\d+)[_\-].*\.sql$/) - if (match && sql) { + const parsedPath = parseMigrationPath(path) + if (parsedPath && sql) { migrations.push({ - filename: path.split('/').pop()!, + filename: parsedPath.filename, sql: sql as string, - index: parseInt(match[1], 10), + index: parsedPath.index, + isFolderV3: parsedPath.isFolderV3, }) } } @@ -104,12 +138,19 @@ export async function migrate( // Parse and sort migration files const migrations = parseMigrations(migrationFiles) + const legacyFolderV3Migration = appliedSet.has('migration.sql') + ? migrations.find((migration) => migration.isFolderV3) + : undefined for (const migration of migrations) { if (appliedSet.has(migration.filename)) { continue } + if (migration.filename === legacyFolderV3Migration?.filename) { + continue + } + // Split on semicolons to get individual statements. // Note: this is a naive split — semicolons inside string literals will // cause incorrect splits. drizzle-kit generated SQL does not produce this. diff --git a/package.json b/package.json index 9df3bd0..8d48d6b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "tauri-plugin-libsql-api", - "version": "0.1.0", + "version": "0.1.1-beta.0", "author": "HuakunShen", "repository": { "type": "git", @@ -25,13 +25,16 @@ "scripts": { "build": "rollup -c", "prepublishOnly": "pnpm build", - "pretest": "pnpm build" + "pretest": "rollup -c", + "test": "bun test test/*.test.ts", + "typecheck:test": "tsc --noEmit -p test/tsconfig.json" }, "dependencies": { "@tauri-apps/api": "^2.0.0" }, "devDependencies": { "@rollup/plugin-typescript": "^12.0.0", + "@types/bun": "^1.3.14", "rollup": "^4.9.6", "tslib": "^2.6.2", "typescript": "^5.3.3" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 24c7dbd..771f0ee 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,6 +15,9 @@ importers: '@rollup/plugin-typescript': specifier: ^12.0.0 version: 12.3.0(rollup@4.59.0)(tslib@2.8.1)(typescript@5.9.3) + '@types/bun': + specifier: ^1.3.14 + version: 1.3.14 rollup: specifier: ^4.9.6 version: 4.59.0 @@ -83,66 +86,79 @@ packages: resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.59.0': resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.59.0': resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.59.0': resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.59.0': resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.59.0': resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==} cpu: [loong64] os: [linux] + libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.59.0': resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.59.0': resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==} cpu: [ppc64] os: [linux] + libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.59.0': resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.59.0': resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.59.0': resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.59.0': resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.59.0': resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-openbsd-x64@4.59.0': resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==} @@ -177,9 +193,18 @@ packages: '@tauri-apps/api@2.10.1': resolution: {integrity: sha512-hKL/jWf293UDSUN09rR69hrToyIXBb8CjGaWC7gfinvnQrBVvnLr08FeFi38gxtugAVyVcTa5/FD/Xnkb1siBw==} + '@types/bun@1.3.14': + resolution: {integrity: sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw==} + '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} + '@types/node@26.0.0': + resolution: {integrity: sha512-vf2YFi1iY9lHGwNJMs01biZFbKJkrZR1T6/MlzjhJLPdntOHLhTrDSnSVcdtvjihi4VQNlrFRIxLsDBlQpAipA==} + + bun-types@1.3.14: + resolution: {integrity: sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ==} + estree-walker@2.0.2: resolution: {integrity: sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==} @@ -228,6 +253,9 @@ packages: engines: {node: '>=14.17'} hasBin: true + undici-types@8.3.0: + resolution: {integrity: sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==} + snapshots: '@rollup/plugin-typescript@12.3.0(rollup@4.59.0)(tslib@2.8.1)(typescript@5.9.3)': @@ -324,8 +352,20 @@ snapshots: '@tauri-apps/api@2.10.1': {} + '@types/bun@1.3.14': + dependencies: + bun-types: 1.3.14 + '@types/estree@1.0.8': {} + '@types/node@26.0.0': + dependencies: + undici-types: 8.3.0 + + bun-types@1.3.14: + dependencies: + '@types/node': 26.0.0 + estree-walker@2.0.2: {} fsevents@2.3.3: @@ -387,3 +427,5 @@ snapshots: tslib@2.8.1: {} typescript@5.9.3: {} + + undici-types@8.3.0: {} diff --git a/test/migrate.test.ts b/test/migrate.test.ts new file mode 100644 index 0000000..d6a961a --- /dev/null +++ b/test/migrate.test.ts @@ -0,0 +1,168 @@ +import { expect, mock, test } from "bun:test"; + +interface HarnessState { + hashes: string[]; + userColumns: string[]; +} + +interface InvokeArgs { + query?: string; + queries?: string[]; +} + +interface Harness { + readonly hashes: string[]; + readonly userColumns: string[]; + invoke(command: string, args: InvokeArgs): Promise; +} + +let activeHarness: Harness | undefined; + +mock.module("@tauri-apps/api/core", () => ({ + invoke(command: string, args: InvokeArgs) { + if (!activeHarness) { + throw new Error("No active migration harness"); + } + + return activeHarness.invoke(command, args); + }, +})); + +type Migrate = ( + dbPath: string, + migrationFiles: Record, +) => Promise; + +let migratePromise: Promise | undefined; + +function getMigrate() { + migratePromise ??= import("../dist-js/index.js").then( + (module) => module.migrate as Migrate, + ); + return migratePromise; +} + +function createHarness({ + hashes = [], + userColumns = [], +}: Partial = {}): Harness { + const state: HarnessState = { + hashes: [...hashes], + userColumns: [...userColumns], + }; + + function applyQuery(next: HarnessState, query: string) { + const trimmed = query.trim(); + + if (/^CREATE TABLE users\b/i.test(trimmed)) { + next.userColumns = ["id"]; + return; + } + + if (/^ALTER TABLE users ADD COLUMN name\b/i.test(trimmed)) { + if (!next.userColumns.includes("name")) { + next.userColumns.push("name"); + } + return; + } + + const insertMatch = trimmed.match( + /^INSERT INTO \S+ \(hash\) VALUES \('((?:''|[^'])*)'\)$/i, + ); + + if (insertMatch) { + const hash = insertMatch[1].replace(/''/g, "'"); + if (next.hashes.includes(hash)) { + throw new Error("UNIQUE constraint failed: __drizzle_migrations.hash"); + } + next.hashes.push(hash); + } + } + + return { + get hashes() { + return state.hashes; + }, + get userColumns() { + return state.userColumns; + }, + async invoke(command, args) { + if (command === "plugin:libsql|execute") { + return { rowsAffected: 0, lastInsertId: 0 }; + } + + if (command === "plugin:libsql|select") { + return state.hashes.map((hash) => ({ hash })); + } + + if (command === "plugin:libsql|batch") { + const next: HarnessState = { + hashes: [...state.hashes], + userColumns: [...state.userColumns], + }; + + for (const query of args.queries ?? []) { + applyQuery(next, query); + } + + state.hashes = next.hashes; + state.userColumns = next.userColumns; + return undefined; + } + + throw new Error(`Unexpected command: ${command}`); + }, + }; +} + +test("applies Drizzle v3 folder migrations with unique parent-folder hashes", async () => { + activeHarness = createHarness(); + const migrate = await getMigrate(); + + await migrate("sqlite:app.db", { + "/src/db/migrations/20260508135710_init/migration.sql": + "CREATE TABLE users (id INTEGER PRIMARY KEY);", + "/src/db/migrations/20260605110000_add_users/migration.sql": + "ALTER TABLE users ADD COLUMN name TEXT;", + }); + + expect(activeHarness.hashes).toEqual([ + "20260508135710_init.sql", + "20260605110000_add_users.sql", + ]); + expect(activeHarness.userColumns).toEqual(["id", "name"]); +}); + +test("keeps flat migration filenames as hashes", async () => { + activeHarness = createHarness(); + const migrate = await getMigrate(); + + await migrate("sqlite:app.db", { + "./drizzle/0000_init.sql": "CREATE TABLE users (id INTEGER PRIMARY KEY);", + "./drizzle/0001_add_name.sql": "ALTER TABLE users ADD COLUMN name TEXT;", + }); + + expect(activeHarness.hashes).toEqual(["0000_init.sql", "0001_add_name.sql"]); + expect(activeHarness.userColumns).toEqual(["id", "name"]); +}); + +test("continues Drizzle v3 migrations after a legacy migration.sql hash", async () => { + activeHarness = createHarness({ + hashes: ["migration.sql"], + userColumns: ["id"], + }); + const migrate = await getMigrate(); + + await migrate("sqlite:app.db", { + "\\src\\db\\migrations\\20260508135710_init\\migration.sql": + "CREATE TABLE users (id INTEGER PRIMARY KEY);", + "\\src\\db\\migrations\\20260605110000_add_users\\migration.sql": + "ALTER TABLE users ADD COLUMN name TEXT;", + }); + + expect(activeHarness.hashes).toEqual([ + "migration.sql", + "20260605110000_add_users.sql", + ]); + expect(activeHarness.userColumns).toEqual(["id", "name"]); +}); diff --git a/test/tsconfig.json b/test/tsconfig.json new file mode 100644 index 0000000..8a9c21d --- /dev/null +++ b/test/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "types": ["bun"] + }, + "include": ["*.ts"] +} diff --git a/tsconfig.json b/tsconfig.json index 0591122..46f7d6c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,6 +3,7 @@ "target": "es2021", "module": "esnext", "moduleResolution": "bundler", + "types": [], "skipLibCheck": true, "strict": true, "noUnusedLocals": true,