diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..beeb3831 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# sqlx checksums migration file bytes. Pin LF so a contributor's line-ending +# rewrite does not trip a false checksum mismatch (ADR-0003). +*.sql text eol=lf diff --git a/Cargo.toml b/Cargo.toml index c357d69b..2e0322bc 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,7 @@ inventory = "0.3" moka = { version = "0.12", features = ["future"] } # Database -sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres", "json", "time", "uuid", "bigdecimal"] } +sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres", "json", "time", "uuid", "bigdecimal", "migrate"] } # Crypto & checksums crc32fast = "1" diff --git a/crates/bin/src/cmd_init.rs b/crates/bin/src/cmd_init.rs index 92118bb8..466ed49e 100755 --- a/crates/bin/src/cmd_init.rs +++ b/crates/bin/src/cmd_init.rs @@ -179,6 +179,8 @@ pub async fn run(args: InitArgs) -> anyhow::Result { if initialized { println!("--- Catalog already initialized. Use 'extenddb migrate' for pending migrations."); } else { + // run_catalog_migrations applies the schema and writes catalog_version + // (the version write moved here from the migration SQL; ADR-0003). bootstrapper .run_catalog_migrations() .await diff --git a/crates/bin/src/cmd_migrate.rs b/crates/bin/src/cmd_migrate.rs index 8221a80d..067bf0b5 100755 --- a/crates/bin/src/cmd_migrate.rs +++ b/crates/bin/src/cmd_migrate.rs @@ -3,7 +3,8 @@ //! `extenddb migrate` — apply catalog schema migrations (REQ-CAT-014). //! -//! Reads current catalog version, runs pending migrations, and reports the result. +//! Reads the current catalog version, runs both migrators (which validate the +//! checksums of already-applied migrations), and reports the result. use clap::Args; @@ -52,6 +53,22 @@ pub async fn run(args: MigrateArgs) -> anyhow::Result<()> { .await .map_err(|e| anyhow::anyhow!("{e:?}"))?; + // ADR-0003: a catalog created by the pre-sqlx runner cannot be upgraded in + // place. Refuse with the re-init directive instead of failing later on a + // non-idempotent DDL re-run. + if bootstrap + .catalog_predates_sqlx() + .await + .map_err(|e| anyhow::anyhow!("{e:?}"))? + { + anyhow::bail!( + "This catalog predates the sqlx migration system (ADR-0003). In-place \ + upgrade is not supported. Run 'extenddb destroy' then 'extenddb init' \ + to recreate both databases (this drops all data). See \ + docs/manuals/07-upgrade-manual.md." + ); + } + // Show current catalog version. println!("--- Checking current catalog version..."); let current = bootstrap @@ -79,13 +96,11 @@ pub async fn run(args: MigrateArgs) -> anyhow::Result<()> { println!(" Pending: {}", data_pending.join(", ")); } - if !catalog_pending && data_pending.is_empty() { - println!(); - println!("Everything is up to date (catalog version {expected}). No migrations needed."); - return Ok(()); - } + let has_pending = catalog_pending || !data_pending.is_empty(); - if !args.yes { + // Applying migrations mutates the schema, so require explicit confirmation + // when there is pending work. + if has_pending && !args.yes { let mut what = Vec::new(); if catalog_pending { what.push(format!("catalog {current_display} -> {expected}")); @@ -99,16 +114,15 @@ pub async fn run(args: MigrateArgs) -> anyhow::Result<()> { ); } - if catalog_pending { - bootstrap - .run_catalog_migrations() - .await - .map_err(|e| anyhow::anyhow!("{e:?}"))?; - } - - // Always run data migrations: the data-database ledger is idempotent - // (already-applied migrations are skipped) and independent of the catalog - // version, so an upgrade that only touched the data schema is still applied. + // Run both migrators unconditionally, even when nothing is pending. sqlx + // validates the checksum of every already-applied migration on each run, so + // a migration file edited after it shipped is caught loudly here instead of + // drifting silently. Applying is idempotent; an up-to-date run applies + // nothing. + bootstrap + .run_catalog_migrations() + .await + .map_err(|e| anyhow::anyhow!("{e:?}"))?; bootstrap .run_data_migrations() .await @@ -122,8 +136,12 @@ pub async fn run(args: MigrateArgs) -> anyhow::Result<()> { let new_display = new.as_deref().unwrap_or("none"); println!(); - println!("=== extenddb migrate complete ==="); - println!("Catalog version: {current_display} -> {new_display}"); + if has_pending { + println!("=== extenddb migrate complete ==="); + println!("Catalog version: {current_display} -> {new_display}"); + } else { + println!("Everything is up to date (catalog version {expected}). No migrations applied."); + } Ok(()) } diff --git a/crates/storage-postgres/data_migrations/001_data_schema.sql b/crates/storage-postgres/data_migrations/001_data_schema.sql index 72dce801..ef49ebaf 100755 --- a/crates/storage-postgres/data_migrations/001_data_schema.sql +++ b/crates/storage-postgres/data_migrations/001_data_schema.sql @@ -5,8 +5,6 @@ -- stream records and idempotency tokens can be written atomically with item -- data within a single PostgreSQL transaction (P54 Bug 1). -BEGIN; - -- Stream shards — fixed shards per table, assigned by partition key hash. -- No FK to catalog tables (cross-database FKs are not possible). -- Application-level integrity ensures table_id validity. @@ -37,9 +35,6 @@ CREATE INDEX IF NOT EXISTS idx_stream_records_created ON stream_records (created_at); -- Monotonic sequence for stream record ordering (CB-21). --- Note: the idempotency check in run_data_migrations uses stream_shards --- existence to decide whether to skip this entire migration. If stream_shards --- is created manually without the sequence, stream_seq will not exist. CREATE SEQUENCE IF NOT EXISTS stream_seq START 1; SELECT setval('stream_seq', GREATEST( (EXTRACT(EPOCH FROM now()) * 1000000)::BIGINT, @@ -55,5 +50,3 @@ CREATE TABLE IF NOT EXISTS idempotency_tokens ( CREATE INDEX IF NOT EXISTS idx_idempotency_tokens_created ON idempotency_tokens (created_at); - -COMMIT; diff --git a/crates/storage-postgres/data_migrations/003_idempotency_account_scope.sql b/crates/storage-postgres/data_migrations/003_idempotency_account_scope.sql index d20ed56f..7deaec6e 100644 --- a/crates/storage-postgres/data_migrations/003_idempotency_account_scope.sql +++ b/crates/storage-postgres/data_migrations/003_idempotency_account_scope.sql @@ -15,8 +15,6 @@ -- acceptable for a short-lived cache. Apply data migrations during the -- stop / migrate / restart upgrade sequence to bound that window. -BEGIN; - DROP TABLE IF EXISTS idempotency_tokens; CREATE TABLE idempotency_tokens ( @@ -29,5 +27,3 @@ CREATE TABLE idempotency_tokens ( CREATE INDEX idx_idempotency_tokens_created ON idempotency_tokens (created_at); - -COMMIT; diff --git a/crates/storage-postgres/migrations/001_schema.sql b/crates/storage-postgres/migrations/001_schema.sql index 3a684189..756f672c 100644 --- a/crates/storage-postgres/migrations/001_schema.sql +++ b/crates/storage-postgres/migrations/001_schema.sql @@ -1,9 +1,9 @@ -- Copyright 2026 ExtendDB contributors -- SPDX-License-Identifier: Apache-2.0 --- Consolidated catalog schema for extenddb (catalog version 0.0.2). --- This is the complete schema applied on fresh installs. - -BEGIN; +-- Consolidated catalog schema for extenddb. +-- This is the complete schema applied on fresh installs. The catalog version +-- is written by the migration runner after this file applies (sqlx has no +-- knowledge of our semver), so it is not seeded here. -- Accounts — multi-account support (REQ-AUTH-005). CREATE TABLE IF NOT EXISTS accounts ( @@ -69,12 +69,6 @@ CREATE TABLE IF NOT EXISTS tags ( PRIMARY KEY (resource_arn, tag_key) ); --- Migration tracking. -CREATE TABLE IF NOT EXISTS schema_history ( - filename TEXT PRIMARY KEY, - applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -- Settings (catalog version, data database connection, runtime config). CREATE TABLE IF NOT EXISTS settings ( key TEXT PRIMARY KEY, @@ -310,12 +304,9 @@ SELECT setval('stream_seq', GREATEST( 1 )); --- Seed settings. -INSERT INTO settings (key, value) VALUES ('catalog_version', '0.0.2') -ON CONFLICT (key) DO NOTHING; +-- Seed settings. The catalog version is written by the migration runner after +-- this file applies, not seeded here. INSERT INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0.25') ON CONFLICT (key) DO NOTHING; INSERT INTO settings (key, value) VALUES ('gsi_propagation_delay_ms', '10') ON CONFLICT (key) DO NOTHING; - -COMMIT; diff --git a/crates/storage-postgres/src/bootstrapper.rs b/crates/storage-postgres/src/bootstrapper.rs index 5eee74fb..13a20402 100755 --- a/crates/storage-postgres/src/bootstrapper.rs +++ b/crates/storage-postgres/src/bootstrapper.rs @@ -417,6 +417,23 @@ impl Bootstrapper for PostgresBootstrapper { Ok(row.map(|(v,)| v)) } + async fn catalog_predates_sqlx(&self) -> OpResult { + let pool = self.app_pool(&self.config.catalog_db).await?; + // Uninitialized catalog: nothing to guard (init creates the sqlx ledger). + if !migrations::table_exists(&pool, "settings").await? { + return Ok(false); + } + // The old runner tracked migrations in `schema_history`; sqlx uses + // `_sqlx_migrations`. An initialized catalog carrying the legacy table, + // or lacking the sqlx ledger, predates the sqlx migrator. Checking the + // catalog is sufficient: init creates both databases together (or + // aborts), so a pre-sqlx data database never appears without a pre-sqlx + // catalog, which this refuses first. + let has_legacy = migrations::table_exists(&pool, "schema_history").await?; + let has_sqlx = migrations::table_exists(&pool, "_sqlx_migrations").await?; + Ok(has_legacy || !has_sqlx) + } + fn expected_catalog_version(&self) -> String { CATALOG_VERSION.to_string() } diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index e5e8c78c..bd273b47 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -114,7 +114,7 @@ use sqlx::postgres::PgPoolOptions; /// /// The tuple is the single source of truth. Use `CATALOG_VERSION.to_string()` /// wherever a string representation is needed. -pub const CATALOG_VERSION: CatalogVersion = CatalogVersion::new(0, 0, 2); +pub const CATALOG_VERSION: CatalogVersion = CatalogVersion::new(0, 1, 0); /// Minimum number of connections allowed per pool. /// diff --git a/crates/storage-postgres/src/migrations.rs b/crates/storage-postgres/src/migrations.rs index fd390c32..4f6b80c2 100755 --- a/crates/storage-postgres/src/migrations.rs +++ b/crates/storage-postgres/src/migrations.rs @@ -1,100 +1,102 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! `PostgreSQL` schema migration helpers for catalog and data databases. +//! PostgreSQL schema migrations for the catalog and data databases. +//! +//! Both databases use sqlx's built-in migrator (`sqlx::migrate!`), which embeds +//! the SQL files at compile time, applies them in order, and records each one +//! in a `_sqlx_migrations` table together with a checksum of the file bytes. +//! Editing a migration after it has been applied is a hard error rather than a +//! silent no-op (ADR-0003). +//! +//! Migrations run only during `init` and `migrate`, never while serving. use extenddb_storage::management_store::{OpError, OpResult}; use sqlx::PgPool; +use sqlx::migrate::Migrator; -/// Embedded catalog migration files, applied in order. -pub(crate) const CATALOG_MIGRATIONS: &[(&str, &str)] = &[( - "001_schema.sql", - include_str!("../../storage-postgres/migrations/001_schema.sql"), -)]; +use crate::CATALOG_VERSION; -/// Run catalog migrations, skipping already-applied ones. +/// Catalog database migrator (files under `crates/storage-postgres/migrations`). +pub(crate) static CATALOG_MIGRATOR: Migrator = sqlx::migrate!("./migrations"); + +/// Data database migrator (files under `crates/storage-postgres/data_migrations`). +pub(crate) static DATA_MIGRATOR: Migrator = sqlx::migrate!("./data_migrations"); + +/// Apply catalog migrations, then record the catalog version. +/// +/// sqlx runs each pending migration in its own transaction and skips those +/// already recorded, so this is idempotent. The version write is a separate +/// step after the migrations commit: sqlx knows nothing about our semver. It is +/// intentionally not atomic with the migration (ADR-0003); re-running `migrate` +/// repairs a version left stale by a crash between the two. pub(crate) async fn run_catalog_migrations(pool: &PgPool) -> OpResult<()> { println!("--- Running catalog migrations..."); - for (filename, sql) in CATALOG_MIGRATIONS { - if is_migration_applied(pool, filename).await? { - println!(" {filename} — already applied, skipping."); - continue; - } - println!(" Applying {filename}..."); - sqlx::raw_sql(sql) - .execute(pool) - .await - .map_err(|e| OpError::Internal(format!("Migration {filename} failed: {e}")))?; - record_migration(pool, filename).await?; - } - println!(" Migrations applied."); + CATALOG_MIGRATOR + .run(pool) + .await + .map_err(|e| OpError::Internal(format!("Catalog migration failed: {e}")))?; + write_catalog_version(pool).await?; + println!(" Catalog schema at version {CATALOG_VERSION}."); Ok(()) } -/// Embedded data-database migration files, applied in order. Tracked in the -/// data database's own `schema_history` table (a separate database from the -/// catalog), so `extenddb migrate` applies exactly the pending migrations. -pub(crate) const DATA_MIGRATIONS: &[(&str, &str)] = &[ - ( - "001_data_schema.sql", - include_str!("../../storage-postgres/data_migrations/001_data_schema.sql"), - ), - ( - "002_gsi_pending.sql", - include_str!("../../storage-postgres/data_migrations/002_gsi_pending.sql"), - ), - ( - "003_idempotency_account_scope.sql", - include_str!("../../storage-postgres/data_migrations/003_idempotency_account_scope.sql"), - ), -]; - -/// Run data database migrations, skipping already-applied ones. +/// Apply data-database migrations. /// -/// Mirrors [`run_catalog_migrations`]: each migration is recorded in -/// `schema_history` and skipped on later runs. The data database has its own -/// ledger because it is a separate database from the catalog. +/// The data database is tracked by its own `_sqlx_migrations` table and has no +/// separate version. `migrate`, not just `init`, runs this so existing +/// deployments pick up data-schema changes. pub(crate) async fn run_data_migrations(pool: &PgPool) -> OpResult<()> { println!("--- Running data migrations..."); + DATA_MIGRATOR + .run(pool) + .await + .map_err(|e| OpError::Internal(format!("Data migration failed: {e}")))?; + println!(" Data migrations complete."); + Ok(()) +} - // Ensure the data database has a migration ledger before tracking. (The - // catalog ledger lives in a different database and cannot be reused here.) +/// Write the compiled-in catalog version into the `settings` table. +async fn write_catalog_version(pool: &PgPool) -> OpResult<()> { sqlx::query( - "CREATE TABLE IF NOT EXISTS schema_history (\ - filename TEXT PRIMARY KEY, \ - applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()\ - )", + "INSERT INTO settings (key, value) VALUES ('catalog_version', $1) \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", ) + .bind(CATALOG_VERSION.to_string()) .execute(pool) .await - .map_err(|e| OpError::Internal(format!("Create data schema_history: {e}")))?; - - // Adopt a pre-tracking deployment: if 001 was applied by an earlier version - // (its tables exist) but isn't recorded, record it WITHOUT re-running it. - // Re-running 001 would execute `setval('stream_seq', ...)` again and could - // regress the stream sequence on a live database, producing duplicate - // sequence numbers. - if !is_migration_applied(pool, "001_data_schema.sql").await? - && table_exists(pool, "stream_shards").await? - { - println!(" Adopting existing 001_data_schema.sql (pre-tracking deployment)."); - record_migration(pool, "001_data_schema.sql").await?; - } + .map_err(|e| OpError::Internal(format!("Record catalog version: {e}")))?; + Ok(()) +} - for (filename, sql) in DATA_MIGRATIONS { - if is_migration_applied(pool, filename).await? { - println!(" {filename} — already applied, skipping."); - continue; - } - println!(" Applying {filename}..."); - sqlx::raw_sql(sql) - .execute(pool) +/// Filenames of data migrations not yet applied to this data database. +/// +/// Compares the embedded migrations against the versions recorded in the data +/// database's `_sqlx_migrations` table, without applying anything, so `migrate` +/// can report pending work and gate on it. Independent of the catalog version. +pub(crate) async fn pending_data_migrations(pool: &PgPool) -> OpResult> { + let applied: std::collections::HashSet = if table_exists(pool, "_sqlx_migrations").await? { + sqlx::query_scalar::<_, i64>("SELECT version FROM _sqlx_migrations WHERE success") + .fetch_all(pool) .await - .map_err(|e| OpError::Internal(format!("Data migration {filename} failed: {e}")))?; - record_migration(pool, filename).await?; + .map_err(|e| OpError::Internal(format!("Read _sqlx_migrations: {e}")))? + .into_iter() + .collect() + } else { + std::collections::HashSet::new() + }; + + let mut pending = Vec::new(); + for migration in DATA_MIGRATOR.iter() { + if !applied.contains(&migration.version) { + pending.push(format!( + "{:03}_{}.sql", + migration.version, + migration.description.replace(' ', "_") + )); + } } - println!(" Data migrations applied."); - Ok(()) + Ok(pending) } /// Check if a table exists in the public schema. @@ -110,58 +112,94 @@ pub(crate) async fn table_exists(pool: &PgPool, name: &str) -> OpResult { Ok(exists) } -/// Filenames of [`DATA_MIGRATIONS`] not yet applied to this data database. -/// -/// Mirrors the apply logic in [`run_data_migrations`] without executing -/// anything, so callers (e.g. `extenddb migrate`) can report and gate on -/// pending work. A pre-tracking baseline (`001_data_schema.sql` whose tables -/// already exist but isn't recorded) is treated as already applied: it will be -/// adopted — recorded without re-running — not applied, so it is not reported -/// as pending. -pub(crate) async fn pending_data_migrations(pool: &PgPool) -> OpResult> { - let has_history = table_exists(pool, "schema_history").await?; - // Pre-tracking deployment: 001 ran under an earlier version (its tables - // exist) but was never recorded. It is adopted, not re-run. - let adopts_baseline = !has_history && table_exists(pool, "stream_shards").await?; +#[cfg(test)] +mod tests { + use super::*; - let mut pending = Vec::new(); - for (filename, _sql) in DATA_MIGRATIONS { - if is_migration_applied(pool, filename).await? { - continue; - } - if *filename == "001_data_schema.sql" && adopts_baseline { - continue; - } - pending.push((*filename).to_owned()); + // Checksum lockfile (ADR-0003 CI net): pin the sqlx checksum (SHA-384 of + // the file bytes) of every shipped migration. Editing an already-applied + // migration changes its checksum, so this fails `cargo test` in the PR + // runner: it catches in CI what sqlx otherwise only catches at runtime + // against a live database. `.gitattributes` pins *.sql to LF, so the bytes + // (and these checksums) are stable across platforms. To add a migration, + // append its (version, checksum) below using the value the assertion prints. + // The values are sqlx's SHA-384 migration checksums; a future sqlx bump that + // changed the hash algorithm would require regenerating them. + const CATALOG_CHECKSUMS: &[(i64, &str)] = &[( + 1, + "5fbac4791f1dcdff7fca7682384ead023ce64a377af7f294c9fb2aaed1a5a826b6ffa273f4537fc318ef2480e179ab98", + )]; + const DATA_CHECKSUMS: &[(i64, &str)] = &[ + ( + 1, + "36fb3dc917923ca6f34fda2157999ad132996a937ee9893f91c260f8c09276b237c5279f750ad94af97bd6b1fd966a8f", + ), + ( + 2, + "8da1bcb8c9864258b0c12711b5df5090d0c1caa52a0102466a8ca94084ac56c9db385a650b34fe29a45dc942318a0100", + ), + ( + 3, + "f5a73cbb1bac5e979acb0952973f9d2491a44e80b4eaee175634b35e462bbd3c3090b664fc336ccd473404616bdb81aa", + ), + ]; + + fn assert_checksums(migrator: &Migrator, expected: &[(i64, &str)]) { + let actual: Vec<(i64, String)> = migrator + .iter() + .map(|m| { + let hex = m + .checksum + .iter() + .map(|b| format!("{b:02x}")) + .collect::(); + (m.version, hex) + }) + .collect(); + let expected: Vec<(i64, String)> = expected + .iter() + .map(|(v, h)| (*v, (*h).to_owned())) + .collect(); + assert_eq!( + actual, expected, + "migration checksums changed: an already-shipped migration was edited \ + (forbidden), or one was added/removed. If the change is intentional and \ + the file has not shipped, update the pinned checksum(s) with the actual \ + values shown here." + ); } - Ok(pending) -} -/// Check if a migration has already been applied. -async fn is_migration_applied(pool: &PgPool, filename: &str) -> OpResult { - if table_exists(pool, "schema_history").await? { - let applied: (bool,) = - sqlx::query_as("SELECT EXISTS(SELECT 1 FROM schema_history WHERE filename = $1)") - .bind(filename) - .fetch_one(pool) - .await - .map_err(|e| OpError::Internal(format!("Check migration: {e}")))?; - return Ok(applied.0); + #[test] + fn migration_checksums_are_pinned() { + assert_checksums(&CATALOG_MIGRATOR, CATALOG_CHECKSUMS); + assert_checksums(&DATA_MIGRATOR, DATA_CHECKSUMS); } - Ok(false) -} -/// Record a migration in the `schema_history` table. -async fn record_migration(pool: &PgPool, filename: &str) -> OpResult<()> { - if !table_exists(pool, "schema_history").await? { - return Ok(()); + // Tripwire (ADR-0003): pin the embedded migration counts and the catalog + // version. Adding a catalog migration without bumping CATALOG_VERSION fails + // here, forcing a deliberate version decision. The data database has no + // version, so its count is pinned only to make a new data migration a + // conscious change. + const EXPECTED_CATALOG_MIGRATIONS: usize = 1; + const EXPECTED_DATA_MIGRATIONS: usize = 3; + const EXPECTED_CATALOG_VERSION: &str = "0.1.0"; + + #[test] + fn migration_counts_and_catalog_version_are_pinned() { + assert_eq!( + CATALOG_MIGRATOR.iter().count(), + EXPECTED_CATALOG_MIGRATIONS, + "catalog migration count changed: bump CATALOG_VERSION and update EXPECTED_CATALOG_MIGRATIONS", + ); + assert_eq!( + DATA_MIGRATOR.iter().count(), + EXPECTED_DATA_MIGRATIONS, + "data migration count changed: update EXPECTED_DATA_MIGRATIONS", + ); + assert_eq!( + CATALOG_VERSION.to_string(), + EXPECTED_CATALOG_VERSION, + "CATALOG_VERSION changed: update EXPECTED_CATALOG_VERSION", + ); } - sqlx::query( - "INSERT INTO schema_history (filename) VALUES ($1) ON CONFLICT (filename) DO NOTHING", - ) - .bind(filename) - .execute(pool) - .await - .map_err(|e| OpError::Internal(format!("Record migration: {e}")))?; - Ok(()) } diff --git a/crates/storage/src/bootstrapper.rs b/crates/storage/src/bootstrapper.rs index 9fefd361..b720c91b 100755 --- a/crates/storage/src/bootstrapper.rs +++ b/crates/storage/src/bootstrapper.rs @@ -107,6 +107,13 @@ pub trait Bootstrapper: Send + Sync { /// Read the current catalog schema version. async fn read_catalog_version(&self) -> OpResult>; + /// True if the catalog was created by the pre-sqlx migration runner and + /// therefore cannot be upgraded in place (ADR-0003 requires destroy + init). + /// Backends without a legacy runner return false. + async fn catalog_predates_sqlx(&self) -> OpResult { + Ok(false) + } + /// Get the expected catalog version for this binary. fn expected_catalog_version(&self) -> String; diff --git a/docs/adr/0003-catalog-migration-mechanism.md b/docs/adr/0003-catalog-migration-mechanism.md index e27102db..662f8e7e 100644 --- a/docs/adr/0003-catalog-migration-mechanism.md +++ b/docs/adr/0003-catalog-migration-mechanism.md @@ -137,8 +137,10 @@ be recreated and reloaded after `init`. **Operational notes** - Migrations run only during `init` and `migrate`, never while serving. -- If a migration dies mid-apply, sqlx marks it dirty and refuses to proceed until - an operator resolves it. +- If a migration dies mid-apply, sqlx rolls back its transaction completely, + leaving no partial state and no ledger row, so re-running `migrate` retries it. + (A migration is marked dirty and blocks further runs only if it opts out with a + `-- no-transaction` directive, which none do.) - This is scoped to migration mechanics. It does not touch the separate gap in how ExtendDB checks columns at query time. - CI guardrail (follow-up): apply every migration on a fresh database and assert an diff --git a/docs/design/01-requirements.md b/docs/design/01-requirements.md index e27724f6..709b17ff 100755 --- a/docs/design/01-requirements.md +++ b/docs/design/01-requirements.md @@ -416,7 +416,7 @@ The catalog database stores extenddb metadata: table definitions, indexes, tags, ### 7.2 Catalog Versioning -- REQ-CAT-005: The catalog carries a semver version (`major.minor.patch`). Major = breaking schema changes, minor = additive, patch = non-structural +- REQ-CAT-005: The catalog carries a semver version (`major.minor.patch`). Major = breaking schema changes, minor = additive, patch = non-structural. Pre-1.0, breaking changes ride a minor bump (major stays 0), per standard semver for unstable releases. - REQ-CAT-006: The expected catalog version is a build-time constant compiled into the binary — not configurable at runtime - REQ-CAT-007: On startup, the server validates the catalog version matches the version compiled into the binary. Mismatch → refuse to start with a clear error directing the user to run `extenddb migrate` - REQ-CAT-008: `extenddb --version` prints both binary version and expected catalog version: `extenddb 0.1.0 (catalog 1.0.0)`. Binary version and catalog version are independent diff --git a/docs/design/04-component-storage.md b/docs/design/04-component-storage.md index c721cc47..e79d7c75 100755 --- a/docs/design/04-component-storage.md +++ b/docs/design/04-component-storage.md @@ -551,13 +551,18 @@ async fn transact_write_items(&self, input: TransactWriteInput) -> Result<...> { ### 5.6 Migrations -Migrations are embedded in the binary at compile time via `include_str!` and applied in order by the -`catalog::run_migrations` helper. Each migration is tracked in the `schema_history` table. +Migrations are embedded in the binary at compile time via `sqlx::migrate!` and applied in order by the +`run_catalog_migrations` / `run_data_migrations` helpers. Each migration is tracked, with a per-file +checksum, in the sqlx-managed `_sqlx_migrations` table (ADR-0003). -Migration files are numbered sequentially: +Migration files are numbered sequentially, one tree per database: ``` -migrations/ -└── 001_initial_schema.sql +migrations/ # catalog database +└── 001_schema.sql +data_migrations/ # data database +├── 001_data_schema.sql +├── 002_gsi_pending.sql +└── 003_idempotency_account_scope.sql ``` ## 6. GSI Consistency Model diff --git a/docs/getting-started.md b/docs/getting-started.md index 369c9ae7..299e4644 100755 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -146,7 +146,7 @@ You should see all checks pass: --- Checking catalog connection... OK: Connected to catalog. --- Checking catalog version... - OK: Catalog version 0.0.2 + OK: Catalog version 0.1.0 --- Checking data database... OK: Connected to data database 'extenddb_catalog_data'. --- Enumerating tables... @@ -162,7 +162,7 @@ extenddb runs as a daemon (background process) and logs to syslog. On startup it ```bash ./target/release/extenddb serve --config extenddb.toml -# extenddb 0.0.2 (catalog 0.0.2) listening on 127.0.0.1:18443 +# extenddb 0.1.0 (catalog 0.1.0) listening on 127.0.0.1:18443 ``` Check status (includes the daemon PID): @@ -1239,8 +1239,8 @@ Each runner requires its tools to be installed. The runner checks prerequisites ```bash ./target/release/extenddb version -# extenddb 0.0.2 -# catalog 0.0.2 +# extenddb 0.1.0 +# catalog 0.1.0 # commit abc1234 # built 2026-04-17T12:00:00Z ``` diff --git a/docs/manuals/01-architecture-guide.md b/docs/manuals/01-architecture-guide.md index be2b2e0e..233f97e9 100755 --- a/docs/manuals/01-architecture-guide.md +++ b/docs/manuals/01-architecture-guide.md @@ -160,7 +160,7 @@ extenddb uses a dual-database architecture: - **Catalog database** (e.g., `extenddb_catalog`): Stores table metadata, account/user/group/role/policy definitions, access keys, settings, stream metadata, and metrics. Shared across all accounts. - **Data database** (e.g., `extenddb_catalog_data`): Stores user items, GSI/LSI data, and stream records. Each table gets its own PostgreSQL table. -The catalog version (currently 0.0.2) is stored in the `catalog_metadata` table and checked at startup. Version mismatches prevent the server from starting — run `extenddb migrate` to upgrade. +The catalog version (currently 0.1.0) is stored in the `settings` table and checked at startup. Version mismatches prevent the server from starting — run `extenddb migrate` to upgrade. ## Pluggable Architecture diff --git a/docs/manuals/02-design-guide.md b/docs/manuals/02-design-guide.md index f7754014..39fbcf4b 100755 --- a/docs/manuals/02-design-guide.md +++ b/docs/manuals/02-design-guide.md @@ -22,7 +22,7 @@ The data database connection string is stored in the catalog's `settings` table | `indexes` | GSI/LSI metadata. FK on `table_id` (UUID) with CASCADE delete. | | `tags` | Resource tags. PK: `(resource_arn, tag_key)`. | | `settings` | Key-value store for catalog version, data DB URL, and runtime settings. | -| `schema_history` | Migration tracking. Records which SQL files have been applied. | +| `_sqlx_migrations` | Migration tracking (sqlx-managed). Records applied migrations with a per-file checksum. | | `admin_users` | Admin credentials (bcrypt-hashed passwords). | | `iam_users` | IAM users scoped to accounts. Optional console password. | | `iam_groups` | IAM groups scoped to accounts. | diff --git a/docs/manuals/04-quickstart-setup-guide.md b/docs/manuals/04-quickstart-setup-guide.md index 3a881a3d..96cd71a7 100755 --- a/docs/manuals/04-quickstart-setup-guide.md +++ b/docs/manuals/04-quickstart-setup-guide.md @@ -129,8 +129,8 @@ Check the version: ```bash ./target/release/extenddb version -# extenddb 0.0.2 -# catalog 0.0.2 +# extenddb 0.1.0 +# catalog 0.1.0 # commit abc1234 # built 2026-04-17T12:00:00Z ``` @@ -171,7 +171,7 @@ Expected output: --- Checking catalog connection... OK: Connected to catalog. --- Checking catalog version... - OK: Catalog version 0.0.2 + OK: Catalog version 0.1.0 --- Checking data database... OK: Connected to data database 'extenddb_catalog_data'. --- Enumerating tables... diff --git a/docs/manuals/05-admin-guide.md b/docs/manuals/05-admin-guide.md index 61621ada..25a3df99 100755 --- a/docs/manuals/05-admin-guide.md +++ b/docs/manuals/05-admin-guide.md @@ -487,10 +487,10 @@ Check that PostgreSQL is running and the connection string in `extenddb.toml` is **Catalog version mismatch:** ``` -Error: catalog version mismatch: found 1.0.0, expected 0.0.2 +Error: catalog version mismatch: found 1.0.0, expected 0.1.0 ``` -Run `extenddb migrate --config extenddb.toml` to upgrade the catalog schema. +Run `extenddb migrate --config extenddb.toml` to upgrade the catalog schema. If `migrate` reports that the catalog predates the sqlx migration system (the 0.1.0 adoption in ADR-0003), there is no in-place upgrade: run `extenddb destroy` then `extenddb init` as described in the [upgrade manual](07-upgrade-manual.md). ### Authentication Errors diff --git a/docs/manuals/07-upgrade-manual.md b/docs/manuals/07-upgrade-manual.md index 86f9e8e9..5e404d84 100755 --- a/docs/manuals/07-upgrade-manual.md +++ b/docs/manuals/07-upgrade-manual.md @@ -4,29 +4,35 @@ ## Current Status -ExtendDB 0.0.2 is the initial release. There is no upgrade path from a previous version — all deployments are fresh installs via `extenddb init`. +ExtendDB is at catalog version 0.1.0. Schema migrations for both PostgreSQL databases (the catalog and the data database) are handled by sqlx's built-in migrator (`sqlx::migrate!`). Each database records applied migrations, with a per-file checksum, in a table sqlx manages called `_sqlx_migrations`. -Future releases will include migrations that upgrade the catalog schema in place. The migration infrastructure is built and ready; this document describes how it works and how developers should think about adding new migrations. +Adopting sqlx is a one-time breaking change: an existing catalog created by an earlier build has no `_sqlx_migrations` table and cannot be upgraded in place. Upgrading to 0.1.0 requires `destroy` + `init`, which drops both databases. This is acceptable at v0.1 with no dependent catalogs. See [ADR-0003](../adr/0003-catalog-migration-mechanism.md) for the decision. Later migrations are additive and do not lose data. ## How Catalog Upgrades Work ### The Migration System -Migrations are SQL files in `crates/storage-postgres/migrations/`, applied in filename order: +Migrations are SQL files under two directories, applied in filename order: ``` -001_schema.sql ← current: the complete initial schema -002_.sql ← future: first incremental migration +crates/storage-postgres/migrations/ ← catalog database + 001_schema.sql +crates/storage-postgres/data_migrations/ ← data database + 001_data_schema.sql + 002_gsi_pending.sql + 003_idempotency_account_scope.sql ``` -The `schema_history` table tracks which files have been applied. When `extenddb migrate` runs, it: +sqlx embeds these files into the binary at compile time. When `extenddb init` or `extenddb migrate` runs a migrator, sqlx: -1. Reads all migration files embedded in the binary (via `include_str!`) -2. Checks `schema_history` for each filename -3. Applies any unapplied migrations in order -4. Records each applied filename in `schema_history` +1. Creates the `_sqlx_migrations` table if it does not exist. +2. Reads the version and a checksum of each embedded file. +3. Applies any file not yet recorded, in version order, each in its own transaction. +4. Verifies that every already-applied file still matches its recorded checksum. -Running `extenddb migrate` on an up-to-date catalog is a no-op. +Step 4 is the safety net. If a file that already shipped is edited after it was applied, sqlx refuses to run and reports `migration was previously applied but has been modified`. Editing a shipped migration is a hard error, not a silent no-op. A new schema change is always a new numbered file. + +Running `extenddb migrate` on an up-to-date deployment applies nothing, but still runs both migrators so the checksum check above executes. That is deliberate: it catches an edited migration file even when no new migration is pending. ### The Catalog Version @@ -34,97 +40,99 @@ A single row in the `settings` table stores the catalog version: ```sql SELECT value FROM settings WHERE key = 'catalog_version'; --- '0.0.2' +-- '0.1.0' ``` -The binary embeds an expected catalog version (`CATALOG_VERSION` constant in `crates/storage-postgres/src/lib.rs`). At startup, the server compares the database value against the binary's expectation. If they don't match, the server refuses to start and directs the operator to run `extenddb migrate`. +sqlx has no knowledge of our semver, so the version is not written by a migration file. `init` and `migrate` write it in a separate step, right after the catalog migrator runs, from the compiled-in `CATALOG_VERSION` constant (`crates/storage-postgres/src/lib.rs`). This write is intentionally not atomic with the migration. If a crash lands between the migration and the version write, re-running `extenddb migrate` repairs the version (sqlx skips the already-applied migration). On a first-time `init`, the same crash leaves no version and no config file, so recovery there is `destroy` + `init`. + +At startup the server compares the stored `catalog_version` against the binary's `CATALOG_VERSION`. If they do not match exactly, the server refuses to start and directs the operator to run `extenddb migrate`. The gate is symmetric: an older binary against a newer catalog also refuses. It exists to stop a server serving a schema it was not built for. The data database has no separate version; it relies on its own `_sqlx_migrations` table. ### Version Semantics The catalog version follows semantic versioning: -- **MAJOR**: Breaking schema changes that may require data migration or downtime -- **MINOR**: New tables or columns (backward-compatible, additive) -- **PATCH**: Index changes, constraint fixes, seed data updates +- **MAJOR**: Breaking schema changes that may require data migration or downtime. +- **MINOR**: New tables or columns (backward-compatible, additive). +- **PATCH**: Index changes, constraint fixes, seed data updates. -## Writing a New Migration +Pre-1.0, the project is unstable and a breaking change rides a MINOR bump (MAJOR stays 0), per standard semver. The 0.1.0 sqlx adoption is such a case: a MINOR bump that is breaking (requires re-init). -When you need to change the catalog schema, here's the process: +## Writing a New Migration -### 1. Create the migration file +### Catalog migration -Add a new SQL file with the next sequence number: +1. Add a new SQL file with the next sequence number: ``` crates/storage-postgres/migrations/002_your_feature.sql ``` -The file should be a single transaction: +Do not wrap it in `BEGIN`/`COMMIT`. sqlx runs each migration in its own transaction. Write plain DDL: ```sql -- Copyright 2026 ExtendDB contributors -- SPDX-License-Identifier: Apache-2.0 --- Migration 002: Brief description of what this adds/changes. - -BEGIN; +-- Migration 002: Brief description of what this adds. --- Your DDL here. ALTER TABLE tables ADD COLUMN IF NOT EXISTS new_column TEXT; - --- Bump the catalog version. -UPDATE settings SET value = '0.1.0' WHERE key = 'catalog_version'; - -COMMIT; ``` -### 2. Register it in the migration runner +Do not write the catalog version here. The migration runner writes it after the migrator completes. -Add the file to `CATALOG_MIGRATIONS` in `crates/storage-postgres/src/migrations.rs`: +2. Bump the catalog version constant in `crates/storage-postgres/src/lib.rs`: ```rust -pub(crate) const CATALOG_MIGRATIONS: &[(&str, &str)] = &[ - ( - "001_schema.sql", - include_str!("../../storage-postgres/migrations/001_schema.sql"), - ), - ( - "002_your_feature.sql", - include_str!("../../storage-postgres/migrations/002_your_feature.sql"), - ), -]; +pub const CATALOG_VERSION: CatalogVersion = CatalogVersion::new(0, 2, 0); ``` -### 3. Bump the catalog version constant - -In `crates/storage-postgres/src/lib.rs`: +3. Update the tripwire test in `crates/storage-postgres/src/migrations.rs` so `EXPECTED_CATALOG_MIGRATIONS` and `EXPECTED_CATALOG_VERSION` match the new count and version. The test fails if a migration is added without a matching version decision. -```rust -pub const CATALOG_VERSION: CatalogVersion = CatalogVersion::new(0, 1, 0); -``` +**Do not edit `001_schema.sql` or any file that already shipped.** sqlx checksums file bytes; changing an applied file is a hard error. A new column or table is always a new numbered file. sqlx applies all numbered files in order on a fresh `init`, so a new file reaches fresh installs and existing deployments through the same path. -This must match the version written by your migration's `UPDATE settings` statement. +### Data migration -### 4. Update 001_schema.sql - -The consolidated schema file is what fresh installs get. Add your new column/table/index to `001_schema.sql` as well, and update its `INSERT INTO settings` to seed the new version. This way fresh installs get the final schema in one pass, while existing deployments get there via the incremental migration. +A data-schema change is a new file under `crates/storage-postgres/data_migrations/` (for example `004_your_change.sql`), plus a bump of `EXPECTED_DATA_MIGRATIONS` in the tripwire test. The data database has no version, so `CATALOG_VERSION` does not change. `extenddb migrate`, not just `init`, runs the data migrator, so existing deployments pick up the change. ### Design Considerations -**Idempotency.** Use `IF NOT EXISTS`, `IF EXISTS`, and `ADD COLUMN IF NOT EXISTS` so migrations can be safely re-run. +**Additive only.** Prefer new columns with defaults and new tables over dropping or renaming. There are no down migrations; a mistake is corrected by a new forward migration. -**Backward compatibility.** Prefer additive changes (new columns with defaults, new tables) over destructive ones (dropping columns, renaming tables). A running server on the old binary should survive the schema change until it's restarted with the new binary. +**Idempotent DDL.** Use `IF NOT EXISTS` and `ADD COLUMN IF NOT EXISTS`. sqlx will not re-run an applied file, but idempotent DDL is a cheap safeguard. -**Transaction boundaries.** Wrap each migration in `BEGIN`/`COMMIT`. If any statement fails, the entire migration rolls back and the catalog stays at the previous version. +**Transactions.** sqlx wraps each migration in a transaction. Do not add `BEGIN`/`COMMIT`. A statement that cannot run inside a transaction (for example `CREATE INDEX CONCURRENTLY`) needs a `-- no-transaction` directive on the first line of that migration file; use it only when a specific statement requires it. -**No data migrations in DDL files.** If a schema change requires backfilling data, do it in Rust code triggered by `extenddb migrate`, not in raw SQL. This gives you error handling, progress reporting, and the ability to batch large updates. +**Line endings.** `.gitattributes` pins `*.sql text eol=lf` so a contributor's line-ending rewrite does not change file bytes and trip a false checksum mismatch. Keep migration files LF. -**Test both paths.** Every migration must be tested two ways: -1. Fresh install (`extenddb init`) — verifies `001_schema.sql` is correct -2. Upgrade (`extenddb migrate` on a catalog at the previous version) — verifies the incremental migration works +**No data backfill in DDL files.** If a schema change requires backfilling rows, do it in Rust triggered by `extenddb migrate`, not in raw SQL, so you get error handling and batching. ## General Upgrade Procedure -For future releases that include catalog changes: +### Upgrading to 0.1.0 (adopting sqlx) + +This upgrade is breaking. There is no in-place path from a pre-sqlx catalog. + +1. **Stop the server** + +```bash +extenddb stop --config extenddb.toml +``` + +2. **Destroy and re-initialize** (drops both databases; all tables and items are lost and must be recreated) + +```bash +extenddb destroy --config extenddb.toml --yes +extenddb init --config extenddb.toml +``` + +3. **Start the server** + +```bash +extenddb serve --config extenddb.toml +``` + +### Later releases (additive migrations) + +For future releases that add migrations without a compatibility break: 1. **Stop the server** @@ -149,26 +157,23 @@ cargo build --release 4. **Run migrations** ```bash -extenddb migrate --config extenddb.toml +extenddb migrate --config extenddb.toml --yes ``` -5. **Verify** +5. **Verify and start** ```bash extenddb verify --config extenddb.toml -``` - -6. **Start the server** - -```bash extenddb serve --config extenddb.toml ``` +The server binary and the catalog are version-locked, so a schema-bumping upgrade needs every server moved to the matching version together: a brief coordinated outage (stop old servers, `migrate`, start new). Online / rolling upgrades are out of scope. + ## Rollback Procedure -If an upgrade fails: +If a migration dies mid-apply, sqlx rolls back that migration's transaction completely, so it leaves no partial state and no ledger row: re-running `extenddb migrate` simply retries it. (A migration is only marked dirty, blocking further runs until resolved, if it opts out of the transaction with a `-- no-transaction` directive, which none of ours do.) If an upgrade otherwise fails: -1. Stop the server +1. Stop the server. 2. Restore from backup: ```bash @@ -177,15 +182,21 @@ psql -c "CREATE DATABASE extenddb_catalog OWNER extenddb;" psql -d extenddb_catalog -f catalog_backup_YYYYMMDD.sql ``` -3. Rebuild the previous version and start it +3. Rebuild the previous version and start it. ## Version History -### Catalog 0.0.2 (Current — Initial Release) +> This section is the project changelog: each catalog version and what changed. + +### Catalog 0.1.0 (Current) + +Adopted sqlx's migrator for both databases, replacing the homegrown filename-tracked runner. Each database now tracks applied migrations, with checksums, in `_sqlx_migrations`; the old `schema_history` table is gone. `extenddb migrate` runs both the catalog and data migrators. The catalog version is written by `init` and `migrate` after the migrator runs, not seeded inside a migration file. + +Breaking: upgrading from a pre-sqlx catalog requires `destroy` + `init`, which wipes both databases. Acceptable at v0.1 with no dependent catalogs. -Complete schema: accounts, tables, indexes, tags, streams, IAM (users, groups, roles, policies, access keys, sessions, permissions boundaries), idempotency tokens, metrics, login attempts, backups, continuous backups, TTL support, settings. +### Catalog 0.0.2 (Initial release, superseded) -No prior versions exist. All deployments are fresh installs. +Complete initial schema: accounts, tables, indexes, tags, streams, IAM (users, groups, roles, policies, access keys, sessions, permissions boundaries), idempotency tokens, metrics, login attempts, backups, continuous backups, TTL support, settings. Managed by the homegrown runner and its `schema_history` table. --- diff --git a/docs/manuals/08-install-linux.md b/docs/manuals/08-install-linux.md index 6208193b..9947e96f 100755 --- a/docs/manuals/08-install-linux.md +++ b/docs/manuals/08-install-linux.md @@ -118,7 +118,7 @@ Expected: ``` === extenddb verify === ... - OK: Catalog version 0.0.2 + OK: Catalog version 0.1.0 ... === HEALTHY: All checks passed === ``` diff --git a/docs/manuals/09-install-macos.md b/docs/manuals/09-install-macos.md index 5a22f190..99854803 100755 --- a/docs/manuals/09-install-macos.md +++ b/docs/manuals/09-install-macos.md @@ -96,7 +96,7 @@ Expected: ``` === extenddb verify === ... - OK: Catalog version 0.0.2 + OK: Catalog version 0.1.0 ... === HEALTHY: All checks passed === ``` diff --git a/tests/test_cli_lifecycle.py b/tests/test_cli_lifecycle.py index 7bd3d8fb..6147217f 100644 --- a/tests/test_cli_lifecycle.py +++ b/tests/test_cli_lifecycle.py @@ -54,11 +54,11 @@ def test_init_creates_schema(self, cli_env): assert os.path.isfile(os.path.join(cli_env["tls_dir"], "key.pem")) def test_data_migrations_tracked(self, cli_env): - """init records every data migration in the data DB's schema_history. + """init records every data migration in the data DB's _sqlx_migrations. - Validates the data-migration registry: migrations are tracked (so - `extenddb migrate` knows what is pending and never re-runs an applied - one), and the GSI-pending migration is among them. + Validates the sqlx data-migration ledger: migrations are tracked (so + `extenddb migrate` never re-runs an applied one), and the GSI-pending + migration is among them. """ import psycopg2 @@ -74,28 +74,28 @@ def test_data_migrations_tracked(self, cli_env): conn = psycopg2.connect(PG_ADMIN_CONN + "/" + data_db) try: with conn.cursor() as cur: - cur.execute("SELECT filename FROM schema_history ORDER BY filename") + cur.execute("SELECT version FROM _sqlx_migrations ORDER BY version") tracked = {row[0] for row in cur.fetchall()} finally: conn.close() - assert "001_data_schema.sql" in tracked, tracked - assert "002_gsi_pending.sql" in tracked, tracked + # 001_data_schema, 002_gsi_pending, 003_idempotency_account_scope. + assert {1, 2, 3} <= tracked, tracked def test_migrate_applies_pending_data_migration(self, cli_env): """`extenddb migrate` applies a pending *data* migration on an existing deployment — not just catalog migrations. - Regression: data migrations live in the data database's own - `schema_history`, independent of the catalog version. `migrate` + Regression: data migrations live in the data database's own sqlx ledger + (`_sqlx_migrations`), independent of the catalog version. `migrate` previously only ran catalog migrations and returned "up to date" when the catalog version matched, so a release that only added a data migration (e.g. `002_gsi_pending.sql`) was never applied on upgrade — the `gsi_pending` table was missing and every async-GSI write failed. - Simulates a pre-002 deployment by dropping `gsi_pending` and its - ledger row, then asserts `migrate` re-applies it (and refuses without - `--yes`). + Simulates a pre-002 deployment by dropping `gsi_pending` and the sqlx + ledger rows from version 2 on, then asserts `migrate` re-applies them + (and refuses without `--yes`). """ import psycopg2 @@ -111,15 +111,16 @@ def test_migrate_applies_pending_data_migration(self, cli_env): def _data_conn(): return psycopg2.connect(PG_ADMIN_CONN + "/" + data_db) - # Simulate a deployment created before 002 existed. + # Simulate a deployment created before 002 existed. Delete the ledger + # rows from version 2 on so the applied set is {1}; sqlx re-applies + # every embedded migration missing from _sqlx_migrations, in order + # (2 then 3). conn = _data_conn() try: conn.autocommit = True with conn.cursor() as cur: cur.execute("DROP TABLE IF EXISTS gsi_pending") - cur.execute( - "DELETE FROM schema_history WHERE filename = '002_gsi_pending.sql'" - ) + cur.execute("DELETE FROM _sqlx_migrations WHERE version >= 2") cur.execute("SELECT to_regclass('public.gsi_pending') IS NOT NULL") assert cur.fetchone()[0] is False finally: @@ -147,8 +148,7 @@ def _data_conn(): cur.execute("SELECT to_regclass('public.gsi_pending') IS NOT NULL") assert cur.fetchone()[0] is True cur.execute( - "SELECT EXISTS(SELECT 1 FROM schema_history " - "WHERE filename = '002_gsi_pending.sql')" + "SELECT EXISTS(SELECT 1 FROM _sqlx_migrations WHERE version = 2)" ) assert cur.fetchone()[0] is True # The 002 schema must carry the per-index queue columns. @@ -161,6 +161,60 @@ def _data_conn(): finally: conn.close() + def test_migrate_refuses_pre_sqlx_catalog(self, cli_env): + """`extenddb migrate` refuses a catalog created by the pre-sqlx runner. + + ADR-0003 adopts sqlx with a re-init upgrade path (no in-place shim). A + catalog that predates sqlx (has `schema_history`, no `_sqlx_migrations`) + must not be migrated in place: `migrate` refuses and directs the operator + to `destroy` + `init`, rather than failing later on a non-idempotent DDL + re-run and leaving the catalog half-adopted. + """ + import psycopg2 + + result = _run_extenddb( + "init", *_init_args(cli_env), + config=cli_env["config_path"], + env_override={"EXTENDDB_ADMIN_PASSWORD": "TestPass1!"}, + ) + assert result.returncode == 0 + + # Rewrite the catalog to look like a pre-sqlx deployment: legacy + # schema_history table, no sqlx ledger, old catalog version. + conn = psycopg2.connect(PG_ADMIN_CONN + "/" + cli_env["db_name"]) + try: + conn.autocommit = True + with conn.cursor() as cur: + cur.execute("DROP TABLE IF EXISTS _sqlx_migrations") + cur.execute("CREATE TABLE schema_history (filename TEXT PRIMARY KEY)") + cur.execute( + "UPDATE settings SET value = '0.0.2' WHERE key = 'catalog_version'" + ) + finally: + conn.close() + + refused = _run_extenddb( + "migrate", "--yes", *_pg_args(), + config=cli_env["config_path"], + check=False, + ) + assert refused.returncode != 0, refused.stdout + refused.stderr + combined = (refused.stdout + refused.stderr).lower() + assert "predates" in combined, combined + assert "destroy" in combined and "init" in combined, combined + + # The version must be left untouched (no in-place stamp), and the guard + # must have fired before the migrator connected: no sqlx ledger created. + conn = psycopg2.connect(PG_ADMIN_CONN + "/" + cli_env["db_name"]) + try: + with conn.cursor() as cur: + cur.execute("SELECT value FROM settings WHERE key = 'catalog_version'") + assert cur.fetchone()[0] == "0.0.2" + cur.execute("SELECT to_regclass('public._sqlx_migrations') IS NULL") + assert cur.fetchone()[0] is True + finally: + conn.close() + def test_init_serve_status_stop(self, cli_env): """Full lifecycle: init → serve → status → stop.""" # Init