From 70f5d80ab794178867f43ef3065154ed008638bd Mon Sep 17 00:00:00 2001 From: Ivan Barlog Date: Fri, 22 May 2026 18:33:44 +0200 Subject: [PATCH] feat: add SQLite storage backend (extenddb-storage-sqlite) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements a complete DynamoDB-compatible storage backend backed by SQLite, suitable for local development and CI environments without requiring a PostgreSQL instance. Key design decisions vs the PostgreSQL backend: - Single pool (`self.pool`) for all catalog and data operations - `?` positional placeholders throughout (not `$N`) - No `FOR UPDATE` — SQLite's writer serialization handles concurrency - Synchronous GSI/LSI updates (no async propagation queue) - `seq_counters` table replaces PostgreSQL sequences for stream ordering - `rowid % total_segments` for parallel scan (vs `hashtext(pk) %`) - OR-expansion `(pk > ? OR (pk = ? AND sk > ?))` for pagination (SQLite lacks tuple comparison) - `char(1114111)` as string upper-bound in begins-with queries (not `chr()`) Implemented traits: TableEngine, DataEngine, MetadataEngine, StreamEngine, BackupEngine, WorkerStore, CatalogStore, and all management/auth stores (AdminStore, AuthorizationStore, CredentialStore, ManagementStore). Co-Authored-By: Claude Sonnet 4.6 --- Cargo.lock | 30 + Cargo.toml | 4 +- crates/bin/Cargo.toml | 4 +- crates/storage-sqlite/Cargo.toml | 32 + .../storage-sqlite/migrations/001_schema.sql | 305 +++++++++ crates/storage-sqlite/src/admin_store.rs | 138 +++++ .../storage-sqlite/src/authorization_store.rs | 251 ++++++++ crates/storage-sqlite/src/backup_engine.rs | 545 ++++++++++++++++ crates/storage-sqlite/src/bootstrapper.rs | 369 +++++++++++ crates/storage-sqlite/src/catalog_store.rs | 410 ++++++++++++ crates/storage-sqlite/src/config.rs | 65 ++ crates/storage-sqlite/src/create_table.rs | 396 ++++++++++++ crates/storage-sqlite/src/credential_store.rs | 179 ++++++ crates/storage-sqlite/src/data/data_engine.rs | 335 ++++++++++ crates/storage-sqlite/src/data/ddl.rs | 314 ++++++++++ crates/storage-sqlite/src/data/delete_item.rs | 273 ++++++++ crates/storage-sqlite/src/data/index.rs | 259 ++++++++ crates/storage-sqlite/src/data/mod.rs | 128 ++++ crates/storage-sqlite/src/data/put_item.rs | 330 ++++++++++ crates/storage-sqlite/src/data/query.rs | 180 ++++++ crates/storage-sqlite/src/data/query_scan.rs | 243 ++++++++ .../storage-sqlite/src/data/transactions.rs | 393 ++++++++++++ crates/storage-sqlite/src/data/tx_helpers.rs | 331 ++++++++++ crates/storage-sqlite/src/data/update_item.rs | 234 +++++++ crates/storage-sqlite/src/delete_table.rs | 114 ++++ crates/storage-sqlite/src/engine.rs | 130 ++++ crates/storage-sqlite/src/lib.rs | 268 ++++++++ .../src/management_store/access_keys.rs | 228 +++++++ .../src/management_store/accounts.rs | 219 +++++++ .../src/management_store/groups.rs | 212 +++++++ .../src/management_store/mod.rs | 582 ++++++++++++++++++ .../src/management_store/policies.rs | 248 ++++++++ .../src/management_store/roles.rs | 272 ++++++++ .../src/management_store/users.rs | 370 +++++++++++ crates/storage-sqlite/src/metadata_engine.rs | 448 ++++++++++++++ crates/storage-sqlite/src/migrations.rs | 91 +++ crates/storage-sqlite/src/operations.rs | 71 +++ crates/storage-sqlite/src/sqlite_util.rs | 44 ++ crates/storage-sqlite/src/stream_engine.rs | 491 +++++++++++++++ crates/storage-sqlite/src/table_engine.rs | 146 +++++ crates/storage-sqlite/src/table_helpers.rs | 322 ++++++++++ crates/storage-sqlite/src/update_table.rs | 385 ++++++++++++ crates/storage-sqlite/src/worker_store.rs | 104 ++++ crates/storage-sqlite/src/workers.rs | 151 +++++ 44 files changed, 10642 insertions(+), 2 deletions(-) create mode 100644 crates/storage-sqlite/Cargo.toml create mode 100644 crates/storage-sqlite/migrations/001_schema.sql create mode 100644 crates/storage-sqlite/src/admin_store.rs create mode 100644 crates/storage-sqlite/src/authorization_store.rs create mode 100644 crates/storage-sqlite/src/backup_engine.rs create mode 100644 crates/storage-sqlite/src/bootstrapper.rs create mode 100644 crates/storage-sqlite/src/catalog_store.rs create mode 100644 crates/storage-sqlite/src/config.rs create mode 100644 crates/storage-sqlite/src/create_table.rs create mode 100644 crates/storage-sqlite/src/credential_store.rs create mode 100644 crates/storage-sqlite/src/data/data_engine.rs create mode 100644 crates/storage-sqlite/src/data/ddl.rs create mode 100644 crates/storage-sqlite/src/data/delete_item.rs create mode 100644 crates/storage-sqlite/src/data/index.rs create mode 100644 crates/storage-sqlite/src/data/mod.rs create mode 100644 crates/storage-sqlite/src/data/put_item.rs create mode 100644 crates/storage-sqlite/src/data/query.rs create mode 100644 crates/storage-sqlite/src/data/query_scan.rs create mode 100644 crates/storage-sqlite/src/data/transactions.rs create mode 100644 crates/storage-sqlite/src/data/tx_helpers.rs create mode 100644 crates/storage-sqlite/src/data/update_item.rs create mode 100644 crates/storage-sqlite/src/delete_table.rs create mode 100644 crates/storage-sqlite/src/engine.rs create mode 100644 crates/storage-sqlite/src/lib.rs create mode 100644 crates/storage-sqlite/src/management_store/access_keys.rs create mode 100644 crates/storage-sqlite/src/management_store/accounts.rs create mode 100644 crates/storage-sqlite/src/management_store/groups.rs create mode 100644 crates/storage-sqlite/src/management_store/mod.rs create mode 100644 crates/storage-sqlite/src/management_store/policies.rs create mode 100644 crates/storage-sqlite/src/management_store/roles.rs create mode 100644 crates/storage-sqlite/src/management_store/users.rs create mode 100644 crates/storage-sqlite/src/metadata_engine.rs create mode 100644 crates/storage-sqlite/src/migrations.rs create mode 100644 crates/storage-sqlite/src/operations.rs create mode 100644 crates/storage-sqlite/src/sqlite_util.rs create mode 100644 crates/storage-sqlite/src/stream_engine.rs create mode 100644 crates/storage-sqlite/src/table_engine.rs create mode 100644 crates/storage-sqlite/src/table_helpers.rs create mode 100644 crates/storage-sqlite/src/update_table.rs create mode 100644 crates/storage-sqlite/src/worker_store.rs create mode 100644 crates/storage-sqlite/src/workers.rs diff --git a/Cargo.lock b/Cargo.lock index c74e1077..ae9726a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -852,6 +852,7 @@ dependencies = [ "extenddb-server", "extenddb-storage", "extenddb-storage-postgres", + "extenddb-storage-sqlite", "libc", "rand 0.9.4", "rcgen", @@ -994,6 +995,34 @@ dependencies = [ "zeroize", ] +[[package]] +name = "extenddb-storage-sqlite" +version = "0.1.0" +dependencies = [ + "aes-gcm", + "anyhow", + "async-trait", + "base64 0.22.1", + "bcrypt", + "bigdecimal", + "crc32fast", + "extenddb-auth", + "extenddb-core", + "extenddb-storage", + "futures", + "inventory", + "rand 0.9.4", + "serde", + "serde_json", + "sqlx", + "time", + "tokio", + "toml", + "tracing", + "uuid", + "zeroize", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1654,6 +1683,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ + "cc", "pkg-config", "vcpkg", ] diff --git a/Cargo.toml b/Cargo.toml index b5581126..7371b3b5 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/engine", "crates/storage", "crates/storage-postgres", + "crates/storage-sqlite", "crates/auth", "crates/server", "crates/bin", @@ -24,6 +25,7 @@ extenddb-core = { path = "crates/core" } extenddb-engine = { path = "crates/engine" } extenddb-storage = { path = "crates/storage" } extenddb-storage-postgres = { path = "crates/storage-postgres" } +extenddb-storage-sqlite = { path = "crates/storage-sqlite" } extenddb-auth = { path = "crates/auth" } extenddb-server = { path = "crates/server" } @@ -51,7 +53,7 @@ hyper = { version = "1" } inventory = "0.3" # Database -sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres", "json", "time", "uuid", "bigdecimal"] } +sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres", "sqlite", "json", "time", "uuid", "bigdecimal"] } # Crypto & checksums crc32fast = "1" diff --git a/crates/bin/Cargo.toml b/crates/bin/Cargo.toml index 2a2c646a..05511499 100755 --- a/crates/bin/Cargo.toml +++ b/crates/bin/Cargo.toml @@ -12,14 +12,16 @@ name = "extenddb" path = "src/main.rs" [features] -default = ["postgres"] +default = ["postgres", "sqlite"] postgres = ["extenddb-storage-postgres"] +sqlite = ["extenddb-storage-sqlite"] [dependencies] extenddb-core = { workspace = true } extenddb-engine = { workspace = true } extenddb-storage = { workspace = true } extenddb-storage-postgres = { workspace = true, optional = true } +extenddb-storage-sqlite = { workspace = true, optional = true } extenddb-auth = { workspace = true } extenddb-server = { workspace = true } tokio = { workspace = true } diff --git a/crates/storage-sqlite/Cargo.toml b/crates/storage-sqlite/Cargo.toml new file mode 100644 index 00000000..137a3ccd --- /dev/null +++ b/crates/storage-sqlite/Cargo.toml @@ -0,0 +1,32 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +[package] +name = "extenddb-storage-sqlite" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +anyhow = { workspace = true } +extenddb-core = { workspace = true } +extenddb-storage = { workspace = true } +extenddb-auth = { workspace = true } +futures = { workspace = true } +inventory = { workspace = true } +sqlx = { workspace = true, features = ["sqlite"] } +tokio = { workspace = true, features = ["sync"] } +serde = { workspace = true } +serde_json = { workspace = true } +toml = { workspace = true } +tracing = { workspace = true } +time = { workspace = true } +uuid = { workspace = true } +base64 = { workspace = true } +crc32fast = { workspace = true } +rand = { workspace = true } +bcrypt = { workspace = true } +aes-gcm = { workspace = true } +async-trait = { workspace = true } +zeroize = { workspace = true } +bigdecimal = { version = "0.4" } diff --git a/crates/storage-sqlite/migrations/001_schema.sql b/crates/storage-sqlite/migrations/001_schema.sql new file mode 100644 index 00000000..27b38818 --- /dev/null +++ b/crates/storage-sqlite/migrations/001_schema.sql @@ -0,0 +1,305 @@ +-- Copyright 2026 ExtendDB contributors +-- SPDX-License-Identifier: Apache-2.0 +-- Consolidated schema for extenddb SQLite backend (catalog version 0.0.2). +-- All catalog and data tables live in one SQLite file. +-- JSON stored as TEXT, timestamps as TEXT (RFC 3339 / ISO 8601). + +-- Accounts — multi-account support. +CREATE TABLE IF NOT EXISTS accounts ( + account_id TEXT PRIMARY KEY, + account_name TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- Table metadata. +CREATE TABLE IF NOT EXISTS tables ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + table_name TEXT NOT NULL, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + table_status TEXT NOT NULL DEFAULT 'CREATING', + creation_date_time TEXT NOT NULL DEFAULT (datetime('now')), + table_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + table_arn TEXT NOT NULL, + table_id TEXT NOT NULL UNIQUE, + ttl_attribute TEXT, + deletion_protection_enabled INTEGER NOT NULL DEFAULT 0, + status_transition_at TEXT, + stream_label TEXT, + ttl_index_ready INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (account_id, table_name) +); + +CREATE INDEX IF NOT EXISTS idx_tables_pending_transition + ON tables (status_transition_at) + WHERE status_transition_at IS NOT NULL; + +-- Index metadata. +CREATE TABLE IF NOT EXISTS indexes ( + table_id TEXT NOT NULL, + index_id TEXT NOT NULL, + index_name TEXT NOT NULL, + index_type TEXT NOT NULL, + key_schema TEXT NOT NULL, + projection TEXT NOT NULL, + index_status TEXT NOT NULL DEFAULT 'ACTIVE', + provisioned_throughput TEXT, + propagation_delay_ms INTEGER, + PRIMARY KEY (table_id, index_name), + FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE +); + +-- Resource tags. +CREATE TABLE IF NOT EXISTS tags ( + resource_arn TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (resource_arn, tag_key) +); + +-- Migration tracking. +CREATE TABLE IF NOT EXISTS schema_history ( + filename TEXT PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- Settings (catalog version, runtime config). +CREATE TABLE IF NOT EXISTS settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +-- Stream shards. +CREATE TABLE IF NOT EXISTS stream_shards ( + shard_id TEXT PRIMARY KEY, + table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, + parent_shard_id TEXT, + starting_sequence_number TEXT NOT NULL, + ending_sequence_number TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_stream_shards_table ON stream_shards (table_id); + +-- Stream records. +CREATE TABLE IF NOT EXISTS stream_records ( + shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id) ON DELETE CASCADE, + sequence_number TEXT NOT NULL, + table_id TEXT NOT NULL, + event_name TEXT NOT NULL, + record_data TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (shard_id, sequence_number) +); + +CREATE INDEX IF NOT EXISTS idx_stream_records_created ON stream_records (created_at); + +-- Admin users. +CREATE TABLE IF NOT EXISTS admin_users ( + admin_name TEXT PRIMARY KEY, + password_hash TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- IAM users. +CREATE TABLE IF NOT EXISTS iam_users ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + user_name TEXT NOT NULL, + user_arn TEXT NOT NULL UNIQUE, + password_hash TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (account_id, user_name) +); + +-- IAM user tags. +CREATE TABLE IF NOT EXISTS iam_user_tags ( + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, user_name, tag_key), + FOREIGN KEY (account_id, user_name) REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); + +-- Access keys. +CREATE TABLE IF NOT EXISTS access_keys ( + access_key_id TEXT PRIMARY KEY, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + user_name TEXT NOT NULL, + is_active INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + FOREIGN KEY (account_id, user_name) REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); + +-- IAM groups. +CREATE TABLE IF NOT EXISTS iam_groups ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + group_name TEXT NOT NULL, + group_arn TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (account_id, group_name) +); + +-- IAM group membership. +CREATE TABLE IF NOT EXISTS iam_group_members ( + account_id TEXT NOT NULL, + group_name TEXT NOT NULL, + user_name TEXT NOT NULL, + PRIMARY KEY (account_id, group_name, user_name), + FOREIGN KEY (account_id, group_name) REFERENCES iam_groups(account_id, group_name) ON DELETE CASCADE, + FOREIGN KEY (account_id, user_name) REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE +); + +-- IAM roles. +CREATE TABLE IF NOT EXISTS iam_roles ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + role_name TEXT NOT NULL, + role_arn TEXT NOT NULL UNIQUE, + trust_policy TEXT NOT NULL, + permissions_boundary_arn TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (account_id, role_name) +); + +-- IAM role tags. +CREATE TABLE IF NOT EXISTS iam_role_tags ( + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + tag_key TEXT NOT NULL, + tag_value TEXT NOT NULL, + PRIMARY KEY (account_id, role_name, tag_key), + FOREIGN KEY (account_id, role_name) REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); + +-- IAM sessions. +CREATE TABLE IF NOT EXISTS iam_sessions ( + session_token TEXT PRIMARY KEY, + access_key_id TEXT NOT NULL UNIQUE, + secret_key_encrypted BLOB NOT NULL, + account_id TEXT NOT NULL, + role_name TEXT NOT NULL, + session_name TEXT NOT NULL, + session_tags TEXT, + session_policy TEXT, + expires_at TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + FOREIGN KEY (account_id, role_name) REFERENCES iam_roles(account_id, role_name) ON DELETE CASCADE +); + +-- IAM policies. +CREATE TABLE IF NOT EXISTS iam_policies ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL, + principal_name TEXT NOT NULL, + policy_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (account_id, principal_type, principal_name, policy_name) +); + +-- IAM permissions boundaries. +CREATE TABLE IF NOT EXISTS iam_permissions_boundaries ( + account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, + principal_type TEXT NOT NULL, + principal_name TEXT NOT NULL, + policy_document TEXT NOT NULL, + PRIMARY KEY (account_id, principal_type, principal_name) +); + +-- Idempotency tokens for TransactWriteItems. +CREATE TABLE IF NOT EXISTS idempotency_tokens ( + token TEXT PRIMARY KEY, + fingerprint TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_idempotency_tokens_created ON idempotency_tokens (created_at); + +-- Metrics (1-minute aggregation). +CREATE TABLE IF NOT EXISTS metrics ( + bucket TEXT NOT NULL, + metric TEXT NOT NULL, + table_name TEXT NOT NULL DEFAULT '', + index_name TEXT NOT NULL DEFAULT '', + operation TEXT NOT NULL DEFAULT '', + sum REAL NOT NULL DEFAULT 0, + count INTEGER NOT NULL DEFAULT 0, + min REAL NOT NULL DEFAULT 1e308, + max REAL NOT NULL DEFAULT -1e308, + PRIMARY KEY (bucket, metric, table_name, index_name, operation) +); + +CREATE INDEX IF NOT EXISTS idx_metrics_bucket ON metrics (bucket); + +-- Login attempt tracking. +CREATE TABLE IF NOT EXISTS login_attempts ( + principal TEXT NOT NULL, + attempted_at TEXT NOT NULL DEFAULT (datetime('now')), + success INTEGER NOT NULL, + source_ip TEXT +); + +CREATE INDEX IF NOT EXISTS idx_login_attempts_principal_time + ON login_attempts (principal, attempted_at); + +CREATE INDEX IF NOT EXISTS idx_login_attempts_source_ip_time + ON login_attempts (source_ip, attempted_at) + WHERE source_ip IS NOT NULL; + +-- Backup metadata. +CREATE TABLE IF NOT EXISTS backups ( + backup_arn TEXT PRIMARY KEY, + backup_name TEXT NOT NULL, + table_id TEXT NOT NULL, + table_name TEXT NOT NULL, + account_id TEXT NOT NULL, + backup_status TEXT NOT NULL DEFAULT 'AVAILABLE', + backup_type TEXT NOT NULL DEFAULT 'USER', + backup_size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + key_schema TEXT NOT NULL, + attribute_definitions TEXT NOT NULL, + billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', + provisioned_throughput TEXT, + stream_specification TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_backups_table ON backups (account_id, table_name); + +-- Backup items. +CREATE TABLE IF NOT EXISTS backup_items ( + backup_arn TEXT NOT NULL REFERENCES backups(backup_arn) ON DELETE CASCADE, + pk TEXT NOT NULL, + sk TEXT, + item_data TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_backup_items_arn ON backup_items (backup_arn); + +-- Continuous backups / PITR status. +CREATE TABLE IF NOT EXISTS continuous_backups ( + account_id TEXT NOT NULL, + table_name TEXT NOT NULL, + pitr_enabled INTEGER NOT NULL DEFAULT 0, + earliest_restorable TEXT, + latest_restorable TEXT, + PRIMARY KEY (account_id, table_name) +); + +-- Stream sequence counter (replaces PostgreSQL sequence). +CREATE TABLE IF NOT EXISTS seq_counters ( + name TEXT PRIMARY KEY, + value INTEGER NOT NULL DEFAULT 0 +); +INSERT OR IGNORE INTO seq_counters (name, value) VALUES ('stream', 0); + +-- Seed settings. +INSERT OR IGNORE INTO settings (key, value) VALUES ('catalog_version', '0.0.2'); +INSERT OR IGNORE INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0.25'); +INSERT OR IGNORE INTO settings (key, value) VALUES ('gsi_propagation_delay_ms', '10'); diff --git a/crates/storage-sqlite/src/admin_store.rs b/crates/storage-sqlite/src/admin_store.rs new file mode 100644 index 00000000..b035829f --- /dev/null +++ b/crates/storage-sqlite/src/admin_store.rs @@ -0,0 +1,138 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Admin user management for `SqliteCatalogStore`. + +use extenddb_storage::management_store::{AdminEntry, OpError, OpResult}; +use futures::future::BoxFuture; + +use crate::catalog_store::SqliteCatalogStore; + +impl extenddb_storage::management_store::AdminStore for SqliteCatalogStore { + fn create_admin(&self, admin_name: &str, password_hash: &str) -> BoxFuture<'_, OpResult<()>> { + let admin_name = admin_name.to_owned(); + let password_hash = password_hash.to_owned(); + let pool = self.pool().clone(); + Box::pin(async move { + let result = sqlx::query( + "INSERT INTO admin_users (admin_name, password_hash) VALUES (?, ?)", + ) + .bind(&admin_name) + .bind(&password_hash) + .execute(&pool) + .await; + match result { + Ok(_) => Ok(()), + Err(e) if crate::sqlite_util::is_unique_violation(&e) => { + Err(OpError::AlreadyExists("Admin user already exists".to_owned())) + } + Err(e) => { + tracing::error!("create_admin failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + }) + } + + fn list_admins(&self) -> BoxFuture<'_, OpResult>> { + let pool = self.pool().clone(); + Box::pin(async move { + let rows: Vec<(String, String)> = + sqlx::query_as("SELECT admin_name, created_at FROM admin_users ORDER BY admin_name") + .fetch_all(&pool) + .await + .map_err(|e| { + tracing::error!("list_admins: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(rows + .into_iter() + .filter_map(|(name, created_at_str)| { + let created_at = crate::sqlite_util::parse_timestamp(&created_at_str).ok()?; + Some(AdminEntry { admin_name: name, created_at }) + }) + .collect()) + }) + } + + fn delete_admin(&self, admin_name: &str) -> BoxFuture<'_, OpResult<()>> { + let admin_name = admin_name.to_owned(); + let pool = self.pool().clone(); + Box::pin(async move { + let result = sqlx::query("DELETE FROM admin_users WHERE admin_name = ?") + .bind(&admin_name) + .execute(&pool) + .await; + match result { + Ok(r) if r.rows_affected() == 0 => { + Err(OpError::NotFound("Admin user not found".to_owned())) + } + Ok(_) => Ok(()), + Err(e) => { + tracing::error!("delete_admin failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + }) + } + + fn change_admin_password( + &self, + admin_name: &str, + password_hash: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let admin_name = admin_name.to_owned(); + let password_hash = password_hash.to_owned(); + let pool = self.pool().clone(); + Box::pin(async move { + let result = + sqlx::query("UPDATE admin_users SET password_hash = ? WHERE admin_name = ?") + .bind(&password_hash) + .bind(&admin_name) + .execute(&pool) + .await; + match result { + Ok(r) if r.rows_affected() == 0 => { + Err(OpError::NotFound("Admin user not found".to_owned())) + } + Ok(_) => Ok(()), + Err(e) => { + tracing::error!("change_admin_password failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + }) + } + + fn verify_admin_password( + &self, + admin_name: &str, + password: &str, + ) -> BoxFuture<'_, OpResult>> { + let admin_name = admin_name.to_owned(); + let password = password.to_owned(); + let pool = self.pool().clone(); + Box::pin(async move { + let row: Option<(String,)> = + sqlx::query_as("SELECT password_hash FROM admin_users WHERE admin_name = ?") + .bind(&admin_name) + .fetch_optional(&pool) + .await + .map_err(|e| { + tracing::error!("verify_admin_password: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let Some((hash,)) = row else { + return Ok(None); + }; + + let verified = tokio::task::spawn_blocking(move || bcrypt::verify(&password, &hash)) + .await + .map_err(|e| OpError::Internal(format!("bcrypt verify task failed: {e}")))? + .map_err(|e| OpError::Internal(format!("bcrypt verify failed: {e}")))?; + + Ok(Some(verified)) + }) + } +} diff --git a/crates/storage-sqlite/src/authorization_store.rs b/crates/storage-sqlite/src/authorization_store.rs new file mode 100644 index 00000000..29538de5 --- /dev/null +++ b/crates/storage-sqlite/src/authorization_store.rs @@ -0,0 +1,251 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Authorization storage for IAM policy lookups (SQLite implementation). + +use extenddb_storage::authorization_store::{AuthorizationStore, SessionData}; +use extenddb_storage::management_store::{OpError, OpResult}; +use futures::future::BoxFuture; + +use crate::catalog_store::SqliteCatalogStore; + +impl AuthorizationStore for SqliteCatalogStore { + fn fetch_user_policies( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let pool = self.pool().clone(); + Box::pin(async move { + let rows: Vec<(String,)> = sqlx::query_as( + "SELECT policy_document FROM iam_policies \ + WHERE account_id = ? AND principal_type = 'user' AND principal_name = ?", + ) + .bind(&account_id) + .bind(&user_name) + .fetch_all(&pool) + .await + .map_err(|e| { + tracing::error!("fetch_user_policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(rows.into_iter().map(|(d,)| d).collect()) + }) + } + + fn fetch_user_group_policies( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let pool = self.pool().clone(); + Box::pin(async move { + let rows: Vec<(String,)> = sqlx::query_as( + "SELECT p.policy_document FROM iam_policies p \ + JOIN iam_group_members m ON m.account_id = p.account_id AND m.group_name = p.principal_name \ + WHERE p.account_id = ? AND p.principal_type = 'group' AND m.user_name = ?", + ) + .bind(&account_id) + .bind(&user_name) + .fetch_all(&pool) + .await + .map_err(|e| { + tracing::error!("fetch_user_group_policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(rows.into_iter().map(|(d,)| d).collect()) + }) + } + + fn fetch_user_boundary( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let pool = self.pool().clone(); + Box::pin(async move { + let row: Option<(String,)> = sqlx::query_as( + "SELECT policy_document FROM iam_permissions_boundaries \ + WHERE account_id = ? AND principal_type = 'user' AND principal_name = ?", + ) + .bind(&account_id) + .bind(&user_name) + .fetch_optional(&pool) + .await + .map_err(|e| { + tracing::error!("fetch_user_boundary: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(row.map(|(d,)| d)) + }) + } + + fn fetch_role_policies( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let pool = self.pool().clone(); + Box::pin(async move { + let rows: Vec<(String,)> = sqlx::query_as( + "SELECT policy_document FROM iam_policies \ + WHERE account_id = ? AND principal_type = 'role' AND principal_name = ?", + ) + .bind(&account_id) + .bind(&role_name) + .fetch_all(&pool) + .await + .map_err(|e| { + tracing::error!("fetch_role_policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(rows.into_iter().map(|(d,)| d).collect()) + }) + } + + fn fetch_role_boundary( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let pool = self.pool().clone(); + Box::pin(async move { + let row: Option<(String,)> = sqlx::query_as( + "SELECT policy_document FROM iam_permissions_boundaries \ + WHERE account_id = ? AND principal_type = 'role' AND principal_name = ?", + ) + .bind(&account_id) + .bind(&role_name) + .fetch_optional(&pool) + .await + .map_err(|e| { + tracing::error!("fetch_role_boundary: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(row.map(|(d,)| d)) + }) + } + + fn fetch_session_data( + &self, + account_id: &str, + role_name: &str, + session_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let session_name = session_name.to_owned(); + let pool = self.pool().clone(); + Box::pin(async move { + let row: Option<(Option, Option)> = sqlx::query_as( + "SELECT session_policy, session_tags FROM iam_sessions \ + WHERE account_id = ? AND role_name = ? AND session_name = ?", + ) + .bind(&account_id) + .bind(&role_name) + .bind(&session_name) + .fetch_optional(&pool) + .await + .map_err(|e| { + tracing::error!("fetch_session_data: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let Some((policy_json, tags_json)) = row else { + return Ok(None); + }; + + let session_tags: Vec<(String, String)> = tags_json + .as_deref() + .and_then(|s| serde_json::from_str::(s).ok()) + .and_then(|v| { + v.as_object().map(|obj| { + obj.iter() + .map(|(k, v)| (k.clone(), v.as_str().unwrap_or("").to_owned())) + .collect() + }) + }) + .unwrap_or_default(); + + Ok(Some(SessionData { + session_policy: policy_json, + session_tags, + })) + }) + } + + fn fetch_user_tags( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let pool = self.pool().clone(); + Box::pin(async move { + sqlx::query_as( + "SELECT tag_key, tag_value FROM iam_user_tags \ + WHERE account_id = ? AND user_name = ? ORDER BY tag_key", + ) + .bind(&account_id) + .bind(&user_name) + .fetch_all(&pool) + .await + .map_err(|e| { + tracing::error!("fetch_user_tags: {e}"); + OpError::Internal("Database error".to_owned()) + }) + }) + } + + fn fetch_role_tags( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let pool = self.pool().clone(); + Box::pin(async move { + sqlx::query_as( + "SELECT tag_key, tag_value FROM iam_role_tags \ + WHERE account_id = ? AND role_name = ? ORDER BY tag_key", + ) + .bind(&account_id) + .bind(&role_name) + .fetch_all(&pool) + .await + .map_err(|e| { + tracing::error!("fetch_role_tags: {e}"); + OpError::Internal("Database error".to_owned()) + }) + }) + } + + fn fetch_resource_tags(&self, arn: &str) -> BoxFuture<'_, OpResult>> { + let arn = arn.to_owned(); + let pool = self.pool().clone(); + Box::pin(async move { + sqlx::query_as( + "SELECT tag_key, tag_value FROM tags WHERE resource_arn = ? ORDER BY tag_key", + ) + .bind(&arn) + .fetch_all(&pool) + .await + .map_err(|e| { + tracing::error!("fetch_resource_tags: {e}"); + OpError::Internal("Database error".to_owned()) + }) + }) + } +} diff --git a/crates/storage-sqlite/src/backup_engine.rs b/crates/storage-sqlite/src/backup_engine.rs new file mode 100644 index 00000000..e542d235 --- /dev/null +++ b/crates/storage-sqlite/src/backup_engine.rs @@ -0,0 +1,545 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Backup and point-in-time recovery implementation for the SQLite backend. + +use extenddb_core::types::{ + BackupDescription, BackupDetails, BackupSummary, ContinuousBackupsDescription, + PointInTimeRecoveryDescription, SourceTableDetails, TableDescription, TableKeyInfo, +}; +use extenddb_storage::BackupEngine; +use extenddb_storage::TableEngine; +use extenddb_storage::error::StorageError; +use futures::future::BoxFuture; + +use crate::data::{data_table_name, upsert_item_in_tx}; +use crate::engine::SqliteEngine; +use crate::sqlite_util::parse_timestamp; + +fn epoch_millis() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() +} + +#[allow(clippy::cast_precision_loss)] +fn sqlite_timestamp_to_epoch(s: &str) -> f64 { + parse_timestamp(s) + .map(|dt| dt.unix_timestamp() as f64) + .unwrap_or(0.0) +} + +impl BackupEngine for SqliteEngine { + fn create_backup( + &self, + account_id: &str, + table_name: &str, + backup_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + let backup_name = backup_name.to_string(); + Box::pin(async move { + let row: Option<(String, String, String, String, String, i64, i64)> = sqlx::query_as( + "SELECT table_id, table_arn, key_schema, attribute_definitions, \ + billing_mode, table_size_bytes, item_count \ + FROM tables WHERE account_id = ? AND table_name = ? AND table_status = 'ACTIVE'", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (table_id, _table_arn, key_schema_text, attr_defs_text, billing_mode, size_bytes, _item_count) = + row.ok_or_else(|| StorageError::TableNotFound(format!("Table not found: {table_name}")))?; + + let backup_arn = format!( + "arn:aws:dynamodb:{region}:{account_id}:table/{table_name}/backup/{ts}", + region = self.region, + ts = epoch_millis() + ); + + let ddb_table = data_table_name(&table_id); + let items: Vec<(serde_json::Value,)> = + sqlx::query_as(&format!("SELECT item_data FROM {ddb_table}")) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + #[allow(clippy::cast_possible_wrap)] + let actual_count = items.len() as i64; + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + sqlx::query( + "INSERT INTO backups (backup_arn, backup_name, table_id, table_name, account_id, \ + backup_status, backup_size_bytes, item_count, key_schema, attribute_definitions, \ + billing_mode) \ + VALUES (?, ?, ?, ?, ?, 'AVAILABLE', ?, ?, ?, ?, ?)", + ) + .bind(&backup_arn) + .bind(&backup_name) + .bind(&table_id) + .bind(&table_name) + .bind(&account_id) + .bind(size_bytes) + .bind(actual_count) + .bind(&key_schema_text) + .bind(&attr_defs_text) + .bind(&billing_mode) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + for (item_data,) in &items { + let item_text = serde_json::to_string(item_data) + .map_err(|e| StorageError::Internal(e.to_string()))?; + sqlx::query( + "INSERT INTO backup_items (backup_arn, pk, sk, item_data) VALUES (?, '', NULL, ?)", + ) + .bind(&backup_arn) + .bind(&item_text) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + let created_at_row: (String,) = + sqlx::query_as("SELECT created_at FROM backups WHERE backup_arn = ?") + .bind(&backup_arn) + .fetch_one(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(BackupDetails { + backup_arn, + backup_name, + backup_status: "AVAILABLE".to_owned(), + backup_type: "USER".to_owned(), + backup_size_bytes: size_bytes, + backup_creation_date_time: sqlite_timestamp_to_epoch(&created_at_row.0), + }) + }) + } + + fn describe_backup( + &self, + backup_arn: &str, + ) -> BoxFuture<'_, Result> { + let backup_arn = backup_arn.to_string(); + Box::pin(async move { + #[allow(clippy::type_complexity)] + let row: Option<( + String, + String, + String, + String, + String, + i64, + i64, + String, + String, + String, + String, + String, + )> = sqlx::query_as( + "SELECT b.backup_name, b.backup_status, b.table_id, b.table_name, b.account_id, \ + b.backup_size_bytes, b.item_count, b.key_schema, b.billing_mode, \ + COALESCE(t.table_arn, \ + 'arn:aws:dynamodb:' || ? || ':' || b.account_id || ':table/' || b.table_name), \ + b.created_at, \ + COALESCE(t.creation_date_time, b.created_at) \ + FROM backups b \ + LEFT JOIN tables t ON t.table_id = b.table_id \ + WHERE b.backup_arn = ?", + ) + .bind(&self.region) + .bind(&backup_arn) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let ( + name, + status, + table_id, + table_name, + _account_id, + size, + count, + ks_text, + billing, + table_arn, + backup_created_at, + table_created_at, + ) = row.ok_or_else(|| StorageError::Validation(format!("Backup not found: {backup_arn}")))?; + + let key_schema: Vec = + serde_json::from_str(&ks_text) + .map_err(|e| StorageError::Internal(format!("Parse key schema: {e}")))?; + + Ok(BackupDescription { + backup_details: BackupDetails { + backup_arn: backup_arn.to_owned(), + backup_name: name, + backup_status: status, + backup_type: "USER".to_owned(), + backup_size_bytes: size, + backup_creation_date_time: sqlite_timestamp_to_epoch(&backup_created_at), + }, + source_table_details: SourceTableDetails { + table_name, + table_id, + table_arn, + key_schema, + item_count: count, + table_size_bytes: size, + billing_mode: Some(billing), + table_creation_date_time: sqlite_timestamp_to_epoch(&table_created_at), + }, + }) + }) + } + + fn list_backups( + &self, + account_id: &str, + table_name: Option<&str>, + ) -> BoxFuture<'_, Result, StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.map(|s| s.to_string()); + Box::pin(async move { + let rows: Vec<(String, String, String, String, i64, String, String)> = + if let Some(tn) = table_name { + sqlx::query_as( + "SELECT b.backup_arn, b.backup_name, b.table_name, b.backup_status, \ + b.backup_size_bytes, \ + COALESCE(t.table_arn, \ + 'arn:aws:dynamodb:' || ? || ':' || b.account_id || ':table/' || b.table_name), \ + b.created_at \ + FROM backups b \ + LEFT JOIN tables t ON t.table_id = b.table_id \ + WHERE b.account_id = ? AND b.table_name = ? AND b.backup_status != 'DELETED' \ + ORDER BY b.created_at DESC", + ) + .bind(&self.region) + .bind(&account_id) + .bind(tn) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + } else { + sqlx::query_as( + "SELECT b.backup_arn, b.backup_name, b.table_name, b.backup_status, \ + b.backup_size_bytes, \ + COALESCE(t.table_arn, \ + 'arn:aws:dynamodb:' || ? || ':' || b.account_id || ':table/' || b.table_name), \ + b.created_at \ + FROM backups b \ + LEFT JOIN tables t ON t.table_id = b.table_id \ + WHERE b.account_id = ? AND b.backup_status != 'DELETED' \ + ORDER BY b.created_at DESC", + ) + .bind(&self.region) + .bind(&account_id) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + }; + + Ok(rows + .into_iter() + .map(|(arn, name, tn, status, size, table_arn, created_at)| BackupSummary { + backup_arn: arn, + backup_name: name, + table_name: tn, + table_arn, + backup_status: status, + backup_type: "USER".to_owned(), + backup_size_bytes: size, + backup_creation_date_time: sqlite_timestamp_to_epoch(&created_at), + }) + .collect()) + }) + } + + fn delete_backup( + &self, + backup_arn: &str, + ) -> BoxFuture<'_, Result> { + let backup_arn = backup_arn.to_string(); + Box::pin(async move { + let desc = self.describe_backup(&backup_arn).await?; + + sqlx::query("DELETE FROM backup_items WHERE backup_arn = ?") + .bind(&backup_arn) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + sqlx::query("UPDATE backups SET backup_status = 'DELETED' WHERE backup_arn = ?") + .bind(&backup_arn) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(BackupDescription { + backup_details: BackupDetails { + backup_status: "DELETED".to_owned(), + ..desc.backup_details + }, + source_table_details: desc.source_table_details, + }) + }) + } + + fn restore_table_from_backup( + &self, + account_id: &str, + target_table_name: &str, + backup_arn: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let target_table_name = target_table_name.to_string(); + let backup_arn = backup_arn.to_string(); + Box::pin(async move { + let backup_row: Option<(String, String, String, String)> = sqlx::query_as( + "SELECT table_name, key_schema, attribute_definitions, billing_mode \ + FROM backups WHERE backup_arn = ? AND backup_status = 'AVAILABLE'", + ) + .bind(&backup_arn) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (_orig_table, ks_text, ad_text, billing) = backup_row + .ok_or_else(|| StorageError::Validation(format!("Backup not found: {backup_arn}")))?; + + let key_schema: Vec = + serde_json::from_str(&ks_text) + .map_err(|e| StorageError::Internal(format!("Parse key schema: {e}")))?; + let attr_defs: Vec = + serde_json::from_str(&ad_text) + .map_err(|e| StorageError::Internal(format!("Parse attr defs: {e}")))?; + + let billing_mode = if billing == "PAY_PER_REQUEST" { + Some(extenddb_core::types::BillingMode::PayPerRequest) + } else { + Some(extenddb_core::types::BillingMode::Provisioned) + }; + + let create_input = extenddb_core::types::CreateTableInput { + table_name: target_table_name.to_owned(), + key_schema: key_schema.clone(), + attribute_definitions: attr_defs.clone(), + billing_mode, + provisioned_throughput: Some(extenddb_core::types::ProvisionedThroughput { + read_capacity_units: 5, + write_capacity_units: 5, + }), + global_secondary_indexes: None, + local_secondary_indexes: None, + stream_specification: None, + tags: None, + deletion_protection_enabled: None, + sse_specification: None, + table_class: None, + }; + + let desc = self.create_table(&account_id, create_input).await?; + let new_table_id = desc.table_id.clone(); + + let key_info = TableKeyInfo { + table_name: target_table_name.clone(), + account_id: account_id.clone(), + table_id: new_table_id.clone(), + key_schema, + attribute_definitions: attr_defs, + has_lsi: false, + stream_specification: None, + }; + + let items: Vec<(String,)> = + sqlx::query_as("SELECT item_data FROM backup_items WHERE backup_arn = ?") + .bind(&backup_arn) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + #[allow(clippy::cast_possible_wrap)] + let item_count = items.len() as i64; + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + for (item_json_str,) in &items { + let item: extenddb_core::types::Item = serde_json::from_str(item_json_str) + .map_err(|e| StorageError::Internal(e.to_string()))?; + upsert_item_in_tx(&mut tx, &key_info, &item) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + sqlx::query( + "UPDATE tables SET item_count = ? WHERE account_id = ? AND table_name = ?", + ) + .bind(item_count) + .bind(&account_id) + .bind(&target_table_name) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + sqlx::query( + "UPDATE tables SET table_status = 'ACTIVE', status_transition_at = NULL \ + WHERE account_id = ? AND table_name = ?", + ) + .bind(&account_id) + .bind(&target_table_name) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(desc) + }) + } + + fn describe_continuous_backups( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM tables WHERE account_id = ? AND table_name = ?)", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_one(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if !exists { + return Err(StorageError::TableNotFound(format!( + "Table not found: {table_name}" + ))); + } + + let pitr_row: Option<(bool,)> = sqlx::query_as( + "SELECT pitr_enabled FROM continuous_backups \ + WHERE account_id = ? AND table_name = ?", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let pitr_enabled = pitr_row.is_some_and(|r| r.0); + + #[allow(clippy::cast_precision_loss)] + let now_epoch = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as f64; + + Ok(ContinuousBackupsDescription { + continuous_backups_status: "ENABLED".to_owned(), + point_in_time_recovery_description: Some(PointInTimeRecoveryDescription { + point_in_time_recovery_status: if pitr_enabled { + "ENABLED".to_owned() + } else { + "DISABLED".to_owned() + }, + earliest_restorable_date_time: if pitr_enabled { + Some(now_epoch - 35.0 * 24.0 * 3600.0) + } else { + None + }, + latest_restorable_date_time: if pitr_enabled { Some(now_epoch) } else { None }, + }), + }) + }) + } + + fn update_continuous_backups( + &self, + account_id: &str, + table_name: &str, + pitr_enabled: bool, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM tables WHERE account_id = ? AND table_name = ?)", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_one(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if !exists { + return Err(StorageError::TableNotFound(format!( + "Table not found: {table_name}" + ))); + } + + sqlx::query( + "INSERT INTO continuous_backups (account_id, table_name, pitr_enabled) \ + VALUES (?, ?, ?) \ + ON CONFLICT (account_id, table_name) DO UPDATE SET pitr_enabled = excluded.pitr_enabled", + ) + .bind(&account_id) + .bind(&table_name) + .bind(pitr_enabled) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + self.describe_continuous_backups(&account_id, &table_name) + .await + }) + } + + fn restore_table_to_point_in_time( + &self, + account_id: &str, + source_table_name: &str, + target_table_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let source_table_name = source_table_name.to_string(); + let target_table_name = target_table_name.to_string(); + Box::pin(async move { + let backup = self + .create_backup(&account_id, &source_table_name, "__pitr_restore__") + .await?; + let desc = self + .restore_table_from_backup(&account_id, &target_table_name, &backup.backup_arn) + .await?; + let _ = self.delete_backup(&backup.backup_arn).await; + Ok(desc) + }) + } +} diff --git a/crates/storage-sqlite/src/bootstrapper.rs b/crates/storage-sqlite/src/bootstrapper.rs new file mode 100644 index 00000000..e2b435bb --- /dev/null +++ b/crates/storage-sqlite/src/bootstrapper.rs @@ -0,0 +1,369 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! SQLite implementation of `Bootstrapper`. +//! +//! SQLite has no server concept — initialization simply creates the database +//! file and runs schema migrations. No user provisioning is needed. + +use async_trait::async_trait; +use extenddb_storage::bootstrapper::{AdminBootstrapResult, Bootstrapper}; +use extenddb_storage::management_store::{OpError, OpResult}; +use sqlx::SqlitePool; +use sqlx::sqlite::SqlitePoolOptions; + +use crate::engine::CATALOG_VERSION; +use crate::migrations; + +/// SQLite bootstrapper. +pub struct SqliteBootstrapper { + pub(crate) path: String, +} + +impl SqliteBootstrapper { + pub fn new(path: String) -> Self { + Self { path } + } + + fn connection_string(&self) -> String { + if self.path == ":memory:" { + "sqlite::memory:".to_owned() + } else { + format!("sqlite://{}?mode=rwc", self.path) + } + } + + async fn pool(&self) -> OpResult { + let conn = self.connection_string(); + let pool = SqlitePoolOptions::new() + .max_connections(1) + .after_connect(|conn, _| { + Box::pin(async move { + use sqlx::Executor; + conn.execute("PRAGMA journal_mode=WAL").await?; + conn.execute("PRAGMA foreign_keys=ON").await?; + conn.execute("PRAGMA synchronous=NORMAL").await?; + Ok(()) + }) + }) + .connect(&conn) + .await + .map_err(|e| OpError::Internal(format!("Cannot open SQLite database: {e}")))?; + Ok(pool) + } +} + +#[async_trait] +impl Bootstrapper for SqliteBootstrapper { + async fn ensure_app_user(&self) -> OpResult<()> { + // SQLite has no user concept. + Ok(()) + } + + async fn grant_app_role_to_admin(&self) -> OpResult<()> { + // SQLite has no role concept. + Ok(()) + } + + async fn create_catalog_db(&self) -> OpResult<()> { + // SQLite creates the file on first connect. + println!("--- SQLite database: {}", self.path); + let exists = std::path::Path::new(&self.path).exists(); + if exists && self.path != ":memory:" { + return Err(OpError::AlreadyExists(format!( + "SQLite database '{}' already exists. Run 'destroy' first, then re-run 'init'.", + self.path + ))); + } + Ok(()) + } + + async fn create_data_db(&self) -> OpResult<()> { + // SQLite: catalog and data in same file. No-op. + Ok(()) + } + + async fn run_catalog_migrations(&self) -> OpResult<()> { + let pool = self.pool().await?; + migrations::run_migrations(&pool).await + } + + async fn run_data_migrations(&self) -> OpResult<()> { + // SQLite: catalog and data schema are in the same migration file. No-op. + Ok(()) + } + + async fn record_data_connection(&self) -> OpResult<()> { + // SQLite: no separate data database. Record path as data_database_name for info. + let pool = self.pool().await?; + sqlx::query( + "INSERT OR REPLACE INTO settings (key, value) VALUES ('data_database_name', ?)", + ) + .bind(&self.path) + .execute(&pool) + .await + .map_err(|e| OpError::Internal(format!("Record data db name: {e}")))?; + Ok(()) + } + + async fn bootstrap_encryption_key(&self) -> OpResult<()> { + use aes_gcm::KeyInit; + use base64::Engine; + + let pool = self.pool().await?; + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM settings WHERE key = 'encryption_key')", + ) + .fetch_one(&pool) + .await + .map_err(|e| OpError::Internal(format!("Check encryption key: {e}")))?; + + if exists { + println!("--- Encryption key already exists, skipping."); + return Ok(()); + } + + println!("--- Generating AES-256-GCM encryption key..."); + let key = aes_gcm::Aes256Gcm::generate_key(&mut aes_gcm::aead::OsRng); + let key_b64 = base64::engine::general_purpose::STANDARD.encode(key); + + sqlx::query( + "INSERT OR IGNORE INTO settings (key, value) VALUES ('encryption_key', ?)", + ) + .bind(&key_b64) + .execute(&pool) + .await + .map_err(|e| OpError::Internal(format!("Store encryption key: {e}")))?; + + println!(" Encryption key stored."); + Ok(()) + } + + async fn bootstrap_default_account(&self) -> OpResult<()> { + let pool = self.pool().await?; + let exists: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM accounts)") + .fetch_one(&pool) + .await + .map_err(|e| OpError::Internal(format!("Check accounts: {e}")))?; + + if exists { + println!("--- Default account already exists, skipping."); + return Ok(()); + } + + let account_id = generate_account_id(); + println!("--- Creating default account '{account_id}'..."); + sqlx::query( + "INSERT OR IGNORE INTO accounts (account_id, account_name) VALUES (?, ?)", + ) + .bind(&account_id) + .bind("default") + .execute(&pool) + .await + .map_err(|e| OpError::Internal(format!("Create account: {e}")))?; + + println!(" Account ID: {account_id}"); + Ok(()) + } + + async fn bootstrap_admin_user( + &self, + env_user: Option<&str>, + env_password: Option<&str>, + ) -> OpResult { + let pool = self.pool().await?; + let admin_name = env_user.filter(|s| !s.is_empty()).unwrap_or("admin"); + + let exists: bool = + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM admin_users WHERE admin_name = ?)") + .bind(admin_name) + .fetch_one(&pool) + .await + .map_err(|e| OpError::Internal(format!("Check admin user: {e}")))?; + + if exists { + println!("--- Admin user '{admin_name}' already exists, skipping."); + return Ok(AdminBootstrapResult { + username: admin_name.to_owned(), + generated_password: None, + already_existed: true, + from_env: false, + }); + } + + println!("--- Creating admin user '{admin_name}'..."); + let (password, from_env) = match env_password { + Some(p) if !p.is_empty() => (p.to_owned(), true), + _ => (generate_random_password(), false), + }; + let pw_clone = password.clone(); + let hash = + tokio::task::spawn_blocking(move || bcrypt::hash(pw_clone, bcrypt::DEFAULT_COST)) + .await + .map_err(|e| OpError::Internal(format!("bcrypt hash task failed: {e}")))? + .map_err(|e| OpError::Internal(format!("bcrypt hash failed: {e}")))?; + + sqlx::query( + "INSERT OR IGNORE INTO admin_users (admin_name, password_hash) VALUES (?, ?)", + ) + .bind(admin_name) + .bind(&hash) + .execute(&pool) + .await + .map_err(|e| OpError::Internal(format!("Create admin user: {e}")))?; + + Ok(AdminBootstrapResult { + username: admin_name.to_owned(), + generated_password: if from_env { None } else { Some(password) }, + already_existed: false, + from_env, + }) + } + + async fn is_catalog_initialized(&self) -> OpResult { + // The file must exist and the settings table must be present. + if self.path != ":memory:" && !std::path::Path::new(&self.path).exists() { + return Ok(false); + } + let pool = match self.pool().await { + Ok(p) => p, + Err(_) => return Ok(false), + }; + migrations::table_exists(&pool, "settings").await + } + + async fn list_table_names(&self) -> OpResult> { + let pool = match self.pool().await { + Ok(p) => p, + Err(_) => return Ok(Vec::new()), + }; + let tables: Vec<(String,)> = + sqlx::query_as("SELECT table_name FROM tables ORDER BY table_name") + .fetch_all(&pool) + .await + .unwrap_or_default(); + Ok(tables.into_iter().map(|(n,)| n).collect()) + } + + async fn get_data_db_name(&self) -> OpResult> { + let pool = match self.pool().await { + Ok(p) => p, + Err(_) => return Ok(None), + }; + let row = sqlx::query_as::<_, (String,)>( + "SELECT value FROM settings WHERE key = 'data_database_name'", + ) + .fetch_optional(&pool) + .await + .unwrap_or(None); + Ok(row.map(|(v,)| v)) + } + + async fn drop_databases(&self, _data_db: &str) -> OpResult<()> { + if self.path != ":memory:" { + println!("--- Removing SQLite database file '{}'...", self.path); + if std::path::Path::new(&self.path).exists() { + std::fs::remove_file(&self.path) + .map_err(|e| OpError::Internal(format!("Remove database file: {e}")))?; + } + // Also remove WAL and SHM files if present. + let wal = format!("{}-wal", self.path); + let shm = format!("{}-shm", self.path); + let _ = std::fs::remove_file(&wal); + let _ = std::fs::remove_file(&shm); + } + Ok(()) + } + + async fn read_catalog_version(&self) -> OpResult> { + let pool = match self.pool().await { + Ok(p) => p, + Err(_) => return Ok(None), + }; + + if !migrations::table_exists(&pool, "settings").await? { + return Ok(None); + } + + let row = sqlx::query_as::<_, (String,)>( + "SELECT value FROM settings WHERE key = 'catalog_version'", + ) + .fetch_optional(&pool) + .await + .map_err(|e| OpError::Internal(format!("Read catalog version: {e}")))?; + + Ok(row.map(|(v,)| v)) + } + + fn expected_catalog_version(&self) -> String { + CATALOG_VERSION.to_string() + } + + fn catalog_database_name(&self) -> String { + self.path.clone() + } + + fn endpoint_info(&self) -> String { + format!("sqlite:{}", self.path) + } + + fn catalog_connection_url(&self) -> String { + self.connection_string() + } +} + +// ── Helpers ──────────────────────────────────────────────────────────── + +fn generate_account_id() -> String { + use rand::Rng; + let mut rng = rand::rng(); + let id: u64 = rng.random_range(100_000_000_000..1_000_000_000_000); + id.to_string() +} + +fn generate_random_password() -> String { + use rand::Rng; + const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + let mut rng = rand::rng(); + (0..24) + .map(|_| CHARSET[rng.random_range(0..CHARSET.len())] as char) + .collect() +} + +impl SqliteBootstrapper { + pub async fn from_config( + config_path: &str, + cli_args: &[String], + ) -> Result { + use extenddb_storage::error::StorageError; + + let sqlite_path = extract_arg(cli_args, "--sqlite-path"); + + let path = if let Some(p) = sqlite_path { + p + } else if std::path::Path::new(config_path).exists() { + println!("--- Loading defaults from {}", config_path); + + let config_content = std::fs::read_to_string(config_path) + .map_err(|e| StorageError::Internal(format!("Failed to read config: {e}")))?; + let app_config: toml::Value = toml::from_str(&config_content) + .map_err(|e| StorageError::Internal(format!("Failed to parse config: {e}")))?; + + app_config + .get("storage") + .and_then(|s| s.get("sqlite")) + .and_then(|p| p.get("path")) + .and_then(|c| c.as_str()) + .unwrap_or("extenddb.sqlite") + .to_owned() + } else { + "extenddb.sqlite".to_owned() + }; + + Ok(Self::new(path)) + } +} + +fn extract_arg(args: &[String], flag: &str) -> Option { + args.windows(2).find(|w| w[0] == flag).map(|w| w[1].clone()) +} diff --git a/crates/storage-sqlite/src/catalog_store.rs b/crates/storage-sqlite/src/catalog_store.rs new file mode 100644 index 00000000..2e0bda47 --- /dev/null +++ b/crates/storage-sqlite/src/catalog_store.rs @@ -0,0 +1,410 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! SQLite implementations of `SettingsStore`, `MetricsStore`, `RateLimitStore`, +//! `DiagnosticsStore`, and `CatalogStore`. + +use std::sync::Arc; + +use extenddb_storage::management_store::{MetricsRow, OpError, OpResult}; +use futures::future::BoxFuture; +use sqlx::SqlitePool; + +/// SQLite-backed catalog store for settings, metrics, and rate limiting. +pub struct SqliteCatalogStore { + pool: SqlitePool, + encryption_key: Option>, +} + +impl SqliteCatalogStore { + pub fn new(pool: SqlitePool) -> Self { + Self { + pool, + encryption_key: None, + } + } + + pub fn with_encryption_key(pool: SqlitePool, encryption_key: String) -> Self { + Self { + pool, + encryption_key: Some(Arc::from(encryption_key.as_str())), + } + } + + pub fn pool(&self) -> &SqlitePool { + &self.pool + } + + pub fn encryption_key(&self) -> Option<&Arc> { + self.encryption_key.as_ref() + } +} + +// ── SettingsStore ────────────────────────────────────────────────────── + +impl extenddb_storage::management_store::SettingsStore for SqliteCatalogStore { + fn get_setting(&self, key: &str) -> BoxFuture<'_, OpResult>> { + let key = key.to_string(); + let pool = self.pool.clone(); + Box::pin(async move { + let row: Option<(String,)> = + sqlx::query_as("SELECT value FROM settings WHERE key = ?") + .bind(&key) + .fetch_optional(&pool) + .await + .map_err(|e| { + tracing::error!("get_setting: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(row.map(|(v,)| v)) + }) + } + + fn set_setting(&self, key: &str, value: &str) -> BoxFuture<'_, OpResult<()>> { + let key = key.to_string(); + let value = value.to_string(); + let pool = self.pool.clone(); + Box::pin(async move { + sqlx::query( + "INSERT INTO settings (key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ) + .bind(&key) + .bind(&value) + .execute(&pool) + .await + .map_err(|e| { + tracing::error!("set_setting: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(()) + }) + } + + fn list_settings(&self) -> BoxFuture<'_, OpResult>> { + let pool = self.pool.clone(); + Box::pin(async move { + sqlx::query_as("SELECT key, value FROM settings ORDER BY key") + .fetch_all(&pool) + .await + .map_err(|e| { + tracing::error!("list_settings: {e}"); + OpError::Internal("Database error".to_owned()) + }) + }) + } + + fn cached_encryption_key(&self) -> Option { + self.encryption_key.as_ref().map(|k| k.to_string()) + } +} + +// ── DiagnosticsStore ─────────────────────────────────────────────────── + +impl extenddb_storage::diagnostics::DiagnosticsStore for SqliteCatalogStore { + fn count_tables( + &self, + ) -> BoxFuture<'_, extenddb_storage::diagnostics::DiagResult> { + let pool = self.pool.clone(); + Box::pin(async move { + let (count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM tables") + .fetch_one(&pool) + .await + .map_err(|e| { + extenddb_storage::diagnostics::DiagError::QueryFailed(e.to_string()) + })?; + Ok(count) + }) + } + + fn count_indexes( + &self, + ) -> BoxFuture<'_, extenddb_storage::diagnostics::DiagResult> { + let pool = self.pool.clone(); + Box::pin(async move { + let (count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM indexes") + .fetch_one(&pool) + .await + .map_err(|e| { + extenddb_storage::diagnostics::DiagError::QueryFailed(e.to_string()) + })?; + Ok(count) + }) + } + + fn test_data_database_connection( + &self, + ) -> BoxFuture<'_, extenddb_storage::diagnostics::DiagResult> { + let pool = self.pool.clone(); + Box::pin(async move { + // For SQLite, catalog and data are in the same database. + let name_row: Option<(String,)> = + sqlx::query_as("SELECT value FROM settings WHERE key = 'data_database_name'") + .fetch_optional(&pool) + .await + .map_err(|e| { + extenddb_storage::diagnostics::DiagError::QueryFailed(e.to_string()) + })?; + Ok(name_row + .map(|(n,)| n) + .unwrap_or_else(|| "sqlite (embedded)".to_owned())) + }) + } +} + +// ── MetricsStore ─────────────────────────────────────────────────────── + +impl extenddb_storage::management_store::MetricsStore for SqliteCatalogStore { + fn insert_metrics(&self, rows: &[MetricsRow]) -> BoxFuture<'_, OpResult<()>> { + let rows = rows.to_vec(); + let pool = self.pool.clone(); + Box::pin(async move { + for row in &rows { + let bucket = crate::sqlite_util::format_timestamp(row.bucket); + let result = sqlx::query( + "INSERT INTO metrics \ + (bucket, metric, table_name, index_name, operation, sum, count, min, max) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) \ + ON CONFLICT(bucket, metric, table_name, index_name, operation) \ + DO UPDATE SET \ + sum = metrics.sum + excluded.sum, \ + count = metrics.count + excluded.count, \ + min = MIN(metrics.min, excluded.min), \ + max = MAX(metrics.max, excluded.max)", + ) + .bind(&bucket) + .bind(&row.metric) + .bind(row.table_name.as_deref().unwrap_or("")) + .bind(row.index_name.as_deref().unwrap_or("")) + .bind(row.operation.as_deref().unwrap_or("")) + .bind(row.sum) + .bind(row.count) + .bind(row.min) + .bind(row.max) + .execute(&pool) + .await; + if let Err(e) = result { + tracing::warn!("Failed to upsert metrics row: {e}"); + } + } + Ok(()) + }) + } + + fn query_metrics( + &self, + start: time::OffsetDateTime, + end: time::OffsetDateTime, + table_name: Option<&str>, + metric: Option<&str>, + ) -> BoxFuture<'_, OpResult>> { + let table_name = table_name.map(|s| s.to_owned()); + let metric = metric.map(|s| s.to_owned()); + let start_str = crate::sqlite_util::format_timestamp(start); + let end_str = crate::sqlite_util::format_timestamp(end); + let pool = self.pool.clone(); + Box::pin(async move { + let mut sql = String::from( + "SELECT bucket, metric, table_name, index_name, operation, \ + sum, count, min, max \ + FROM metrics WHERE bucket >= ? AND bucket <= ?", + ); + + let table_filter = table_name.as_deref().filter(|s| !s.is_empty()); + if table_filter.is_some() { + sql.push_str(" AND table_name = ?"); + } + if metric.is_some() { + sql.push_str(" AND metric = ?"); + } + sql.push_str(" ORDER BY bucket"); + + let mut q = sqlx::query_as::<_, DbMetricsRow>(&sql) + .bind(&start_str) + .bind(&end_str); + if let Some(tn) = table_filter { + q = q.bind(tn); + } + if let Some(mn) = metric.as_deref() { + q = q.bind(mn); + } + + let rows = q.fetch_all(&pool).await.map_err(|e| { + tracing::warn!("query_metrics: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + Ok(rows + .into_iter() + .filter_map(|r| { + let bucket = crate::sqlite_util::parse_timestamp(&r.bucket).ok()?; + Some(MetricsRow { + bucket, + metric: r.metric, + table_name: if r.table_name.is_empty() { + None + } else { + Some(r.table_name) + }, + index_name: if r.index_name.is_empty() { + None + } else { + Some(r.index_name) + }, + operation: if r.operation.is_empty() { + None + } else { + Some(r.operation) + }, + sum: r.sum, + count: r.count, + min: r.min, + max: r.max, + }) + }) + .collect()) + }) + } + + fn prune_metrics(&self, retention: std::time::Duration) -> BoxFuture<'_, OpResult<()>> { + let pool = self.pool.clone(); + Box::pin(async move { + #[allow(clippy::cast_possible_wrap)] + let cutoff = time::OffsetDateTime::now_utc() + - time::Duration::seconds(retention.as_secs() as i64); + let cutoff_str = crate::sqlite_util::format_timestamp(cutoff); + sqlx::query("DELETE FROM metrics WHERE bucket < ?") + .bind(&cutoff_str) + .execute(&pool) + .await + .map_err(|e| { + tracing::warn!("prune_metrics: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(()) + }) + } +} + +#[derive(sqlx::FromRow)] +struct DbMetricsRow { + bucket: String, + metric: String, + table_name: String, + index_name: String, + operation: String, + sum: f64, + count: i64, + min: f64, + max: f64, +} + +// ── RateLimitStore ───────────────────────────────────────────────────── + +impl extenddb_storage::management_store::RateLimitStore for SqliteCatalogStore { + fn count_principal_failures( + &self, + principal: &str, + window_seconds: i64, + ) -> BoxFuture<'_, OpResult> { + let principal = principal.to_owned(); + let pool = self.pool.clone(); + Box::pin(async move { + let cutoff = time::OffsetDateTime::now_utc() + - time::Duration::seconds(window_seconds); + let cutoff_str = crate::sqlite_util::format_timestamp(cutoff); + let row: (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM login_attempts \ + WHERE principal = ? AND success = 0 AND attempted_at > ?", + ) + .bind(&principal) + .bind(&cutoff_str) + .fetch_one(&pool) + .await + .map_err(|e| { + tracing::error!("count_principal_failures: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(row.0) + }) + } + + fn count_ip_failures( + &self, + source_ip: &str, + window_seconds: i64, + ) -> BoxFuture<'_, OpResult> { + let source_ip = source_ip.to_owned(); + let pool = self.pool.clone(); + Box::pin(async move { + let cutoff = time::OffsetDateTime::now_utc() + - time::Duration::seconds(window_seconds); + let cutoff_str = crate::sqlite_util::format_timestamp(cutoff); + let row: (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM login_attempts \ + WHERE source_ip = ? AND success = 0 AND attempted_at > ?", + ) + .bind(&source_ip) + .bind(&cutoff_str) + .fetch_one(&pool) + .await + .map_err(|e| { + tracing::error!("count_ip_failures: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(row.0) + }) + } + + fn record_failed_login(&self, principal: &str, source_ip: Option<&str>) -> BoxFuture<'_, ()> { + let principal = principal.to_owned(); + let source_ip = source_ip.map(|s| s.to_owned()); + let pool = self.pool.clone(); + Box::pin(async move { + let now = crate::sqlite_util::format_timestamp(time::OffsetDateTime::now_utc()); + let result = sqlx::query( + "INSERT INTO login_attempts (principal, attempted_at, success, source_ip) \ + VALUES (?, ?, 0, ?)", + ) + .bind(&principal) + .bind(&now) + .bind(source_ip.as_deref()) + .execute(&pool) + .await; + if let Err(e) = result { + tracing::error!("Failed to record login attempt: {e}"); + } + }) + } + + fn cleanup_old_attempts(&self, max_age_seconds: i64) -> BoxFuture<'_, ()> { + let pool = self.pool.clone(); + Box::pin(async move { + let cutoff = time::OffsetDateTime::now_utc() + - time::Duration::seconds(max_age_seconds); + let cutoff_str = crate::sqlite_util::format_timestamp(cutoff); + let result = sqlx::query("DELETE FROM login_attempts WHERE attempted_at < ?") + .bind(&cutoff_str) + .execute(&pool) + .await; + match result { + Ok(r) => { + if r.rows_affected() > 0 { + tracing::debug!( + "Cleaned up {} old login attempt records", + r.rows_affected() + ); + } + } + Err(e) => tracing::error!("Login attempt cleanup failed: {e}"), + } + }) + } +} + +// Implement CatalogStore supertrait +impl extenddb_storage::CatalogStore for SqliteCatalogStore { + fn cached_encryption_key(&self) -> Option { + self.encryption_key.as_ref().map(|arc| arc.to_string()) + } +} diff --git a/crates/storage-sqlite/src/config.rs b/crates/storage-sqlite/src/config.rs new file mode 100644 index 00000000..6ea82808 --- /dev/null +++ b/crates/storage-sqlite/src/config.rs @@ -0,0 +1,65 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! SQLite connection configuration. + +use serde::Deserialize; + +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SqliteStorageConfig { + /// Path to the SQLite database file. + /// Use `:memory:` for an in-memory database. + #[serde(default = "default_path")] + pub path: String, + #[serde(default = "default_pool_size")] + pub pool_size: u32, +} + +impl Default for SqliteStorageConfig { + fn default() -> Self { + Self { + path: default_path(), + pool_size: default_pool_size(), + } + } +} + +fn default_path() -> String { + "extenddb.sqlite".to_owned() +} + +fn default_pool_size() -> u32 { + 10 +} + +impl SqliteStorageConfig { + /// Build the sqlx connection string from the path. + pub fn connection_string(&self) -> String { + if self.path == ":memory:" { + "sqlite::memory:".to_owned() + } else { + format!("sqlite://{}?mode=rwc", self.path) + } + } +} + +// ── StorageConfig trait implementation ──────────────────────────────── + +impl extenddb_storage::config::StorageConfig for SqliteStorageConfig { + fn connection_config(&self) -> &str { + &self.path + } + + fn max_connections(&self) -> u32 { + self.pool_size + } + + fn max_catalog_connections(&self) -> u32 { + self.pool_size + } + + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} diff --git a/crates/storage-sqlite/src/create_table.rs b/crates/storage-sqlite/src/create_table.rs new file mode 100644 index 00000000..86f1f3be --- /dev/null +++ b/crates/storage-sqlite/src/create_table.rs @@ -0,0 +1,396 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `create_table` implementation for `SqliteEngine`. + +use extenddb_core::types::{ + BillingMode, BillingModeSummary, CreateTableInput, GsiDescription, LsiDescription, + ProvisionedThroughputDescription, TableDescription, TableStatus, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{index_arn, stream_arn, table_arn}; + +use crate::engine::SqliteEngine; +use crate::sqlite_util::format_timestamp; + +impl SqliteEngine { + pub(crate) async fn create_table_impl( + &self, + account_id: &str, + input: CreateTableInput, + ) -> Result { + Self::validate_account_id(account_id)?; + let table_id = uuid::Uuid::new_v4().to_string(); + let table_arn = table_arn(&self.region, account_id, &input.table_name); + let billing_mode = input.billing_mode.unwrap_or(BillingMode::Provisioned); + let key_schema_json = serde_json::to_string(&input.key_schema) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let attr_defs_json = serde_json::to_string(&input.attribute_definitions) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let billing_str = match billing_mode { + BillingMode::Provisioned => "PROVISIONED", + BillingMode::PayPerRequest => "PAY_PER_REQUEST", + }; + let pt_json = input + .provisioned_throughput + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let stream_json = input + .stream_specification + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let deletion_protection = input.deletion_protection_enabled.unwrap_or(false); + + // Read control plane delay before starting the transaction. + let delay_row: Option<(String,)> = sqlx::query_as( + "SELECT value FROM settings WHERE key = 'control_plane_delay_seconds'", + ) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let delay_secs: f64 = delay_row + .and_then(|(v,)| v.parse::().ok()) + .unwrap_or(0.25); + + let initial_status = if delay_secs == 0.0 { "ACTIVE" } else { "CREATING" }; + + let now = time::OffsetDateTime::now_utc(); + let creation_ts = format_timestamp(now); + let creation_epoch = now.unix_timestamp() as f64; + + let transition_at = if delay_secs == 0.0 { + None + } else { + let secs = delay_secs as i64; + Some(format!( + "datetime('now', '+{secs} seconds')" + )) + }; + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Insert with or without transition_at using a computed SQL string. + let insert_sql = if transition_at.is_some() { + "INSERT INTO tables \ + (account_id, table_name, key_schema, attribute_definitions, billing_mode, \ + provisioned_throughput, stream_specification, table_status, \ + creation_date_time, table_arn, table_id, deletion_protection_enabled, \ + status_transition_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, \ + datetime('now', '+' || (SELECT value FROM settings WHERE key = 'control_plane_delay_seconds') || ' seconds'))" + } else { + "INSERT INTO tables \ + (account_id, table_name, key_schema, attribute_definitions, billing_mode, \ + provisioned_throughput, stream_specification, table_status, \ + creation_date_time, table_arn, table_id, deletion_protection_enabled) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + }; + + sqlx::query(insert_sql) + .bind(account_id) + .bind(&input.table_name) + .bind(&key_schema_json) + .bind(&attr_defs_json) + .bind(billing_str) + .bind(&pt_json) + .bind(&stream_json) + .bind(initial_status) + .bind(&creation_ts) + .bind(&table_arn) + .bind(&table_id) + .bind(deletion_protection) + .execute(&mut *tx) + .await + .map_err(|e| match &e { + sqlx::Error::Database(db_err) + if db_err.message().contains("UNIQUE constraint failed") => + { + StorageError::TableAlreadyExists(input.table_name.clone()) + } + _ => StorageError::Internal(e.to_string()), + })?; + + // Insert GSI metadata + let mut gsi_index_ids: Vec = Vec::new(); + if let Some(gsis) = &input.global_secondary_indexes { + for gsi in gsis { + let gsi_ks = serde_json::to_string(&gsi.key_schema) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let gsi_proj = serde_json::to_string(&gsi.projection) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let gsi_pt = gsi + .provisioned_throughput + .as_ref() + .map(|pt| { + serde_json::to_string(&ProvisionedThroughputDescription { + read_capacity_units: pt.read_capacity_units, + write_capacity_units: pt.write_capacity_units, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }) + }) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let index_id = uuid::Uuid::new_v4().to_string(); + sqlx::query( + "INSERT INTO indexes \ + (table_id, index_name, index_id, index_type, key_schema, projection, \ + index_status, provisioned_throughput) \ + VALUES (?, ?, ?, 'GSI', ?, ?, 'ACTIVE', ?)", + ) + .bind(&table_id) + .bind(&gsi.index_name) + .bind(&index_id) + .bind(&gsi_ks) + .bind(&gsi_proj) + .bind(&gsi_pt) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + gsi_index_ids.push(index_id); + } + } + + // Insert LSI metadata + let mut lsi_index_ids: Vec = Vec::new(); + if let Some(lsis) = &input.local_secondary_indexes { + for lsi in lsis { + let lsi_ks = serde_json::to_string(&lsi.key_schema) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let lsi_proj = serde_json::to_string(&lsi.projection) + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let index_id = uuid::Uuid::new_v4().to_string(); + sqlx::query( + "INSERT INTO indexes \ + (table_id, index_name, index_id, index_type, key_schema, projection, \ + index_status, provisioned_throughput) \ + VALUES (?, ?, ?, 'LSI', ?, ?, 'ACTIVE', NULL)", + ) + .bind(&table_id) + .bind(&lsi.index_name) + .bind(&index_id) + .bind(&lsi_ks) + .bind(&lsi_proj) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + lsi_index_ids.push(index_id); + } + } + + // Insert tags + if let Some(tags) = &input.tags { + for tag in tags { + sqlx::query( + "INSERT INTO tags (resource_arn, tag_key, tag_value) VALUES (?, ?, ?)", + ) + .bind(&table_arn) + .bind(&tag.key) + .bind(&tag.value) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + } + + // Initialize stream shards if streams are enabled. + let stream_label = if input + .stream_specification + .as_ref() + .is_some_and(|s| s.stream_enabled) + { + let label = Self::init_stream_shards( + &mut tx, + account_id, + &input.table_name, + &table_id, + ) + .await?; + Some(label) + } else { + None + }; + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Create data tables in a separate transaction after catalog commit. + let data_ddl_result = async { + let mut data_tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Self::create_data_table( + &mut data_tx, + &table_id, + &input.key_schema, + &input.attribute_definitions, + ) + .await?; + + if let Some(gsis) = &input.global_secondary_indexes { + for (i, gsi) in gsis.iter().enumerate() { + Self::create_index_data_table( + &mut data_tx, + &gsi_index_ids[i], + &gsi.key_schema, + &input.attribute_definitions, + &input.key_schema, + &input.attribute_definitions, + ) + .await?; + } + } + if let Some(lsis) = &input.local_secondary_indexes { + for (i, lsi) in lsis.iter().enumerate() { + Self::create_index_data_table( + &mut data_tx, + &lsi_index_ids[i], + &lsi.key_schema, + &input.attribute_definitions, + &input.key_schema, + &input.attribute_definitions, + ) + .await?; + } + } + + data_tx + .commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok::<(), StorageError>(()) + } + .await; + + if let Err(e) = data_ddl_result { + tracing::error!( + "Failed to create data tables for '{}', cleaning up catalog: {e}", + input.table_name, + ); + let _ = sqlx::query("DELETE FROM tables WHERE account_id = ? AND table_name = ?") + .bind(account_id) + .bind(&input.table_name) + .execute(&self.pool) + .await; + return Err(e); + } + + self.control_plane_notify.notify_one(); + + let (rcu, wcu) = input.provisioned_throughput.as_ref().map_or((0, 0), |pt| { + (pt.read_capacity_units, pt.write_capacity_units) + }); + + let gsis = input.global_secondary_indexes.as_ref().map(|gs| { + gs.iter() + .map(|g| GsiDescription { + index_name: g.index_name.clone(), + key_schema: g.key_schema.clone(), + projection: g.projection.clone(), + index_status: "ACTIVE".to_owned(), + provisioned_throughput: Some(ProvisionedThroughputDescription { + read_capacity_units: g + .provisioned_throughput + .as_ref() + .map_or(0, |pt| pt.read_capacity_units), + write_capacity_units: g + .provisioned_throughput + .as_ref() + .map_or(0, |pt| pt.write_capacity_units), + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }), + index_size_bytes: 0, + item_count: 0, + index_arn: index_arn( + &self.region, + account_id, + &input.table_name, + &g.index_name, + ), + }) + .collect() + }); + + let lsis = input.local_secondary_indexes.as_ref().map(|ls| { + ls.iter() + .map(|l| LsiDescription { + index_name: l.index_name.clone(), + key_schema: l.key_schema.clone(), + projection: l.projection.clone(), + index_size_bytes: 0, + item_count: 0, + index_arn: index_arn( + &self.region, + account_id, + &input.table_name, + &l.index_name, + ), + }) + .collect() + }); + + let billing_mode_summary = if billing_mode == BillingMode::PayPerRequest { + Some(BillingModeSummary { + billing_mode: BillingMode::PayPerRequest, + last_update_to_pay_per_request_date_time: Some(creation_epoch), + }) + } else { + None + }; + + let latest_stream_arn = stream_label + .as_ref() + .map(|label| stream_arn(&self.region, account_id, &input.table_name, label)); + + let response_status = if initial_status == "ACTIVE" { + TableStatus::Active + } else { + TableStatus::Creating + }; + + Ok(TableDescription { + table_name: input.table_name, + key_schema: input.key_schema, + attribute_definitions: input.attribute_definitions, + table_status: response_status, + creation_date_time: creation_epoch, + table_size_bytes: 0, + item_count: 0, + table_arn, + table_id, + provisioned_throughput: ProvisionedThroughputDescription { + read_capacity_units: rcu, + write_capacity_units: wcu, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }, + billing_mode_summary, + global_secondary_indexes: gsis, + local_secondary_indexes: lsis, + stream_specification: input.stream_specification, + latest_stream_arn, + latest_stream_label: stream_label, + deletion_protection_enabled: input.deletion_protection_enabled.unwrap_or(false), + sse_description: None, + table_class_summary: None, + }) + } +} diff --git a/crates/storage-sqlite/src/credential_store.rs b/crates/storage-sqlite/src/credential_store.rs new file mode 100644 index 00000000..d33a63d5 --- /dev/null +++ b/crates/storage-sqlite/src/credential_store.rs @@ -0,0 +1,179 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! SQLite-backed credential store for SigV4 authentication. + +use extenddb_auth::{CredentialStore, StoredCredential}; +use extenddb_core::error::DynamoDbError; +use sqlx::SqlitePool; +use zeroize::{Zeroize, ZeroizeOnDrop}; + +fn decrypt_secret(encrypted: &[u8], key_b64: &str, aad: &str) -> Result { + use aes_gcm::Aes256Gcm; + use aes_gcm::KeyInit; + use aes_gcm::aead::Aead; + use aes_gcm::aead::Payload; + use base64::Engine; + + if encrypted.len() < 28 { + return Err( + "ciphertext too short (need at least 12-byte nonce + 16-byte auth tag)".to_owned(), + ); + } + + let key_bytes = base64::engine::general_purpose::STANDARD + .decode(key_b64) + .map_err(|e| format!("decode encryption key: {e}"))?; + + let key = aes_gcm::Key::::from_slice(&key_bytes); + let cipher = Aes256Gcm::new(key); + let nonce = aes_gcm::Nonce::from_slice(&encrypted[..12]); + + let payload_with_aad = Payload { + msg: &encrypted[12..], + aad: aad.as_bytes(), + }; + if let Ok(plaintext_bytes) = cipher.decrypt(nonce, payload_with_aad) { + return String::from_utf8(plaintext_bytes) + .map_err(|e| format!("decrypted secret is not valid UTF-8: {e}")); + } + + tracing::debug!("Decrypting secret without AAD (pre-CB-11 format) for {aad}"); + let plaintext_bytes = cipher + .decrypt(nonce, &encrypted[12..]) + .map_err(|e| format!("decrypt: {e}"))?; + + String::from_utf8(plaintext_bytes) + .map_err(|e| format!("decrypted secret is not valid UTF-8: {e}")) +} + +/// Credential store backed by the SQLite catalog database. +#[derive(Zeroize, ZeroizeOnDrop)] +pub struct SqliteCredentialStore { + #[zeroize(skip)] + pool: SqlitePool, + encryption_key: String, +} + +impl SqliteCredentialStore { + pub fn new(pool: SqlitePool, encryption_key: String) -> Self { + Self { + pool, + encryption_key, + } + } +} + +#[async_trait::async_trait] +impl CredentialStore for SqliteCredentialStore { + async fn lookup_credential( + &self, + access_key_id: &str, + ) -> Result, DynamoDbError> { + if access_key_id.starts_with("AKIA") { + return self.lookup_user_credential(access_key_id).await; + } + if access_key_id.starts_with("ASIA") { + return self.lookup_session_credential(access_key_id).await; + } + Ok(None) + } +} + +impl SqliteCredentialStore { + async fn lookup_user_credential( + &self, + access_key_id: &str, + ) -> Result, DynamoDbError> { + let row: Option<(Vec, String, String, bool)> = sqlx::query_as( + "SELECT secret_key_encrypted, account_id, user_name, is_active \ + FROM access_keys WHERE access_key_id = ?", + ) + .bind(access_key_id) + .fetch_optional(&self.pool) + .await + .map_err(|e| { + tracing::error!("Credential lookup failed for access key {access_key_id}: {e}"); + DynamoDbError::InternalServerError("Internal error during authentication".to_owned()) + })?; + + let Some((encrypted, account_id, user_name, is_active)) = row else { + return Ok(None); + }; + + let secret_key = + decrypt_secret(&encrypted, &self.encryption_key, access_key_id).map_err(|e| { + tracing::error!("Secret key decryption failed for access key {access_key_id}: {e}"); + DynamoDbError::InternalServerError( + "Internal error during authentication".to_owned(), + ) + })?; + + Ok(Some(StoredCredential { + secret_key, + account_id, + principal_name: user_name, + session_name: None, + is_session: false, + session_token: None, + is_active, + })) + } + + async fn lookup_session_credential( + &self, + access_key_id: &str, + ) -> Result, DynamoDbError> { + let row: Option<(Vec, String, String, String, String, String)> = sqlx::query_as( + "SELECT secret_key_encrypted, account_id, role_name, session_name, \ + session_token, expires_at \ + FROM iam_sessions WHERE access_key_id = ?", + ) + .bind(access_key_id) + .fetch_optional(&self.pool) + .await + .map_err(|e| { + tracing::error!( + "Session credential lookup failed for access key {access_key_id}: {e}" + ); + DynamoDbError::InternalServerError("Internal error during authentication".to_owned()) + })?; + + let Some((encrypted, account_id, role_name, session_name, session_token, expires_at_str)) = + row + else { + return Ok(None); + }; + + let expires_at = crate::sqlite_util::parse_timestamp(&expires_at_str).map_err(|e| { + tracing::error!("Session expiry parse error: {e}"); + DynamoDbError::InternalServerError("Internal error during authentication".to_owned()) + })?; + + if expires_at < time::OffsetDateTime::now_utc() { + return Err(DynamoDbError::ExpiredTokenException( + "The security token included in the request is expired".to_owned(), + )); + } + + let secret_key = + decrypt_secret(&encrypted, &self.encryption_key, access_key_id).map_err(|e| { + tracing::error!( + "Session secret key decryption failed for access key {access_key_id}: {e}" + ); + DynamoDbError::InternalServerError( + "Internal error during authentication".to_owned(), + ) + })?; + + Ok(Some(StoredCredential { + secret_key, + account_id, + principal_name: role_name, + session_name: Some(session_name), + is_session: true, + session_token: Some(session_token), + is_active: true, + })) + } +} diff --git a/crates/storage-sqlite/src/data/data_engine.rs b/crates/storage-sqlite/src/data/data_engine.rs new file mode 100644 index 00000000..7957db9e --- /dev/null +++ b/crates/storage-sqlite/src/data/data_engine.rs @@ -0,0 +1,335 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `DataEngine` trait implementation for `SqliteEngine`. + +use extenddb_core::expression::{Expr, ExpressionMaps, KeyCondition, UpdateAction}; +use extenddb_core::types::{Item, TableKeyInfo}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{DataEngine, StreamCapture, TransactGetOp, TransactWriteOp}; +use futures::future::BoxFuture; + +use crate::engine::SqliteEngine; + +impl DataEngine for SqliteEngine { + fn put_item( + &self, + key_info: &TableKeyInfo, + item: Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, Result, StorageError>> { + let key_info = key_info.clone(); + let condition = condition.cloned(); + let maps = maps.clone(); + let stream = stream.cloned(); + Box::pin(async move { + self.put_item_impl( + &key_info, + item, + return_old, + condition.as_ref(), + &maps, + stream.as_ref(), + ) + .await + }) + } + + fn get_item( + &self, + key_info: &TableKeyInfo, + key: &Item, + ) -> BoxFuture<'_, Result, StorageError>> { + let key_info = key_info.clone(); + let key = key.clone(); + Box::pin(async move { self.get_item_impl(&key_info, &key).await }) + } + + fn delete_item( + &self, + key_info: &TableKeyInfo, + key: &Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, Result, StorageError>> { + let key_info = key_info.clone(); + let key = key.clone(); + let condition = condition.cloned(); + let maps = maps.clone(); + let stream = stream.cloned(); + Box::pin(async move { + self.delete_item_impl( + &key_info, + &key, + return_old, + condition.as_ref(), + &maps, + stream.as_ref(), + ) + .await + }) + } + + fn update_item( + &self, + key_info: &TableKeyInfo, + key: &Item, + actions: &[UpdateAction], + return_old: bool, + return_new: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, Result<(Option, Option), StorageError>> { + let key_info = key_info.clone(); + let key = key.clone(); + let actions = actions.to_vec(); + let condition = condition.cloned(); + let maps = maps.clone(); + let stream = stream.cloned(); + Box::pin(async move { + self.update_item_impl( + &key_info, + &key, + &actions, + return_old, + return_new, + condition.as_ref(), + &maps, + stream.as_ref(), + ) + .await + }) + } + + fn query( + &self, + key_info: &TableKeyInfo, + key_condition: &KeyCondition, + maps: &ExpressionMaps, + forward: bool, + limit: Option, + exclusive_start_key: Option<&Item>, + index_name: Option<&str>, + ) -> BoxFuture<'_, Result<(Vec, Option), StorageError>> { + let key_info = key_info.clone(); + let key_condition = key_condition.clone(); + let maps = maps.clone(); + let exclusive_start_key = exclusive_start_key.cloned(); + let index_name = index_name.map(|s| s.to_string()); + Box::pin(async move { + self.query_impl( + &key_info, + &key_condition, + &maps, + forward, + limit, + exclusive_start_key.as_ref(), + index_name.as_deref(), + ) + .await + }) + } + + fn scan( + &self, + key_info: &TableKeyInfo, + limit: Option, + exclusive_start_key: Option<&Item>, + segment: Option, + total_segments: Option, + index_name: Option<&str>, + ) -> BoxFuture<'_, Result<(Vec, Option), StorageError>> { + let key_info = key_info.clone(); + let exclusive_start_key = exclusive_start_key.cloned(); + let index_name = index_name.map(|s| s.to_string()); + Box::pin(async move { + self.scan_impl( + &key_info, + limit, + exclusive_start_key.as_ref(), + segment, + total_segments, + index_name.as_deref(), + ) + .await + }) + } + + fn transact_get_items( + &self, + ops: &[TransactGetOp<'_>], + ) -> BoxFuture<'_, Result>, StorageError>> { + let owned_ops: Vec<_> = ops + .iter() + .map(|op| (op.key_info.clone(), op.key.clone())) + .collect(); + Box::pin(async move { + let borrowed_ops: Vec = owned_ops + .iter() + .map(|(key_info, key)| TransactGetOp { key_info, key }) + .collect(); + self.transact_get_items_impl(&borrowed_ops).await + }) + } + + fn transact_write_items( + &self, + ops: &[TransactWriteOp<'_>], + token: Option<(&str, &str)>, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let owned_ops: Vec<_> = ops + .iter() + .map(|op| match op { + TransactWriteOp::Put { + key_info, + item, + condition, + maps, + return_values_on_ccf, + stream, + } => ( + 0u8, + (*key_info).clone(), + (*item).clone(), + None, + Vec::new(), + condition.cloned(), + (*maps).clone(), + *return_values_on_ccf, + stream.clone(), + ), + TransactWriteOp::Delete { + key_info, + key, + condition, + maps, + return_values_on_ccf, + stream, + } => ( + 1u8, + (*key_info).clone(), + (*key).clone(), + None, + Vec::new(), + condition.cloned(), + (*maps).clone(), + *return_values_on_ccf, + stream.clone(), + ), + TransactWriteOp::Update { + key_info, + key, + actions, + condition, + maps, + return_values_on_ccf, + stream, + } => ( + 2u8, + (*key_info).clone(), + (*key).clone(), + None, + actions.to_vec(), + condition.cloned(), + (*maps).clone(), + *return_values_on_ccf, + stream.clone(), + ), + TransactWriteOp::ConditionCheck { + key_info, + key, + condition, + maps, + return_values_on_ccf, + } => ( + 3u8, + (*key_info).clone(), + (*key).clone(), + Some((*condition).clone()), + Vec::new(), + None, + (*maps).clone(), + *return_values_on_ccf, + None, + ), + }) + .collect(); + let token = token.map(|(a, b)| (a.to_string(), b.to_string())); + + Box::pin(async move { + let borrowed_ops: Vec = owned_ops + .iter() + .map( + |( + tag, + key_info, + item_or_key, + cond_check, + actions, + condition, + maps, + rv, + stream, + )| { + match tag { + 0 => TransactWriteOp::Put { + key_info, + item: item_or_key, + condition: condition.as_ref(), + maps, + return_values_on_ccf: *rv, + stream: stream.clone(), + }, + 1 => TransactWriteOp::Delete { + key_info, + key: item_or_key, + condition: condition.as_ref(), + maps, + return_values_on_ccf: *rv, + stream: stream.clone(), + }, + 2 => TransactWriteOp::Update { + key_info, + key: item_or_key, + actions, + condition: condition.as_ref(), + maps, + return_values_on_ccf: *rv, + stream: stream.clone(), + }, + 3 => TransactWriteOp::ConditionCheck { + key_info, + key: item_or_key, + condition: cond_check.as_ref().unwrap(), + maps, + return_values_on_ccf: *rv, + }, + _ => unreachable!(), + } + }, + ) + .collect(); + self.transact_write_items_impl( + &borrowed_ops, + token.as_ref().map(|(a, b)| (a.as_str(), b.as_str())), + ) + .await + }) + } + + fn cleanup_expired_idempotency_tokens( + &self, + max_age_seconds: i64, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { + self.cleanup_expired_idempotency_tokens_impl(max_age_seconds) + .await + }) + } +} diff --git a/crates/storage-sqlite/src/data/ddl.rs b/crates/storage-sqlite/src/data/ddl.rs new file mode 100644 index 00000000..dff9389c --- /dev/null +++ b/crates/storage-sqlite/src/data/ddl.rs @@ -0,0 +1,314 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! DDL helpers for creating and dropping per-DynamoDB-table data tables in SQLite. + +use extenddb_core::types::{ + AttributeDefinition, IndexInfo, IndexType, KeySchemaElement, Projection, StreamSpecification, + TableKeyInfo, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{sk_column, sk_column_n}; + +use super::{all_sort_key_info, data_table_name, index_table_name}; +use crate::engine::SqliteEngine; + +impl SqliteEngine { + /// Create the per-DynamoDB-table data table in SQLite. + pub(crate) async fn create_data_table( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, + key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + ) -> Result<(), StorageError> { + let ddb_table = data_table_name(table_id); + let sk_infos = all_sort_key_info(key_schema, attr_defs); + + let ddl = if sk_infos.is_empty() { + format!( + "CREATE TABLE {ddb_table} (\ + pk TEXT NOT NULL PRIMARY KEY,\ + item_data TEXT NOT NULL\ + )" + ) + } else if sk_infos.len() == 1 { + let sk_col = sk_column(sk_infos[0].1); + format!( + "CREATE TABLE {ddb_table} (\ + pk TEXT NOT NULL,\ + sk_s TEXT,\ + sk_n REAL,\ + sk_b BLOB,\ + item_data TEXT NOT NULL,\ + PRIMARY KEY (pk, {sk_col})\ + )" + ) + } else { + let mut col_defs = vec!["pk TEXT NOT NULL".to_owned()]; + let mut pk_cols = vec!["pk".to_owned()]; + for (i, &(_, sk_type)) in sk_infos.iter().enumerate() { + let col = sk_column_n(i, sk_type); + if i == 0 { + col_defs.push("sk_s TEXT".to_owned()); + col_defs.push("sk_n REAL".to_owned()); + col_defs.push("sk_b BLOB".to_owned()); + } else { + let n = i + 1; + col_defs.push(format!("sk{n}_s TEXT")); + col_defs.push(format!("sk{n}_n REAL")); + col_defs.push(format!("sk{n}_b BLOB")); + } + pk_cols.push(col); + } + col_defs.push("item_data TEXT NOT NULL".to_owned()); + format!( + "CREATE TABLE {ddb_table} (\n {},\n PRIMARY KEY ({})\n)", + col_defs.join(",\n "), + pk_cols.join(", ") + ) + }; + + sqlx::query(&ddl) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(()) + } + + /// Drop the per-DynamoDB-table data table. + pub(crate) async fn drop_data_table( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, + ) -> Result<(), StorageError> { + let ddb_table = data_table_name(table_id); + let ddl = format!("DROP TABLE IF EXISTS {ddb_table}"); + sqlx::query(&ddl) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + /// Create a GSI/LSI data table in SQLite. + #[allow(clippy::too_many_arguments)] + pub(crate) async fn create_index_data_table( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + index_id: &str, + index_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + base_key_schema: &[KeySchemaElement], + base_attr_defs: &[AttributeDefinition], + ) -> Result<(), StorageError> { + let idx_table = index_table_name(index_id); + + let base_sks = all_sort_key_info(base_key_schema, base_attr_defs); + let idx_sks = all_sort_key_info(index_key_schema, attr_defs); + + let mut col_defs = vec!["pk TEXT NOT NULL".to_owned()]; + + for (i, &(_, _)) in idx_sks.iter().enumerate() { + if i == 0 { + col_defs.push("sk_s TEXT".to_owned()); + col_defs.push("sk_n REAL".to_owned()); + col_defs.push("sk_b BLOB".to_owned()); + } else { + let n = i + 1; + col_defs.push(format!("sk{n}_s TEXT")); + col_defs.push(format!("sk{n}_n REAL")); + col_defs.push(format!("sk{n}_b BLOB")); + } + } + + col_defs.push("base_pk TEXT NOT NULL".to_owned()); + for (i, &(_, _)) in base_sks.iter().enumerate() { + if i == 0 { + col_defs.push("base_sk_s TEXT".to_owned()); + col_defs.push("base_sk_n REAL".to_owned()); + col_defs.push("base_sk_b BLOB".to_owned()); + } else { + let n = i + 1; + col_defs.push(format!("base_sk{n}_s TEXT")); + col_defs.push(format!("base_sk{n}_n REAL")); + col_defs.push(format!("base_sk{n}_b BLOB")); + } + } + + col_defs.push("item_data TEXT NOT NULL".to_owned()); + + let mut pk_cols = vec!["pk".to_owned(), "base_pk".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + let col = if i == 0 { + format!("base_{}", sk_column(sk_type)) + } else { + format!("base_{}", sk_column_n(i, sk_type)) + }; + pk_cols.push(col); + } + + let ddl = format!( + "CREATE TABLE {idx_table} (\n {},\n PRIMARY KEY ({})\n)", + col_defs.join(",\n "), + pk_cols.join(", ") + ); + + sqlx::query(&ddl) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if !idx_sks.is_empty() { + let mut order_cols = vec!["pk".to_owned()]; + for (i, &(_, sk_type)) in idx_sks.iter().enumerate() { + order_cols.push(sk_column_n(i, sk_type)); + } + order_cols.push("base_pk".to_owned()); + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + let col = if i == 0 { + format!("base_{}", sk_column(sk_type)) + } else { + format!("base_{}", sk_column_n(i, sk_type)) + }; + order_cols.push(col); + } + let order_idx = format!("CREATE INDEX ON {idx_table} ({})", order_cols.join(", ")); + sqlx::query(&order_idx) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + Ok(()) + } + + /// Drop a GSI/LSI data table. + pub(crate) async fn drop_index_data_table( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + index_id: &str, + ) -> Result<(), StorageError> { + let idx_table = index_table_name(index_id); + let ddl = format!("DROP TABLE IF EXISTS {idx_table}"); + sqlx::query(&ddl) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + /// Fetch key schema and attribute definitions for a table from the catalog. + pub(crate) async fn fetch_table_key_info( + &self, + account_id: &str, + table_name: &str, + ) -> Result { + let row: Option<(String, String, String, String, Option, i64)> = sqlx::query_as( + "SELECT key_schema, attribute_definitions, table_status, table_id, \ + stream_specification, \ + (SELECT COUNT(*) FROM indexes WHERE table_id = tables.table_id AND index_type = 'LSI') \ + FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (ks_str, ad_str, status, table_id, stream_spec_str, lsi_count) = + row.ok_or_else(|| StorageError::TableNotFound(table_name.to_owned()))?; + + if status != "ACTIVE" { + return Err(StorageError::TableNotActive(table_name.to_owned())); + } + + let key_schema: Vec = + serde_json::from_str(&ks_str).map_err(|e| StorageError::Internal(e.to_string()))?; + let attribute_definitions: Vec = + serde_json::from_str(&ad_str).map_err(|e| StorageError::Internal(e.to_string()))?; + + let stream_specification: Option = stream_spec_str + .as_deref() + .map(serde_json::from_str) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(TableKeyInfo { + table_name: table_name.to_owned(), + account_id: account_id.to_owned(), + table_id, + key_schema, + attribute_definitions, + has_lsi: lsi_count > 0, + stream_specification, + }) + } + + /// Fetch metadata for a secondary index from the catalog. + pub(crate) async fn fetch_index_info( + &self, + account_id: &str, + table_name: &str, + index_name: &str, + ) -> Result { + let row: Option<(String, String)> = sqlx::query_as( + "SELECT table_id, table_status FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (table_id, status) = + row.ok_or_else(|| StorageError::TableNotFound(table_name.to_owned()))?; + + if status != "ACTIVE" { + return Err(StorageError::TableNotActive(table_name.to_owned())); + } + + self.fetch_index_info_by_table_id(&table_id, index_name) + .await + } + + /// Fetch metadata for a secondary index using a known `table_id`. + pub(crate) async fn fetch_index_info_by_table_id( + &self, + table_id: &str, + index_name: &str, + ) -> Result { + let idx_row: Option<(String, String, String, String)> = sqlx::query_as( + "SELECT index_type, index_id, key_schema, projection \ + FROM indexes WHERE table_id = ? AND index_name = ?", + ) + .bind(table_id) + .bind(index_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (idx_type_str, idx_id, ks_str, proj_str) = + idx_row.ok_or_else(|| StorageError::IndexNotFound(index_name.to_owned()))?; + + let index_type = match idx_type_str.as_str() { + "GSI" => IndexType::Gsi, + "LSI" => IndexType::Lsi, + other => { + return Err(StorageError::Internal(format!( + "unknown index type in database: {other}" + ))); + } + }; + + let key_schema: Vec = + serde_json::from_str(&ks_str).map_err(|e| StorageError::Internal(e.to_string()))?; + let projection: Projection = + serde_json::from_str(&proj_str).map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(IndexInfo { + index_name: index_name.to_owned(), + index_id: idx_id, + index_type, + key_schema, + projection, + }) + } +} diff --git a/crates/storage-sqlite/src/data/delete_item.rs b/crates/storage-sqlite/src/data/delete_item.rs new file mode 100644 index 00000000..ca646dab --- /dev/null +++ b/crates/storage-sqlite/src/data/delete_item.rs @@ -0,0 +1,273 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `delete_item` implementation for the SQLite backend. + +use extenddb_core::expression::{Expr, ExpressionMaps}; +use extenddb_core::types::{Item, TableKeyInfo}; +use extenddb_storage::StreamCapture; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{SortKeyValue, parse_sk, pk_to_text, sk_column, sk_info}; + +use super::index::{fetch_indexes_for_table, sync_indexes}; +use super::query::check_condition; +use super::tx_helpers::write_stream_record_in_tx; +use super::{bigdecimal_to_f64, data_table_name, json_to_item}; +use crate::engine::SqliteEngine; + +impl SqliteEngine { + pub(crate) async fn delete_item_impl( + &self, + key_info: &TableKeyInfo, + key: &Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> Result, StorageError> { + let ddb_table = data_table_name(&key_info.table_id); + + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_value = key + .get(pk_name) + .ok_or_else(|| StorageError::Internal("missing partition key".to_owned()))?; + let pk_text = pk_to_text(pk_value)?; + + let indexes = fetch_indexes_for_table(&key_info.table_id, &self.pool).await?; + let needs_tx = + condition.is_some() || return_old || !indexes.is_empty() || stream.is_some(); + + if let Some((sk_name, sk_type)) = + sk_info(&key_info.key_schema, &key_info.attribute_definitions) + { + let sk_value = key + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk = parse_sk(sk_value, sk_type)?; + let sk_col = sk_column(sk_type); + + if needs_tx { + let select_sql = + format!("SELECT item_data FROM {ddb_table} WHERE pk = ? AND {sk_col} = ?"); + let delete_sql = + format!("DELETE FROM {ddb_table} WHERE pk = ? AND {sk_col} = ?"); + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let old: Option<(serde_json::Value,)> = + bind_sk_fetch_optional!(&select_sql, pk_text.as_ref(), &sk, &mut *tx)?; + + if let Some((ref old_json,)) = old { + let old_item: Item = json_to_item(old_json.clone())?; + match check_condition(condition, &old_item, maps) { + Ok(()) => {} + Err(StorageError::ConditionFailed(_)) => { + return Err(StorageError::ConditionFailed(Some(old_item))); + } + Err(e) => return Err(e), + } + } else { + let empty = std::collections::BTreeMap::new(); + match check_condition(condition, &empty, maps) { + Ok(()) => {} + Err(StorageError::ConditionFailed(_)) => { + return Err(StorageError::ConditionFailed(None)); + } + Err(e) => return Err(e), + } + return Ok(None); + } + + match &sk { + SortKeyValue::S(s) => { + sqlx::query(&delete_sql) + .bind(pk_text.as_ref()) + .bind(s) + .execute(&mut *tx) + .await + } + SortKeyValue::N(n) => { + sqlx::query(&delete_sql) + .bind(pk_text.as_ref()) + .bind(bigdecimal_to_f64(n)) + .execute(&mut *tx) + .await + } + SortKeyValue::B(b) => { + sqlx::query(&delete_sql) + .bind(pk_text.as_ref()) + .bind(b) + .execute(&mut *tx) + .await + } + } + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if !indexes.is_empty() { + let old_item = old + .as_ref() + .map(|(v,)| json_to_item(v.clone())) + .transpose()?; + sync_indexes( + &mut tx, + &key_info.key_schema, + &key_info.attribute_definitions, + &indexes, + old_item.as_ref(), + None, + ) + .await?; + } + + if let Some(capture) = stream { + let old_for_stream = old + .as_ref() + .map(|(v,)| json_to_item(v.clone())) + .transpose()?; + write_stream_record_in_tx( + &mut tx, + key_info, + capture, + old_for_stream.as_ref(), + None, + ) + .await?; + } + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if return_old { + old.map(|(v,)| json_to_item(v)).transpose() + } else { + Ok(None) + } + } else { + let delete_sql = + format!("DELETE FROM {ddb_table} WHERE pk = ? AND {sk_col} = ?"); + match &sk { + SortKeyValue::S(s) => { + sqlx::query(&delete_sql) + .bind(pk_text.as_ref()) + .bind(s) + .execute(&self.pool) + .await + } + SortKeyValue::N(n) => { + sqlx::query(&delete_sql) + .bind(pk_text.as_ref()) + .bind(bigdecimal_to_f64(n)) + .execute(&self.pool) + .await + } + SortKeyValue::B(b) => { + sqlx::query(&delete_sql) + .bind(pk_text.as_ref()) + .bind(b) + .execute(&self.pool) + .await + } + } + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(None) + } + } else { + // PK-only table + if needs_tx { + let select_sql = format!("SELECT item_data FROM {ddb_table} WHERE pk = ?"); + let delete_sql = format!("DELETE FROM {ddb_table} WHERE pk = ?"); + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let old: Option<(serde_json::Value,)> = sqlx::query_as(&select_sql) + .bind(pk_text.as_ref()) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if let Some((ref old_json,)) = old { + let old_item: Item = json_to_item(old_json.clone())?; + match check_condition(condition, &old_item, maps) { + Ok(()) => {} + Err(StorageError::ConditionFailed(_)) => { + return Err(StorageError::ConditionFailed(Some(old_item))); + } + Err(e) => return Err(e), + } + } else { + let empty = std::collections::BTreeMap::new(); + match check_condition(condition, &empty, maps) { + Ok(()) => {} + Err(StorageError::ConditionFailed(_)) => { + return Err(StorageError::ConditionFailed(None)); + } + Err(e) => return Err(e), + } + return Ok(None); + } + + sqlx::query(&delete_sql) + .bind(pk_text.as_ref()) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if !indexes.is_empty() { + let old_item = old + .as_ref() + .map(|(v,)| json_to_item(v.clone())) + .transpose()?; + sync_indexes( + &mut tx, + &key_info.key_schema, + &key_info.attribute_definitions, + &indexes, + old_item.as_ref(), + None, + ) + .await?; + } + + if let Some(capture) = stream { + let old_for_stream = old + .as_ref() + .map(|(v,)| json_to_item(v.clone())) + .transpose()?; + write_stream_record_in_tx( + &mut tx, + key_info, + capture, + old_for_stream.as_ref(), + None, + ) + .await?; + } + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if return_old { + old.map(|(v,)| json_to_item(v)).transpose() + } else { + Ok(None) + } + } else { + let delete_sql = format!("DELETE FROM {ddb_table} WHERE pk = ?"); + sqlx::query(&delete_sql) + .bind(pk_text.as_ref()) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(None) + } + } + } +} diff --git a/crates/storage-sqlite/src/data/index.rs b/crates/storage-sqlite/src/data/index.rs new file mode 100644 index 00000000..78ef897a --- /dev/null +++ b/crates/storage-sqlite/src/data/index.rs @@ -0,0 +1,259 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! GSI/LSI index operations for the SQLite backend. + +use extenddb_core::types::{ + AttributeDefinition, Item, KeySchemaElement, Projection, ProjectionType, ScalarAttributeType, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::SortKeyValue; +use extenddb_storage::util::{composite_pk_to_text, parse_sk, sk_column, sk_column_n}; + +use super::{all_sort_key_info, index_table_name}; + +/// Metadata for a single index, used during write-path GSI/LSI sync. +pub(crate) struct IndexMeta { + pub(super) index_id: String, + pub(super) key_schema: Vec, + pub(super) projection: Projection, +} + +/// Fetch all index metadata for a table from the catalog. +pub(crate) async fn fetch_indexes_for_table( + table_id: &str, + pool: &sqlx::SqlitePool, +) -> Result, StorageError> { + let rows: Vec<(String, String, String, String)> = sqlx::query_as( + "SELECT index_id, index_type, key_schema, projection FROM indexes WHERE table_id = ?", + ) + .bind(table_id) + .fetch_all(pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + rows.into_iter() + .map(|(id, _idx_type, ks_str, proj_str)| { + let key_schema: Vec = serde_json::from_str(&ks_str) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let projection: Projection = serde_json::from_str(&proj_str) + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(IndexMeta { + index_id: id, + key_schema, + projection, + }) + }) + .collect() +} + +/// Project an item according to an index's projection configuration. +pub(crate) fn project_item_for_index( + item: &Item, + index_ks: &[KeySchemaElement], + base_ks: &[KeySchemaElement], + projection: &Projection, +) -> Item { + match projection.projection_type { + ProjectionType::All => item.clone(), + ProjectionType::KeysOnly => { + let mut projected = Item::new(); + for ks in base_ks.iter().chain(index_ks.iter()) { + if let Some(v) = item.get(&ks.attribute_name) { + projected.insert(ks.attribute_name.clone(), v.clone()); + } + } + projected + } + ProjectionType::Include => { + let mut projected = Item::new(); + for ks in base_ks.iter().chain(index_ks.iter()) { + if let Some(v) = item.get(&ks.attribute_name) { + projected.insert(ks.attribute_name.clone(), v.clone()); + } + } + if let Some(ref attrs) = projection.non_key_attributes { + for attr in attrs { + if let Some(v) = item.get(attr) { + projected.insert(attr.clone(), v.clone()); + } + } + } + projected + } + } +} + +/// Check if an item has all the key attributes required by an index. +pub(crate) fn item_has_index_keys(item: &Item, index_ks: &[KeySchemaElement]) -> bool { + index_ks + .iter() + .all(|ks| item.contains_key(&ks.attribute_name)) +} + +/// Synchronously update all index tables for all indexes. +/// +/// In SQLite all indexes are always synchronous (no async propagation queue). +#[allow(clippy::too_many_arguments)] +pub(crate) async fn sync_indexes( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + base_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + indexes: &[IndexMeta], + old_item: Option<&Item>, + new_item: Option<&Item>, +) -> Result<(), StorageError> { + for idx in indexes { + let idx_table = index_table_name(&idx.index_id); + let idx_sks = all_sort_key_info(&idx.key_schema, attr_defs); + let base_sks = all_sort_key_info(base_key_schema, attr_defs); + + if let Some(old) = old_item { + if item_has_index_keys(old, &idx.key_schema) { + delete_index_row_multi(tx, &idx_table, old, base_key_schema, attr_defs, &base_sks) + .await?; + } + } + + if let Some(new) = new_item { + if item_has_index_keys(new, &idx.key_schema) { + let projected = + project_item_for_index(new, &idx.key_schema, base_key_schema, &idx.projection); + insert_index_row_multi( + tx, + &idx_table, + new, + &projected, + &idx.key_schema, + base_key_schema, + attr_defs, + &idx_sks, + &base_sks, + ) + .await?; + } + } + } + Ok(()) +} + +/// Delete a row from an index table using base table key columns. +pub(crate) async fn delete_index_row_multi( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + idx_table: &str, + item: &Item, + base_ks: &[KeySchemaElement], + _attr_defs: &[AttributeDefinition], + base_sks: &[(&str, ScalarAttributeType)], +) -> Result<(), StorageError> { + let base_pk_text = composite_pk_to_text(item, base_ks)?; + + let mut where_parts = vec!["base_pk = ?".to_owned()]; + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + let col = if i == 0 { + format!("base_{}", sk_column(sk_type)) + } else { + format!("base_{}", sk_column_n(i, sk_type)) + }; + where_parts.push(format!("{col} = ?")); + } + + let sql = format!( + "DELETE FROM {idx_table} WHERE {}", + where_parts.join(" AND ") + ); + let mut query = sqlx::query(&sql).bind(base_pk_text); + + for &(sk_name, sk_type) in base_sks { + if let Some(sk_val) = item.get(sk_name) { + let sk = parse_sk(sk_val, sk_type)?; + query = match sk { + SortKeyValue::S(s) => query.bind(s), + SortKeyValue::N(n) => query.bind(super::bigdecimal_to_f64(&n)), + SortKeyValue::B(b) => query.bind(b), + }; + } + } + + query + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) +} + +/// Insert a row into an index table with multi-part key support. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn insert_index_row_multi( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + idx_table: &str, + item: &Item, + projected: &Item, + index_ks: &[KeySchemaElement], + base_ks: &[KeySchemaElement], + _attr_defs: &[AttributeDefinition], + idx_sks: &[(&str, ScalarAttributeType)], + base_sks: &[(&str, ScalarAttributeType)], +) -> Result<(), StorageError> { + let idx_pk_text = composite_pk_to_text(item, index_ks)?; + let base_pk_text = composite_pk_to_text(item, base_ks)?; + + let item_json = + serde_json::to_value(projected).map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut cols = vec!["pk".to_owned()]; + for (i, &(_, sk_type)) in idx_sks.iter().enumerate() { + cols.push(sk_column_n(i, sk_type)); + } + cols.push("base_pk".to_owned()); + for (i, &(_, sk_type)) in base_sks.iter().enumerate() { + let col = if i == 0 { + format!("base_{}", sk_column(sk_type)) + } else { + format!("base_{}", sk_column_n(i, sk_type)) + }; + cols.push(col); + } + cols.push("item_data".to_owned()); + + let placeholders: Vec<&str> = cols.iter().map(|_| "?").collect(); + let sql = format!( + "INSERT OR REPLACE INTO {idx_table} ({}) VALUES ({})", + cols.join(", "), + placeholders.join(", ") + ); + + let mut query = sqlx::query(&sql).bind(idx_pk_text); + + for &(sk_name, sk_type) in idx_sks { + if let Some(sk_val) = item.get(sk_name) { + let sk = parse_sk(sk_val, sk_type)?; + query = match sk { + SortKeyValue::S(s) => query.bind(s), + SortKeyValue::N(n) => query.bind(super::bigdecimal_to_f64(&n)), + SortKeyValue::B(b) => query.bind(b), + }; + } + } + + query = query.bind(base_pk_text); + + for &(sk_name, sk_type) in base_sks { + if let Some(sk_val) = item.get(sk_name) { + let sk = parse_sk(sk_val, sk_type)?; + query = match sk { + SortKeyValue::S(s) => query.bind(s), + SortKeyValue::N(n) => query.bind(super::bigdecimal_to_f64(&n)), + SortKeyValue::B(b) => query.bind(b), + }; + } + } + + query = query.bind(item_json); + + query + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) +} diff --git a/crates/storage-sqlite/src/data/mod.rs b/crates/storage-sqlite/src/data/mod.rs new file mode 100644 index 00000000..806aacfc --- /dev/null +++ b/crates/storage-sqlite/src/data/mod.rs @@ -0,0 +1,128 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Per-DynamoDB-table DDL and item CRUD for the SQLite backend. +//! +//! Each virtual DynamoDB table maps to a SQLite table named `_ddb_`. +//! Partition keys are stored as TEXT. Sort keys use typed columns (`sk_s`, `sk_n`, `sk_b`) +//! for ordering. The full item is stored as JSON TEXT in `item_data`. + +use extenddb_core::types::{AttributeDefinition, Item, KeySchemaElement, ScalarAttributeType}; +use extenddb_storage::error::StorageError; + +/// SQL table name for a virtual DynamoDB table. +pub(crate) fn data_table_name(table_id: &str) -> String { + format!("\"_ddb_{table_id}\"") +} + +/// SQL table name for a GSI/LSI data table. +pub(crate) fn index_table_name(index_id: &str) -> String { + format!("\"_ddb_{index_id}\"") +} + +/// Look up all RANGE key attribute definitions from the key schema (preserving order). +pub(crate) fn all_sort_key_info<'a>( + key_schema: &'a [KeySchemaElement], + attr_defs: &'a [AttributeDefinition], +) -> Vec<(&'a str, ScalarAttributeType)> { + key_schema + .iter() + .filter(|ks| ks.key_type == extenddb_core::types::KeyType::Range) + .filter_map(|ks| { + attr_defs + .iter() + .find(|ad| ad.attribute_name == ks.attribute_name) + .map(|ad| (ks.attribute_name.as_str(), ad.attribute_type)) + }) + .collect() +} + +/// Deserialize an `item_data` JSON value into an `Item`. +pub(crate) fn json_to_item(v: serde_json::Value) -> Result { + serde_json::from_value(v).map_err(|e| StorageError::Internal(e.to_string())) +} + +/// Convert BigDecimal to f64 for storage in SQLite REAL column. +pub(crate) fn bigdecimal_to_f64(n: &bigdecimal::BigDecimal) -> f64 { + use bigdecimal::ToPrimitive; + n.to_f64().unwrap_or(0.0) +} + +/// Bind a `SortKeyValue` to a positional parameter in a sqlx query and fetch optional. +/// +/// SQLite uses `?` placeholders. The N variant uses `f64` instead of `BigDecimal`. +macro_rules! bind_sk_fetch_optional { + ($sql:expr, $pk:expr, $sk:expr, $executor:expr) => { + match $sk { + extenddb_storage::util::SortKeyValue::S(s) => { + sqlx::query_as($sql) + .bind($pk) + .bind(s) + .fetch_optional($executor) + .await + } + extenddb_storage::util::SortKeyValue::N(n) => { + sqlx::query_as($sql) + .bind($pk) + .bind(crate::data::bigdecimal_to_f64(n)) + .fetch_optional($executor) + .await + } + extenddb_storage::util::SortKeyValue::B(b) => { + sqlx::query_as($sql) + .bind($pk) + .bind(b) + .fetch_optional($executor) + .await + } + } + .map_err(|e| extenddb_storage::error::StorageError::Internal(e.to_string())) + }; +} + +macro_rules! bind_sk_execute { + ($sql:expr, $pk:expr, $sk:expr, $item_json:expr, $executor:expr) => { + match $sk { + extenddb_storage::util::SortKeyValue::S(s) => { + sqlx::query($sql) + .bind($pk) + .bind(s) + .bind($item_json) + .execute($executor) + .await + } + extenddb_storage::util::SortKeyValue::N(n) => { + sqlx::query($sql) + .bind($pk) + .bind(crate::data::bigdecimal_to_f64(n)) + .bind($item_json) + .execute($executor) + .await + } + extenddb_storage::util::SortKeyValue::B(b) => { + sqlx::query($sql) + .bind($pk) + .bind(b) + .bind($item_json) + .execute($executor) + .await + } + } + .map_err(|e| extenddb_storage::error::StorageError::Internal(e.to_string())) + }; +} + +// Submodules declared after macros so they can use the macros. +mod data_engine; +mod ddl; +mod delete_item; +mod index; +mod put_item; +mod query; +mod query_scan; +mod transactions; +mod tx_helpers; +mod update_item; + +pub(crate) use index::{insert_index_row_multi, project_item_for_index}; +pub(crate) use tx_helpers::upsert_item_in_tx; diff --git a/crates/storage-sqlite/src/data/put_item.rs b/crates/storage-sqlite/src/data/put_item.rs new file mode 100644 index 00000000..37a63a61 --- /dev/null +++ b/crates/storage-sqlite/src/data/put_item.rs @@ -0,0 +1,330 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `put_item` and `get_item` implementations for the SQLite backend. + +use extenddb_core::expression::{Expr, ExpressionMaps}; +use extenddb_core::types::{Item, TableKeyInfo}; +use extenddb_storage::StreamCapture; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{composite_pk_to_text, parse_sk, pk_to_text, sk_column, sk_info}; + +use super::index::{fetch_indexes_for_table, sync_indexes}; +use super::query::check_condition; +use super::tx_helpers::write_stream_record_in_tx; +use super::{data_table_name, json_to_item}; +use crate::engine::SqliteEngine; + +impl SqliteEngine { + pub(crate) async fn put_item_impl( + &self, + key_info: &TableKeyInfo, + item: Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> Result, StorageError> { + let ddb_table = data_table_name(&key_info.table_id); + let pk_text = composite_pk_to_text(&item, &key_info.key_schema)?; + let item_json = + serde_json::to_value(&item).map_err(|e| StorageError::Internal(e.to_string()))?; + + let indexes = fetch_indexes_for_table(&key_info.table_id, &self.pool).await?; + let needs_tx = + condition.is_some() || return_old || !indexes.is_empty() || stream.is_some(); + + if let Some((sk_name, sk_type)) = + sk_info(&key_info.key_schema, &key_info.attribute_definitions) + { + let sk_value = item + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk = parse_sk(sk_value, sk_type)?; + let sk_col = sk_column(sk_type); + + if needs_tx { + let select_sql = + format!("SELECT item_data FROM {ddb_table} WHERE pk = ? AND {sk_col} = ?"); + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let old: Option<(serde_json::Value,)> = + bind_sk_fetch_optional!(&select_sql, pk_text.as_str(), &sk, &mut *tx)?; + + if let Some((ref old_json,)) = old { + let old_item: Item = json_to_item(old_json.clone())?; + match check_condition(condition, &old_item, maps) { + Ok(()) => {} + Err(StorageError::ConditionFailed(_)) => { + return Err(StorageError::ConditionFailed(Some(old_item))); + } + Err(e) => return Err(e), + } + let update_sql = format!( + "UPDATE {ddb_table} SET item_data = ? WHERE pk = ? AND {sk_col} = ?" + ); + match &sk { + extenddb_storage::util::SortKeyValue::S(s) => { + sqlx::query(&update_sql) + .bind(&item_json) + .bind(pk_text.as_str()) + .bind(s) + .execute(&mut *tx) + .await + } + extenddb_storage::util::SortKeyValue::N(n) => { + sqlx::query(&update_sql) + .bind(&item_json) + .bind(pk_text.as_str()) + .bind(super::bigdecimal_to_f64(n)) + .execute(&mut *tx) + .await + } + extenddb_storage::util::SortKeyValue::B(b) => { + sqlx::query(&update_sql) + .bind(&item_json) + .bind(pk_text.as_str()) + .bind(b) + .execute(&mut *tx) + .await + } + } + .map_err(|e| StorageError::Internal(e.to_string()))?; + } else { + let empty = std::collections::BTreeMap::new(); + match check_condition(condition, &empty, maps) { + Ok(()) => {} + Err(StorageError::ConditionFailed(_)) => { + return Err(StorageError::ConditionFailed(None)); + } + Err(e) => return Err(e), + } + let insert_sql = format!( + "INSERT INTO {ddb_table} (pk, {sk_col}, item_data) VALUES (?, ?, ?) \ + ON CONFLICT (pk, {sk_col}) DO NOTHING" + ); + let result = + bind_sk_execute!(&insert_sql, pk_text.as_str(), &sk, &item_json, &mut *tx)?; + if result.rows_affected() == 0 { + let winner: Option<(serde_json::Value,)> = + bind_sk_fetch_optional!(&select_sql, pk_text.as_str(), &sk, &mut *tx)?; + let winner_item = winner.map(|(v,)| json_to_item(v)).transpose()?; + return Err(StorageError::ConditionFailed(winner_item)); + } + } + + if !indexes.is_empty() { + let old_item = old + .as_ref() + .map(|(v,)| json_to_item(v.clone())) + .transpose()?; + sync_indexes( + &mut tx, + &key_info.key_schema, + &key_info.attribute_definitions, + &indexes, + old_item.as_ref(), + Some(&item), + ) + .await?; + } + + if let Some(capture) = stream { + let old_for_stream = old + .as_ref() + .map(|(v,)| json_to_item(v.clone())) + .transpose()?; + write_stream_record_in_tx( + &mut tx, + key_info, + capture, + old_for_stream.as_ref(), + Some(&item), + ) + .await?; + } + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if return_old { + old.map(|(v,)| json_to_item(v)).transpose() + } else { + Ok(None) + } + } else { + let upsert_sql = format!( + "INSERT INTO {ddb_table} (pk, {sk_col}, item_data) VALUES (?, ?, ?) \ + ON CONFLICT (pk, {sk_col}) DO UPDATE SET item_data = EXCLUDED.item_data" + ); + bind_sk_execute!( + &upsert_sql, + pk_text.as_str(), + &sk, + &item_json, + &self.pool + )?; + Ok(None) + } + } else { + // PK-only table + if needs_tx { + let select_sql = format!("SELECT item_data FROM {ddb_table} WHERE pk = ?"); + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let old: Option<(serde_json::Value,)> = sqlx::query_as(&select_sql) + .bind(pk_text.as_str()) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if let Some((ref old_json,)) = old { + let old_item: Item = json_to_item(old_json.clone())?; + match check_condition(condition, &old_item, maps) { + Ok(()) => {} + Err(StorageError::ConditionFailed(_)) => { + return Err(StorageError::ConditionFailed(Some(old_item))); + } + Err(e) => return Err(e), + } + let update_sql = format!("UPDATE {ddb_table} SET item_data = ? WHERE pk = ?"); + sqlx::query(&update_sql) + .bind(&item_json) + .bind(pk_text.as_str()) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } else { + let empty = std::collections::BTreeMap::new(); + match check_condition(condition, &empty, maps) { + Ok(()) => {} + Err(StorageError::ConditionFailed(_)) => { + return Err(StorageError::ConditionFailed(None)); + } + Err(e) => return Err(e), + } + let insert_sql = format!( + "INSERT INTO {ddb_table} (pk, item_data) VALUES (?, ?) \ + ON CONFLICT (pk) DO NOTHING" + ); + let result = sqlx::query(&insert_sql) + .bind(pk_text.as_str()) + .bind(&item_json) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if result.rows_affected() == 0 { + let winner: Option<(serde_json::Value,)> = sqlx::query_as(&select_sql) + .bind(pk_text.as_str()) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let winner_item = winner.map(|(v,)| json_to_item(v)).transpose()?; + return Err(StorageError::ConditionFailed(winner_item)); + } + } + + if !indexes.is_empty() { + let old_item = old + .as_ref() + .map(|(v,)| json_to_item(v.clone())) + .transpose()?; + sync_indexes( + &mut tx, + &key_info.key_schema, + &key_info.attribute_definitions, + &indexes, + old_item.as_ref(), + Some(&item), + ) + .await?; + } + + if let Some(capture) = stream { + let old_for_stream = old + .as_ref() + .map(|(v,)| json_to_item(v.clone())) + .transpose()?; + write_stream_record_in_tx( + &mut tx, + key_info, + capture, + old_for_stream.as_ref(), + Some(&item), + ) + .await?; + } + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if return_old { + old.map(|(v,)| json_to_item(v)).transpose() + } else { + Ok(None) + } + } else { + let upsert_sql = format!( + "INSERT INTO {ddb_table} (pk, item_data) VALUES (?, ?) \ + ON CONFLICT (pk) DO UPDATE SET item_data = EXCLUDED.item_data" + ); + sqlx::query(&upsert_sql) + .bind(pk_text.as_str()) + .bind(&item_json) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(None) + } + } + } + + pub(crate) async fn get_item_impl( + &self, + key_info: &TableKeyInfo, + key: &Item, + ) -> Result, StorageError> { + let ddb_table = data_table_name(&key_info.table_id); + + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_value = key + .get(pk_name) + .ok_or_else(|| StorageError::Internal("missing partition key".to_owned()))?; + let pk_text = pk_to_text(pk_value)?; + + let json_opt = if let Some((sk_name, sk_type)) = + sk_info(&key_info.key_schema, &key_info.attribute_definitions) + { + let sk_value = key + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk = parse_sk(sk_value, sk_type)?; + let sk_col = sk_column(sk_type); + let sql = + format!("SELECT item_data FROM {ddb_table} WHERE pk = ? AND {sk_col} = ?"); + let row: Option<(serde_json::Value,)> = + bind_sk_fetch_optional!(&sql, pk_text.as_ref(), &sk, &self.pool)?; + row.map(|(v,)| v) + } else { + let sql = format!("SELECT item_data FROM {ddb_table} WHERE pk = ?"); + let row: Option<(serde_json::Value,)> = sqlx::query_as(&sql) + .bind(pk_text.as_ref()) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + row.map(|(v,)| v) + }; + + json_opt.map(json_to_item).transpose() + } +} diff --git a/crates/storage-sqlite/src/data/query.rs b/crates/storage-sqlite/src/data/query.rs new file mode 100644 index 00000000..4019c973 --- /dev/null +++ b/crates/storage-sqlite/src/data/query.rs @@ -0,0 +1,180 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Query and scan SQL helpers for the SQLite backend. + +use extenddb_core::expression::{self, Expr, ExpressionMaps, SortKeyCondition}; +use extenddb_core::types::{AttributeValue, Item, KeySchemaElement, extract_key}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::SortKeyValue; +use extenddb_storage::util::parse_sk; + +/// Evaluate a condition expression against an item. +pub(crate) fn check_condition( + condition: Option<&Expr>, + item: &std::collections::BTreeMap, + maps: &ExpressionMaps, +) -> Result<(), StorageError> { + if let Some(cond) = condition { + let passed = expression::evaluate_condition(cond, item, maps) + .map_err(|e| StorageError::Validation(e.to_string()))?; + if !passed { + return Err(StorageError::ConditionFailed(None)); + } + } + Ok(()) +} + +/// Resolve an expression (placeholder) to an `AttributeValue`. +pub(crate) fn resolve_expr_to_av( + expr: &expression::Expr, + maps: &ExpressionMaps, +) -> Result { + match expr { + expression::Expr::Placeholder(name) => maps + .resolve_value(name) + .cloned() + .map_err(|e| StorageError::Validation(e.to_string())), + _ => Err(StorageError::Internal( + "expected placeholder in key condition".to_owned(), + )), + } +} + +/// SQL fragment for a sort key condition. +pub(crate) struct SkSqlInfo { + pub(crate) fragment: String, +} + +/// Build a SQL WHERE fragment for a sort key condition. +/// +/// SQLite uses byte-order comparison for TEXT by default, which matches +/// DynamoDB's UTF-8 byte order for strings. +pub(crate) fn build_sk_sql(sk_cond: &SortKeyCondition, sk_col: &str) -> SkSqlInfo { + match sk_cond { + SortKeyCondition::Compare { op, .. } => { + let sql_op = match op { + expression::CompareOp::Eq => "=", + expression::CompareOp::Ne => "<>", + expression::CompareOp::Lt => "<", + expression::CompareOp::Le => "<=", + expression::CompareOp::Gt => ">", + expression::CompareOp::Ge => ">=", + }; + SkSqlInfo { + fragment: format!(" AND {sk_col} {sql_op} ?"), + } + } + SortKeyCondition::Between { .. } => SkSqlInfo { + fragment: format!(" AND {sk_col} BETWEEN ? AND ?"), + }, + SortKeyCondition::BeginsWith { .. } => { + let is_binary = sk_col == "sk_b" || sk_col.ends_with("_b"); + if is_binary { + SkSqlInfo { + fragment: format!(" AND {sk_col} >= ? AND {sk_col} < ?"), + } + } else { + // For string columns: prefix range using unicode char 1114111 as upper bound. + SkSqlInfo { + fragment: format!( + " AND {sk_col} >= ? AND {sk_col} < (? || char(1114111))" + ), + } + } + } + } +} + +/// Compute the exclusive upper bound for a binary prefix range query. +fn increment_bytes(prefix: &[u8]) -> Vec { + let mut result = prefix.to_vec(); + for i in (0..result.len()).rev() { + if result[i] < 0xFF { + result[i] += 1; + return result; + } + result.pop(); + } + vec![0xFF; 1025] +} + +/// Bind sort key condition values to a query, returning the sk values to bind. +pub(crate) fn sk_condition_bind_values( + sk_cond: &SortKeyCondition, + sk_type: extenddb_core::types::ScalarAttributeType, + maps: &ExpressionMaps, +) -> Result, StorageError> { + match sk_cond { + SortKeyCondition::Compare { value, .. } => { + let av = resolve_expr_to_av(value, maps)?; + Ok(vec![parse_sk(&av, sk_type)?]) + } + SortKeyCondition::BeginsWith { prefix: value, .. } => { + let av = resolve_expr_to_av(value, maps)?; + let sk = parse_sk(&av, sk_type)?; + if sk_type == extenddb_core::types::ScalarAttributeType::B { + let prefix_bytes = match &sk { + SortKeyValue::B(b) => b.clone(), + _ => unreachable!(), + }; + let upper = increment_bytes(&prefix_bytes); + Ok(vec![sk, SortKeyValue::B(upper)]) + } else { + // For string: bind the prefix twice (>= prefix, < prefix || maxchar) + let prefix_str = match &sk { + SortKeyValue::S(s) => s.clone(), + _ => unreachable!(), + }; + Ok(vec![sk, SortKeyValue::S(prefix_str)]) + } + } + SortKeyCondition::Between { low, high, .. } => { + let lo_av = resolve_expr_to_av(low, maps)?; + let hi_av = resolve_expr_to_av(high, maps)?; + Ok(vec![parse_sk(&lo_av, sk_type)?, parse_sk(&hi_av, sk_type)?]) + } + } +} + +/// Build a `LastEvaluatedKey` from an item by extracting key attributes. +pub(crate) fn build_key(item: &Item, key_schema: &[KeySchemaElement]) -> Item { + extract_key(item, key_schema) +} + +/// A bound value for dynamic query building. +pub(crate) enum BoundValue { + Text(String), + Real(f64), + Blob(Vec), +} + +/// Execute a dynamic query with collected bind values. +pub(crate) async fn execute_dynamic_query( + sql: &str, + values: Vec, + pool: &sqlx::SqlitePool, +) -> Result, StorageError> { + let mut query = sqlx::query_as::<_, (serde_json::Value,)>(sql); + for v in values { + query = match v { + BoundValue::Text(s) => query.bind(s), + BoundValue::Real(f) => query.bind(f), + BoundValue::Blob(b) => query.bind(b), + }; + } + let rows: Vec<(serde_json::Value,)> = query + .fetch_all(pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(rows.into_iter().map(|(v,)| v).collect()) +} + +/// Convert a `SortKeyValue` to a `BoundValue`. +pub(crate) fn sk_to_bound(sk: &SortKeyValue) -> BoundValue { + match sk { + SortKeyValue::S(s) => BoundValue::Text(s.clone()), + SortKeyValue::N(n) => BoundValue::Real(super::bigdecimal_to_f64(n)), + SortKeyValue::B(b) => BoundValue::Blob(b.clone()), + } +} diff --git a/crates/storage-sqlite/src/data/query_scan.rs b/crates/storage-sqlite/src/data/query_scan.rs new file mode 100644 index 00000000..37d93460 --- /dev/null +++ b/crates/storage-sqlite/src/data/query_scan.rs @@ -0,0 +1,243 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `query` and `scan` implementations for the SQLite backend. + +use extenddb_core::expression::{ExpressionMaps, KeyCondition}; +use extenddb_core::types::{Item, TableKeyInfo}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{ + encode_netstring_composite, parse_sk, pk_to_text, sk_column, sk_column_n, sk_info, +}; + +use super::query::{ + BoundValue, build_key, build_sk_sql, execute_dynamic_query, resolve_expr_to_av, + sk_condition_bind_values, sk_to_bound, +}; +use super::{all_sort_key_info, data_table_name, index_table_name, json_to_item}; +use crate::engine::SqliteEngine; + +impl SqliteEngine { + #[allow(clippy::too_many_arguments)] + pub(crate) async fn query_impl( + &self, + key_info: &TableKeyInfo, + key_condition: &KeyCondition, + maps: &ExpressionMaps, + forward: bool, + limit: Option, + exclusive_start_key: Option<&Item>, + index_name: Option<&str>, + ) -> Result<(Vec, Option), StorageError> { + use std::fmt::Write; + + let ddb_table = if let Some(idx_name) = index_name { + let idx_info = self + .fetch_index_info_by_table_id(&key_info.table_id, idx_name) + .await?; + index_table_name(&idx_info.index_id) + } else { + data_table_name(&key_info.table_id) + }; + + let pk_text = if key_condition.extra_pk_conditions.is_empty() { + let pk_expr_val = resolve_expr_to_av(&key_condition.pk_value, maps)?; + pk_to_text(&pk_expr_val)?.into_owned() + } else { + let mut parts = Vec::with_capacity(1 + key_condition.extra_pk_conditions.len()); + let first_val = resolve_expr_to_av(&key_condition.pk_value, maps)?; + parts.push(pk_to_text(&first_val)?.into_owned()); + for (_, value) in &key_condition.extra_pk_conditions { + let val = resolve_expr_to_av(value, maps)?; + parts.push(pk_to_text(&val)?.into_owned()); + } + encode_netstring_composite(&parts) + }; + + let sk_info_val = sk_info(&key_info.key_schema, &key_info.attribute_definitions); + let all_sks = all_sort_key_info(&key_info.key_schema, &key_info.attribute_definitions); + + let mut sql = format!("SELECT item_data FROM {ddb_table} WHERE pk = ?"); + let mut bind_values: Vec = vec![BoundValue::Text(pk_text.clone())]; + + // Sort key condition SQL fragment. + if let (Some(sk_cond), Some((_, sk_type))) = (&key_condition.sk_condition, sk_info_val) { + let sk_col = sk_column(sk_type); + let info = build_sk_sql(sk_cond, sk_col); + sql.push_str(&info.fragment); + let vals = sk_condition_bind_values(sk_cond, sk_type, maps)?; + for sk in &vals { + bind_values.push(sk_to_bound(sk)); + } + } + + // Extra RANGE key equality conditions. + for (path, value) in &key_condition.extra_sk_conditions { + let attr_name = match path.first() { + Some(extenddb_core::expression::PathElement::Attribute(name)) => { + if let Some(ref_name) = name.strip_prefix('#') { + match maps.names.get(ref_name) { + Some(resolved) => resolved.clone(), + None => continue, + } + } else { + name.clone() + } + } + _ => continue, + }; + if let Some(pos) = all_sks.iter().position(|(sk_name, _)| *sk_name == attr_name) { + if pos > 0 { + let (_, sk_type) = all_sks[pos]; + let col = sk_column_n(pos, sk_type); + let _ = write!(sql, " AND {col} = ?"); + let av = resolve_expr_to_av(value, maps)?; + let sk = parse_sk(&av, sk_type)?; + bind_values.push(sk_to_bound(&sk)); + } + } + } + + // Pagination: exclusive start key + if let (Some(start_key), Some((sk_name, sk_type))) = (exclusive_start_key, sk_info_val) { + let sk_col = sk_column(sk_type); + if forward { + let _ = write!(sql, " AND {sk_col} > ?"); + } else { + let _ = write!(sql, " AND {sk_col} < ?"); + } + if let Some(sk_val) = start_key.get(sk_name) { + let sk = parse_sk(sk_val, sk_type)?; + bind_values.push(sk_to_bound(&sk)); + } + } else if exclusive_start_key.is_some() && sk_info_val.is_none() { + return Ok((Vec::new(), None)); + } + + // ORDER BY + if let Some((_, sk_type)) = sk_info_val { + let sk_col = sk_column(sk_type); + let dir = if forward { "ASC" } else { "DESC" }; + let _ = write!(sql, " ORDER BY {sk_col} {dir}"); + } + + let fetch_limit = limit.map_or(1_000_001, |l| l + 1); + let _ = write!(sql, " LIMIT {fetch_limit}"); + + let rows = execute_dynamic_query(&sql, bind_values, &self.pool).await?; + + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + let actual_limit = limit.map_or(1_000_000_usize, |l| l.max(0) as usize); + let has_more = rows.len() > actual_limit; + let items: Vec = rows + .into_iter() + .take(actual_limit) + .map(json_to_item) + .collect::, _>>()?; + + let last_key = if has_more { + items + .last() + .map(|item| build_key(item, &key_info.key_schema)) + } else { + None + }; + + Ok((items, last_key)) + } + + pub(crate) async fn scan_impl( + &self, + key_info: &TableKeyInfo, + limit: Option, + exclusive_start_key: Option<&Item>, + segment: Option, + total_segments: Option, + index_name: Option<&str>, + ) -> Result<(Vec, Option), StorageError> { + use std::fmt::Write; + + let ddb_table = if let Some(idx_name) = index_name { + let idx_info = self + .fetch_index_info_by_table_id(&key_info.table_id, idx_name) + .await?; + index_table_name(&idx_info.index_id) + } else { + data_table_name(&key_info.table_id) + }; + let sk_info_val = sk_info(&key_info.key_schema, &key_info.attribute_definitions); + + let mut sql = format!("SELECT item_data FROM {ddb_table}"); + let mut conditions: Vec = Vec::new(); + let mut bind_values: Vec = Vec::new(); + + // Parallel scan: use rowid modulo for segment distribution in SQLite. + if let (Some(seg), Some(total)) = (segment, total_segments) { + conditions.push(format!("(rowid % {total}) = {seg}")); + } + + // Pagination via exclusive start key. + if let Some(start_key) = exclusive_start_key { + let pk_name = &key_info.key_schema[0].attribute_name; + if !start_key.contains_key(pk_name) { + return Err(StorageError::Validation( + "The provided starting key is invalid: The provided key element does not match the schema".to_owned(), + )); + } + let pk_val = start_key.get(pk_name).unwrap(); + let pk_text = pk_to_text(pk_val)?; + + if let Some((sk_name, sk_type)) = sk_info_val { + let sk_col = sk_column(sk_type); + // SQLite doesn't support tuple comparison, so use explicit OR expansion. + conditions.push(format!("(pk > ? OR (pk = ? AND {sk_col} > ?))")); + bind_values.push(BoundValue::Text(pk_text.clone().into_owned())); + bind_values.push(BoundValue::Text(pk_text.into_owned())); + if let Some(sk_val) = start_key.get(sk_name) { + let sk = parse_sk(sk_val, sk_type)?; + bind_values.push(sk_to_bound(&sk)); + } + } else { + conditions.push("pk > ?".to_owned()); + bind_values.push(BoundValue::Text(pk_text.into_owned())); + } + } + + if !conditions.is_empty() { + sql.push_str(" WHERE "); + sql.push_str(&conditions.join(" AND ")); + } + + // ORDER BY + if let Some((_, sk_type)) = sk_info_val { + let sk_col = sk_column(sk_type); + let _ = write!(sql, " ORDER BY pk, {sk_col}"); + } else { + sql.push_str(" ORDER BY pk"); + } + + let fetch_limit = limit.map_or(1_000_001, |l| l + 1); + let _ = write!(sql, " LIMIT {fetch_limit}"); + + let rows = execute_dynamic_query(&sql, bind_values, &self.pool).await?; + + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + let actual_limit = limit.map_or(1_000_000_usize, |l| l.max(0) as usize); + let has_more = rows.len() > actual_limit; + let items: Vec = rows + .into_iter() + .take(actual_limit) + .map(json_to_item) + .collect::, _>>()?; + + let last_key = if has_more { + items + .last() + .map(|item| build_key(item, &key_info.key_schema)) + } else { + None + }; + + Ok((items, last_key)) + } +} diff --git a/crates/storage-sqlite/src/data/transactions.rs b/crates/storage-sqlite/src/data/transactions.rs new file mode 100644 index 00000000..4d23a51d --- /dev/null +++ b/crates/storage-sqlite/src/data/transactions.rs @@ -0,0 +1,393 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Transactional read/write implementations for the SQLite backend. + +use std::collections::HashMap; + +use extenddb_core::expression::{self, ExpressionMaps}; +use extenddb_core::types::{ + AttributeValue, CancellationReason, Item, ReturnValuesOnConditionCheckFailure, +}; +use extenddb_core::validation; +use extenddb_storage::error::StorageError; +use extenddb_storage::{TransactGetOp, TransactWriteOp}; + +use super::index::{IndexMeta, fetch_indexes_for_table, sync_indexes}; +use super::tx_helpers::{ + check_idempotency_token_in_tx, delete_item_in_tx, fetch_item_for_update, fetch_item_in_tx, + upsert_item_in_tx, write_stream_record_in_tx, +}; +use crate::engine::SqliteEngine; + +impl SqliteEngine { + pub(crate) async fn transact_get_items_impl( + &self, + ops: &[TransactGetOp<'_>], + ) -> Result>, StorageError> { + let mut reasons: Vec = Vec::with_capacity(ops.len()); + let mut any_failed = false; + for op in ops { + match validation::validate_key_only( + op.key, + &op.key_info.key_schema, + &op.key_info.attribute_definitions, + ) { + Ok(()) => reasons.push(CancellationReason::none()), + Err(e) => { + any_failed = true; + reasons.push(CancellationReason::validation_error(e.to_string())); + } + } + } + if any_failed { + return Err(StorageError::TransactionCanceled(reasons)); + } + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut results = Vec::with_capacity(ops.len()); + for op in ops { + let item = fetch_item_in_tx(&mut tx, op.key_info, op.key).await?; + results.push(item); + } + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(results) + } + + pub(crate) async fn transact_write_items_impl( + &self, + ops: &[TransactWriteOp<'_>], + token: Option<(&str, &str)>, + ) -> Result<(), StorageError> { + let mut table_indexes: HashMap> = HashMap::new(); + for op in ops { + let name = transact_op_table_name(op); + if !table_indexes.contains_key(name) { + let tid = transact_op_table_id(op); + let indexes = fetch_indexes_for_table(tid, &self.pool).await?; + table_indexes.insert(name.to_owned(), indexes); + } + } + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if let Some((tok, fp)) = token { + check_idempotency_token_in_tx(&mut tx, tok, fp).await?; + } + + let mut reasons: Vec = Vec::with_capacity(ops.len()); + let mut op_items: Vec<(Option, Option)> = Vec::with_capacity(ops.len()); + let mut any_failed = false; + + for op in ops { + let indexes = &table_indexes[transact_op_table_name(op)]; + let reason = + execute_transact_write_op(&mut tx, op, indexes, self.max_item_size_bytes).await; + match reason { + Ok(items) => { + op_items.push(items); + reasons.push(CancellationReason::none()); + } + Err(TxnOpError::Cancel(r)) => { + op_items.push((None, None)); + any_failed = true; + reasons.push(r); + } + Err(TxnOpError::Storage(e)) => { + return Err(StorageError::Internal(e.to_string())); + } + } + } + + if any_failed { + return Err(StorageError::TransactionCanceled(reasons)); + } + + for (op, (old_item, new_item)) in ops.iter().zip(op_items.iter()) { + let capture = match op { + TransactWriteOp::Put { stream, .. } + | TransactWriteOp::Delete { stream, .. } + | TransactWriteOp::Update { stream, .. } => stream.as_ref(), + TransactWriteOp::ConditionCheck { .. } => None, + }; + if let Some(capture) = capture { + write_stream_record_in_tx( + &mut tx, + match op { + TransactWriteOp::Put { key_info, .. } + | TransactWriteOp::Delete { key_info, .. } + | TransactWriteOp::Update { key_info, .. } + | TransactWriteOp::ConditionCheck { key_info, .. } => key_info, + }, + capture, + old_item.as_ref(), + new_item.as_ref(), + ) + .await?; + } + } + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(()) + } + + pub(crate) async fn cleanup_expired_idempotency_tokens_impl( + &self, + max_age_seconds: i64, + ) -> Result { + let result = sqlx::query( + "DELETE FROM idempotency_tokens \ + WHERE created_at < datetime('now', '-' || ? || ' seconds')", + ) + .bind(max_age_seconds) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(result.rows_affected()) + } +} + +fn transact_op_table_name<'a>(op: &'a TransactWriteOp<'_>) -> &'a str { + match op { + TransactWriteOp::Put { key_info, .. } + | TransactWriteOp::Delete { key_info, .. } + | TransactWriteOp::Update { key_info, .. } + | TransactWriteOp::ConditionCheck { key_info, .. } => &key_info.table_name, + } +} + +fn transact_op_table_id<'a>(op: &'a TransactWriteOp<'_>) -> &'a str { + match op { + TransactWriteOp::Put { key_info, .. } + | TransactWriteOp::Delete { key_info, .. } + | TransactWriteOp::Update { key_info, .. } + | TransactWriteOp::ConditionCheck { key_info, .. } => &key_info.table_id, + } +} + +enum TxnOpError { + Cancel(CancellationReason), + Storage(StorageError), +} + +impl From for TxnOpError { + fn from(r: CancellationReason) -> Self { + Self::Cancel(r) + } +} + +async fn execute_transact_write_op( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + op: &TransactWriteOp<'_>, + indexes: &[IndexMeta], + max_item_size_bytes: usize, +) -> Result<(Option, Option), TxnOpError> { + match op { + TransactWriteOp::Put { + key_info, + item, + condition, + maps, + return_values_on_ccf, + .. + } => { + validation::validate_item_keys( + item, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; + let existing = fetch_item_for_update(tx, key_info, item) + .await + .map_err(TxnOpError::Storage)?; + let empty = Item::new(); + eval_condition( + *condition, + existing.as_ref().unwrap_or(&empty), + maps, + *return_values_on_ccf, + existing.as_ref(), + )?; + upsert_item_in_tx(tx, key_info, item) + .await + .map_err(TxnOpError::Storage)?; + if !indexes.is_empty() { + sync_indexes( + tx, + &key_info.key_schema, + &key_info.attribute_definitions, + indexes, + existing.as_ref(), + Some(item), + ) + .await + .map_err(TxnOpError::Storage)?; + } + Ok((existing, Some((*item).clone()))) + } + TransactWriteOp::Delete { + key_info, + key, + condition, + maps, + return_values_on_ccf, + .. + } => { + validation::validate_key_only( + key, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; + let existing = fetch_item_for_update(tx, key_info, key) + .await + .map_err(TxnOpError::Storage)?; + let empty = Item::new(); + eval_condition( + *condition, + existing.as_ref().unwrap_or(&empty), + maps, + *return_values_on_ccf, + existing.as_ref(), + )?; + delete_item_in_tx(tx, key_info, key) + .await + .map_err(TxnOpError::Storage)?; + if !indexes.is_empty() { + sync_indexes( + tx, + &key_info.key_schema, + &key_info.attribute_definitions, + indexes, + existing.as_ref(), + None, + ) + .await + .map_err(TxnOpError::Storage)?; + } + Ok((existing, None)) + } + TransactWriteOp::Update { + key_info, + key, + actions, + condition, + maps, + return_values_on_ccf, + .. + } => { + validation::validate_key_only( + key, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; + let existing = fetch_item_for_update(tx, key_info, key) + .await + .map_err(TxnOpError::Storage)?; + let mut item = existing.clone().unwrap_or_else(|| (*key).clone()); + let condition_item = if existing.is_some() { + &item + } else { + &std::collections::BTreeMap::new() + }; + eval_condition( + *condition, + condition_item, + maps, + *return_values_on_ccf, + existing.as_ref(), + )?; + expression::apply_update(actions, &mut item, maps).map_err(|e| { + TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + validation::validate_item_size(&item, max_item_size_bytes).map_err(|e| { + TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + upsert_item_in_tx(tx, key_info, &item) + .await + .map_err(TxnOpError::Storage)?; + if !indexes.is_empty() { + sync_indexes( + tx, + &key_info.key_schema, + &key_info.attribute_definitions, + indexes, + existing.as_ref(), + Some(&item), + ) + .await + .map_err(TxnOpError::Storage)?; + } + Ok((existing, Some(item))) + } + TransactWriteOp::ConditionCheck { + key_info, + key, + condition, + maps, + return_values_on_ccf, + } => { + validation::validate_key_only( + key, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; + let existing = fetch_item_for_update(tx, key_info, key) + .await + .map_err(TxnOpError::Storage)?; + let empty = Item::new(); + let check_against = existing.as_ref().unwrap_or(&empty); + eval_condition( + Some(condition), + check_against, + maps, + *return_values_on_ccf, + existing.as_ref(), + )?; + Ok((None, None)) + } + } +} + +fn eval_condition( + condition: Option<&extenddb_core::expression::Expr>, + item: &std::collections::BTreeMap, + maps: &ExpressionMaps, + return_values_on_ccf: ReturnValuesOnConditionCheckFailure, + existing: Option<&Item>, +) -> Result<(), CancellationReason> { + if let Some(cond) = condition { + let passed = expression::evaluate_condition(cond, item, maps) + .map_err(|e| CancellationReason::validation_error(e.to_string()))?; + if !passed { + let item_to_return = + if return_values_on_ccf == ReturnValuesOnConditionCheckFailure::AllOld { + existing.cloned() + } else { + None + }; + return Err(CancellationReason::condition_check_failed_with_item( + item_to_return, + )); + } + } + Ok(()) +} diff --git a/crates/storage-sqlite/src/data/tx_helpers.rs b/crates/storage-sqlite/src/data/tx_helpers.rs new file mode 100644 index 00000000..6934ce24 --- /dev/null +++ b/crates/storage-sqlite/src/data/tx_helpers.rs @@ -0,0 +1,331 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Transaction helper functions for the SQLite backend. + +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; +use extenddb_core::types::{ + AttributeValue, Item, StreamEventName, StreamRecord, StreamRecordData, StreamViewType, + TableKeyInfo, item_size_bytes, +}; +use extenddb_storage::StreamCapture; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{SortKeyValue, parse_sk, pk_to_text, sk_column, sk_info}; + +use super::{bigdecimal_to_f64, data_table_name, json_to_item}; + +/// Fetch a single item within an existing transaction (no locking — SQLite serializes writes). +pub(super) async fn fetch_item_in_tx( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + key_info: &TableKeyInfo, + key: &Item, +) -> Result, StorageError> { + let ddb_table = data_table_name(&key_info.table_id); + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_value = key + .get(pk_name) + .ok_or_else(|| StorageError::Internal("missing partition key".to_owned()))?; + let pk_text = pk_to_text(pk_value)?; + + let json_opt = if let Some((sk_name, sk_type)) = + sk_info(&key_info.key_schema, &key_info.attribute_definitions) + { + let sk_value = key + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk = parse_sk(sk_value, sk_type)?; + let sk_col = sk_column(sk_type); + let sql = format!("SELECT item_data FROM {ddb_table} WHERE pk = ? AND {sk_col} = ?"); + let row: Option<(serde_json::Value,)> = + bind_sk_fetch_optional!(&sql, pk_text.as_ref(), &sk, &mut **tx)?; + row.map(|(v,)| v) + } else { + let sql = format!("SELECT item_data FROM {ddb_table} WHERE pk = ?"); + let row: Option<(serde_json::Value,)> = sqlx::query_as(&sql) + .bind(pk_text.as_ref()) + .fetch_optional(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + row.map(|(v,)| v) + }; + + json_opt.map(json_to_item).transpose() +} + +/// Fetch a single item within a transaction for write (SQLite serializes at DB level). +pub(super) async fn fetch_item_for_update( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + key_info: &TableKeyInfo, + key: &Item, +) -> Result, StorageError> { + // SQLite doesn't support FOR UPDATE; writer serialization handles conflicts. + fetch_item_in_tx(tx, key_info, key).await +} + +/// Upsert an item within a transaction. +pub(crate) async fn upsert_item_in_tx( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + key_info: &TableKeyInfo, + item: &Item, +) -> Result<(), StorageError> { + let ddb_table = data_table_name(&key_info.table_id); + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_value = item + .get(pk_name) + .ok_or_else(|| StorageError::Internal("missing partition key".to_owned()))?; + let pk_text = pk_to_text(pk_value)?; + let item_json = + serde_json::to_value(item).map_err(|e| StorageError::Internal(e.to_string()))?; + + if let Some((sk_name, sk_type)) = sk_info(&key_info.key_schema, &key_info.attribute_definitions) + { + let sk_value = item + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk = parse_sk(sk_value, sk_type)?; + let sk_col = sk_column(sk_type); + let sql = format!( + "INSERT INTO {ddb_table} (pk, {sk_col}, item_data) VALUES (?, ?, ?) \ + ON CONFLICT (pk, {sk_col}) DO UPDATE SET item_data = EXCLUDED.item_data" + ); + bind_sk_execute!(&sql, pk_text.as_ref(), &sk, &item_json, &mut **tx)?; + } else { + let sql = format!( + "INSERT INTO {ddb_table} (pk, item_data) VALUES (?, ?) \ + ON CONFLICT (pk) DO UPDATE SET item_data = EXCLUDED.item_data" + ); + sqlx::query(&sql) + .bind(pk_text.as_ref()) + .bind(&item_json) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) +} + +/// Delete an item by key within a transaction. +pub(super) async fn delete_item_in_tx( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + key_info: &TableKeyInfo, + key: &Item, +) -> Result<(), StorageError> { + let ddb_table = data_table_name(&key_info.table_id); + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_value = key + .get(pk_name) + .ok_or_else(|| StorageError::Internal("missing partition key".to_owned()))?; + let pk_text = pk_to_text(pk_value)?; + + if let Some((sk_name, sk_type)) = sk_info(&key_info.key_schema, &key_info.attribute_definitions) + { + let sk_value = key + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk = parse_sk(sk_value, sk_type)?; + let sk_col = sk_column(sk_type); + let sql = format!("DELETE FROM {ddb_table} WHERE pk = ? AND {sk_col} = ?"); + match &sk { + SortKeyValue::S(s) => { + sqlx::query(&sql) + .bind(pk_text.as_ref()) + .bind(s) + .execute(&mut **tx) + .await + } + SortKeyValue::N(n) => { + sqlx::query(&sql) + .bind(pk_text.as_ref()) + .bind(bigdecimal_to_f64(n)) + .execute(&mut **tx) + .await + } + SortKeyValue::B(b) => { + sqlx::query(&sql) + .bind(pk_text.as_ref()) + .bind(b) + .execute(&mut **tx) + .await + } + } + .map_err(|e| StorageError::Internal(e.to_string()))?; + } else { + let sql = format!("DELETE FROM {ddb_table} WHERE pk = ?"); + sqlx::query(&sql) + .bind(pk_text.as_ref()) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) +} + +/// Generate a monotonic sequence number for stream records using the seq_counters table. +async fn next_stream_seq( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, +) -> Result { + sqlx::query("UPDATE seq_counters SET value = value + 1 WHERE name = 'stream'") + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let val: i64 = sqlx::query_scalar("SELECT value FROM seq_counters WHERE name = 'stream'") + .fetch_one(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(val) +} + +/// Write a stream record within an existing transaction. +pub(super) async fn write_stream_record_in_tx( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + key_info: &TableKeyInfo, + capture: &StreamCapture, + old_item: Option<&Item>, + new_item: Option<&Item>, +) -> Result<(), StorageError> { + let source_item = new_item.or(old_item); + let Some(source) = source_item else { + return Ok(()); + }; + + let event = match (old_item, new_item) { + (None, Some(_)) => StreamEventName::Insert, + (Some(_), Some(_)) => StreamEventName::Modify, + (Some(_), None) => StreamEventName::Remove, + (None, None) => return Ok(()), + }; + + let keys: std::collections::BTreeMap = key_info + .key_schema + .iter() + .filter_map(|ks| { + source + .get(&ks.attribute_name) + .map(|v| (ks.attribute_name.clone(), v.clone())) + }) + .collect(); + + let new_image = match capture.view_type { + StreamViewType::NewImage | StreamViewType::NewAndOldImages => new_item.cloned(), + _ => None, + }; + let old_image = match capture.view_type { + StreamViewType::OldImage | StreamViewType::NewAndOldImages => old_item.cloned(), + _ => None, + }; + + let size = source_item.map_or(0, |i| i64::try_from(item_size_bytes(i)).unwrap_or(i64::MAX)); + + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_str = source + .get(pk_name) + .map(|v| match v { + AttributeValue::S(s) => s.clone(), + AttributeValue::N(n) => n.clone(), + AttributeValue::B(b) => BASE64.encode(b), + _ => String::new(), + }) + .unwrap_or_default(); + + let shards: Vec<(String,)> = sqlx::query_as( + "SELECT shard_id FROM stream_shards WHERE table_id = ? ORDER BY shard_id", + ) + .bind(&key_info.table_id) + .fetch_all(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if shards.is_empty() { + return Ok(()); + } + + let hash = crc32fast::hash(pk_str.as_bytes()); + #[allow(clippy::cast_possible_truncation)] + let idx = (hash as usize) % shards.len(); + let shard_id = &shards[idx].0; + + let seq_val = next_stream_seq(tx).await?; + let seq = format!("{seq_val:021}"); + + let record = StreamRecord { + event_id: uuid::Uuid::new_v4().to_string(), + event_name: event, + event_version: "1.1".to_owned(), + event_source: "aws:dynamodb".to_owned(), + aws_region: capture.region.to_string(), + dynamodb: StreamRecordData { + approximate_creation_date_time: i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + ) + .unwrap_or(i64::MAX), + keys, + new_image, + old_image, + sequence_number: seq, + size_bytes: size, + stream_view_type: capture.view_type, + }, + user_identity: capture.user_identity.clone(), + }; + + let record_json = + serde_json::to_string(&record).map_err(|e| StorageError::Internal(e.to_string()))?; + + sqlx::query( + "INSERT INTO stream_records (sequence_number, shard_id, table_id, event_name, record_data) \ + VALUES (?, ?, ?, ?, ?)", + ) + .bind(&record.dynamodb.sequence_number) + .bind(shard_id) + .bind(&key_info.table_id) + .bind(format!("{:?}", record.event_name)) + .bind(&record_json) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(()) +} + +/// Check an idempotency token within an existing transaction. +pub(super) async fn check_idempotency_token_in_tx( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + token: &str, + fingerprint: &str, +) -> Result<(), StorageError> { + // Check for existing valid token (within 10 minutes). + let existing: Option<(String,)> = sqlx::query_as( + "SELECT fingerprint FROM idempotency_tokens \ + WHERE token = ? AND created_at > datetime('now', '-10 minutes')", + ) + .bind(token) + .fetch_optional(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if let Some((stored_fp,)) = existing { + if stored_fp == fingerprint { + return Err(StorageError::IdempotentReplay); + } else { + return Err(StorageError::IdempotentMismatch); + } + } + + // No valid existing token — insert (or replace expired token). + sqlx::query( + "INSERT INTO idempotency_tokens (token, fingerprint) VALUES (?, ?) \ + ON CONFLICT (token) DO UPDATE SET fingerprint = EXCLUDED.fingerprint, \ + created_at = datetime('now')", + ) + .bind(token) + .bind(fingerprint) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(()) +} diff --git a/crates/storage-sqlite/src/data/update_item.rs b/crates/storage-sqlite/src/data/update_item.rs new file mode 100644 index 00000000..084293c5 --- /dev/null +++ b/crates/storage-sqlite/src/data/update_item.rs @@ -0,0 +1,234 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `update_item` implementation for the SQLite backend. + +use extenddb_core::expression::{self, Expr, ExpressionMaps, UpdateAction}; +use extenddb_core::types::{Item, KeyType, TableKeyInfo}; +use extenddb_core::validation; +use extenddb_storage::StreamCapture; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{parse_sk, pk_to_text, sk_column, sk_info}; + +use super::index::{fetch_indexes_for_table, sync_indexes}; +use super::query::check_condition; +use super::tx_helpers::write_stream_record_in_tx; +use super::{bigdecimal_to_f64, data_table_name, json_to_item}; +use crate::engine::SqliteEngine; + +impl SqliteEngine { + #[allow(clippy::too_many_arguments)] + pub(crate) async fn update_item_impl( + &self, + key_info: &TableKeyInfo, + key: &Item, + actions: &[UpdateAction], + return_old: bool, + return_new: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> Result<(Option, Option), StorageError> { + let ddb_table = data_table_name(&key_info.table_id); + + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_value = key + .get(pk_name) + .ok_or_else(|| StorageError::Internal("missing partition key".to_owned()))?; + let pk_text = pk_to_text(pk_value)?; + + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let indexes = fetch_indexes_for_table(&key_info.table_id, &self.pool).await?; + + let old_json = if let Some((sk_name, sk_type)) = + sk_info(&key_info.key_schema, &key_info.attribute_definitions) + { + let sk_value = key + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk = parse_sk(sk_value, sk_type)?; + let sk_col = sk_column(sk_type); + let select_sql = + format!("SELECT item_data FROM {ddb_table} WHERE pk = ? AND {sk_col} = ?"); + let row: Option<(serde_json::Value,)> = + bind_sk_fetch_optional!(&select_sql, pk_text.as_ref(), &sk, &mut *tx)?; + row.map(|(v,)| v) + } else { + let select_sql = format!("SELECT item_data FROM {ddb_table} WHERE pk = ?"); + let row: Option<(serde_json::Value,)> = sqlx::query_as(&select_sql) + .bind(pk_text.as_ref()) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + row.map(|(v,)| v) + }; + + let mut item = if let Some(json) = old_json.clone() { + json_to_item(json)? + } else { + key.clone() + }; + + let pre_mutation_item = + if (!indexes.is_empty() || stream.is_some()) && old_json.is_some() { + Some(item.clone()) + } else { + None + }; + + let old_item = if return_old { Some(item.clone()) } else { None }; + + let condition_item = if old_json.is_some() { + &item + } else { + &std::collections::BTreeMap::new() + }; + match check_condition(condition, condition_item, maps) { + Ok(()) => {} + Err(StorageError::ConditionFailed(_)) => { + if old_json.is_some() { + return Err(StorageError::ConditionFailed(Some(item))); + } + return Err(StorageError::ConditionFailed(None)); + } + Err(e) => return Err(e), + } + + expression::apply_update(actions, &mut item, maps) + .map_err(|e| StorageError::Validation(e.to_string()))?; + + validation::validate_item_size(&item, self.max_item_size_bytes) + .map_err(|e| StorageError::Validation(e.to_string()))?; + + let new_item = if return_new { Some(item.clone()) } else { None }; + + let item_json = + serde_json::to_value(&item).map_err(|e| StorageError::Internal(e.to_string()))?; + + if let Some((_, sk_type)) = sk_info(&key_info.key_schema, &key_info.attribute_definitions) { + let sk_name_ref = key_info + .key_schema + .iter() + .find(|ks| ks.key_type == KeyType::Range) + .map(|ks| ks.attribute_name.as_str()) + .ok_or_else(|| StorageError::Internal("missing sort key schema".to_owned()))?; + let sk_value = key + .get(sk_name_ref) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk = parse_sk(sk_value, sk_type)?; + let sk_col = sk_column(sk_type); + if old_json.is_some() { + let update_sql = format!( + "UPDATE {ddb_table} SET item_data = ? WHERE pk = ? AND {sk_col} = ?" + ); + match &sk { + extenddb_storage::util::SortKeyValue::S(s) => { + sqlx::query(&update_sql) + .bind(&item_json) + .bind(pk_text.as_ref()) + .bind(s) + .execute(&mut *tx) + .await + } + extenddb_storage::util::SortKeyValue::N(n) => { + sqlx::query(&update_sql) + .bind(&item_json) + .bind(pk_text.as_ref()) + .bind(bigdecimal_to_f64(n)) + .execute(&mut *tx) + .await + } + extenddb_storage::util::SortKeyValue::B(b) => { + sqlx::query(&update_sql) + .bind(&item_json) + .bind(pk_text.as_ref()) + .bind(b) + .execute(&mut *tx) + .await + } + } + .map_err(|e| StorageError::Internal(e.to_string()))?; + } else { + let insert_sql = format!( + "INSERT INTO {ddb_table} (pk, {sk_col}, item_data) VALUES (?, ?, ?) \ + ON CONFLICT (pk, {sk_col}) DO NOTHING" + ); + let result = + bind_sk_execute!(&insert_sql, pk_text.as_ref(), &sk, &item_json, &mut *tx)?; + if result.rows_affected() == 0 { + let winner_sql = format!( + "SELECT item_data FROM {ddb_table} WHERE pk = ? AND {sk_col} = ?" + ); + let winner: Option<(serde_json::Value,)> = + bind_sk_fetch_optional!(&winner_sql, pk_text.as_ref(), &sk, &mut *tx)?; + let winner_item = winner.map(|(v,)| json_to_item(v)).transpose()?; + return Err(StorageError::ConditionFailed(winner_item)); + } + } + } else { + if old_json.is_some() { + let update_sql = format!("UPDATE {ddb_table} SET item_data = ? WHERE pk = ?"); + sqlx::query(&update_sql) + .bind(&item_json) + .bind(pk_text.as_ref()) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } else { + let insert_sql = format!( + "INSERT INTO {ddb_table} (pk, item_data) VALUES (?, ?) \ + ON CONFLICT (pk) DO NOTHING" + ); + let result = sqlx::query(&insert_sql) + .bind(pk_text.as_ref()) + .bind(&item_json) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if result.rows_affected() == 0 { + let winner_sql = format!("SELECT item_data FROM {ddb_table} WHERE pk = ?"); + let winner: Option<(serde_json::Value,)> = sqlx::query_as(&winner_sql) + .bind(pk_text.as_ref()) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let winner_item = winner.map(|(v,)| json_to_item(v)).transpose()?; + return Err(StorageError::ConditionFailed(winner_item)); + } + } + } + + if !indexes.is_empty() { + sync_indexes( + &mut tx, + &key_info.key_schema, + &key_info.attribute_definitions, + &indexes, + pre_mutation_item.as_ref(), + Some(&item), + ) + .await?; + } + + if let Some(capture) = stream { + write_stream_record_in_tx( + &mut tx, + key_info, + capture, + pre_mutation_item.as_ref(), + Some(&item), + ) + .await?; + } + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok((old_item, new_item)) + } +} diff --git a/crates/storage-sqlite/src/delete_table.rs b/crates/storage-sqlite/src/delete_table.rs new file mode 100644 index 00000000..152947ab --- /dev/null +++ b/crates/storage-sqlite/src/delete_table.rs @@ -0,0 +1,114 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `delete_table` implementation for `SqliteEngine`. + +use extenddb_core::types::{DeleteTableInput, TableDescription, TableStatus}; +use extenddb_storage::error::StorageError; + +use crate::engine::SqliteEngine; +use crate::table_helpers::{IndexRow, TableRow}; + +impl SqliteEngine { + pub(crate) async fn delete_table_impl( + &self, + account_id: &str, + input: DeleteTableInput, + ) -> Result { + Self::validate_account_id(account_id)?; + + let row: Option = sqlx::query_as( + "SELECT table_name, key_schema, attribute_definitions, billing_mode, \ + provisioned_throughput, stream_specification, table_status, \ + creation_date_time, \ + table_size_bytes, item_count, table_arn, table_id, \ + deletion_protection_enabled, stream_label \ + FROM tables \ + WHERE account_id = ? AND table_name = ? AND table_status IN ('ACTIVE', 'CREATING')", + ) + .bind(account_id) + .bind(&input.table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let row = row.ok_or_else(|| StorageError::TableNotFound(input.table_name.clone()))?; + + if row.deletion_protection_enabled { + return Err(StorageError::DeletionProtected(row.table_arn.clone())); + } + + let index_rows: Vec = sqlx::query_as( + "SELECT index_name, index_id, index_type, key_schema, projection, \ + index_status, provisioned_throughput \ + FROM indexes WHERE table_id = ?", + ) + .bind(&row.table_id) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let delay_row: Option<(String,)> = sqlx::query_as( + "SELECT value FROM settings WHERE key = 'control_plane_delay_seconds'", + ) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let delay_secs: f64 = delay_row + .and_then(|(v,)| v.parse::().ok()) + .unwrap_or(0.25); + + let index_ids: Vec = index_rows.iter().map(|r| r.index_id.clone()).collect(); + + if delay_secs < 1.0 { + sqlx::query("DELETE FROM tags WHERE resource_arn = ?") + .bind(&row.table_arn) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + sqlx::query("DELETE FROM tables WHERE table_id = ?") + .bind(&row.table_id) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Drop data tables. + let mut data_tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + for idx_id in &index_ids { + Self::drop_index_data_table(&mut data_tx, idx_id).await?; + } + Self::drop_data_table(&mut data_tx, &row.table_id).await?; + data_tx + .commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } else { + let secs = delay_secs as i64; + sqlx::query( + "UPDATE tables SET table_status = 'DELETING', \ + status_transition_at = datetime('now', '+' || ? || ' seconds') \ + WHERE account_id = ? AND table_name = ?", + ) + .bind(secs) + .bind(account_id) + .bind(&input.table_name) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + self.control_plane_notify.notify_one(); + } + + let desc = self.build_table_description_from_row(account_id, row, index_rows)?; + + Ok(TableDescription { + table_status: TableStatus::Deleting, + ..desc + }) + } +} diff --git a/crates/storage-sqlite/src/engine.rs b/crates/storage-sqlite/src/engine.rs new file mode 100644 index 00000000..81b9f2b6 --- /dev/null +++ b/crates/storage-sqlite/src/engine.rs @@ -0,0 +1,130 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! SQLite storage engine struct and construction. + +use std::sync::Arc; + +use extenddb_storage::error::StorageError; +use sqlx::SqlitePool; +use sqlx::sqlite::SqlitePoolOptions; + +/// Expected catalog version for the SQLite backend. +pub const CATALOG_VERSION: extenddb_core::version::CatalogVersion = + extenddb_core::version::CatalogVersion::new(0, 0, 2); + +/// SQLite storage backend. +/// +/// Uses a single SQLite pool for all catalog and data operations. +/// WAL mode is enabled for concurrent reads alongside writes. +pub struct SqliteEngine { + pub(crate) pool: SqlitePool, + pub(crate) region: String, + pub(crate) max_item_size_bytes: usize, + pub(crate) control_plane_notify: Arc, + #[allow(dead_code)] + pub(crate) gsi_default_delay_ms: Arc, +} + +impl SqliteEngine { + pub async fn new(config: &SqliteConfig, region: &str) -> Result { + let pool = SqlitePoolOptions::new() + .max_connections(config.pool_size) + .min_connections(2) + .after_connect(|conn, _| { + Box::pin(async move { + use sqlx::Executor; + conn.execute("PRAGMA journal_mode=WAL").await?; + conn.execute("PRAGMA foreign_keys=ON").await?; + conn.execute("PRAGMA synchronous=NORMAL").await?; + conn.execute("PRAGMA busy_timeout=5000").await?; + conn.execute("PRAGMA cache_size=-32000").await?; + Ok(()) + }) + }) + .connect(&config.connection_string) + .await + .map_err(|e| StorageError::Connection(e.to_string()))?; + + let initial_gsi_delay: u64 = sqlx::query_as::<_, (String,)>( + "SELECT value FROM settings WHERE key = 'gsi_propagation_delay_ms'", + ) + .fetch_optional(&pool) + .await + .ok() + .flatten() + .and_then(|(v,)| v.parse::().ok()) + .unwrap_or(10); + + Ok(Self { + pool, + region: region.to_owned(), + max_item_size_bytes: config.max_item_size_bytes, + control_plane_notify: Arc::new(tokio::sync::Notify::new()), + gsi_default_delay_ms: Arc::new(std::sync::atomic::AtomicU64::new(initial_gsi_delay)), + }) + } + + #[allow(dead_code)] + pub fn control_plane_notify(&self) -> Arc { + Arc::clone(&self.control_plane_notify) + } + + pub(crate) fn validate_account_id(account_id: &str) -> Result<(), StorageError> { + if account_id.contains('"') || account_id.contains('\0') || !account_id.is_ascii() { + return Err(StorageError::Internal( + "account_id contains invalid characters for use in SQL identifiers".to_owned(), + )); + } + Ok(()) + } + + pub async fn check_catalog_version(&self) -> Result<(), StorageError> { + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='settings')", + ) + .fetch_one(&self.pool) + .await + .map_err(|e| StorageError::Connection(e.to_string()))?; + + if !exists { + return Err(StorageError::CatalogNotInitialized); + } + + let row: Option<(String,)> = + sqlx::query_as("SELECT value FROM settings WHERE key = 'catalog_version'") + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Connection(e.to_string()))?; + + let found_str = row.ok_or(StorageError::CatalogNotInitialized)?.0; + + let found = found_str + .parse::() + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if found != CATALOG_VERSION { + return Err(StorageError::CatalogVersionMismatch { + expected: CATALOG_VERSION.to_string(), + found: found_str, + }); + } + + Ok(()) + } + + #[allow(dead_code)] + pub fn pool(&self) -> &SqlitePool { + &self.pool + } +} + +/// Configuration for the SQLite storage backend. +pub struct SqliteConfig { + /// SQLite connection string. e.g. `sqlite:///path/to/db.sqlite` + pub connection_string: String, + /// Maximum pool size. + pub pool_size: u32, + /// Maximum item size in bytes. + pub max_item_size_bytes: usize, +} diff --git a/crates/storage-sqlite/src/lib.rs b/crates/storage-sqlite/src/lib.rs new file mode 100644 index 00000000..893a217d --- /dev/null +++ b/crates/storage-sqlite/src/lib.rs @@ -0,0 +1,268 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! SQLite storage backend for extenddb. +//! +//! Implements the `TableEngine`, `DataEngine`, `MetadataEngine`, `StreamEngine`, +//! `BackupEngine`, and `WorkerStore` traits using SQLite via sqlx. + +mod admin_store; +mod authorization_store; +mod backup_engine; +mod bootstrapper; +mod catalog_store; +pub mod config; +mod create_table; +mod credential_store; +mod data; +mod delete_table; +mod management_store; +mod metadata_engine; +mod migrations; +mod operations; +mod sqlite_util; +mod stream_engine; +mod table_engine; +mod table_helpers; +mod update_table; +mod worker_store; +mod workers; + +pub use bootstrapper::SqliteBootstrapper; +pub use catalog_store::SqliteCatalogStore; +pub use config::SqliteStorageConfig; +pub use credential_store::SqliteCredentialStore; + +// Auto-register the SQLite backend at compile time +inventory::submit! { + extenddb_storage::bootstrapper::BackendRegistration { + name: "sqlite", + factory: |config_path, cli_args| { + Box::pin(async move { + let store = SqliteBootstrapper::from_config(&config_path, &cli_args).await?; + Ok(Box::new(store) as Box) + }) + } + } +} + +// Auto-register SQLite operations engine +inventory::submit! { + extenddb_storage::operations::OperationsEngineRegistration { + name: "sqlite", + operations: &operations::SqliteOperationsEngine, + } +} + +// Auto-register SQLite config deserializer +inventory::submit! { + extenddb_storage::config::StorageConfigRegistration { + backend: "sqlite", + deserializer: |table| { + let config: SqliteStorageConfig = table.clone().try_into() + .map_err(|e: toml::de::Error| format!("Failed to parse sqlite config: {}", e))?; + Ok(Box::new(config) as Box) + }, + } +} + +// Auto-register SQLite settings store factory +inventory::submit! { + extenddb_storage::settings_store::SettingsStoreRegistration { + backend: "sqlite", + factory: |connection_string| { + let connection_string = connection_string.to_string(); + Box::pin(async move { + use sqlx::sqlite::SqlitePoolOptions; + let pool = SqlitePoolOptions::new() + .max_connections(5) + .connect(&connection_string) + .await + .map_err(|e| extenddb_storage::settings_store::SettingsStoreError::ConnectionFailed(e.to_string()))?; + Ok(Box::new(SqliteCatalogStore::new(pool)) as Box) + }) + }, + } +} + +// Auto-register SQLite diagnostics store factory +inventory::submit! { + extenddb_storage::diagnostics_store::DiagnosticsStoreRegistration { + backend: "sqlite", + factory: |connection_string| { + let connection_string = connection_string.to_string(); + Box::pin(async move { + use sqlx::sqlite::SqlitePoolOptions; + let pool = SqlitePoolOptions::new() + .max_connections(5) + .connect(&connection_string) + .await + .map_err(|e| extenddb_storage::diagnostics_store::DiagnosticsStoreError::ConnectionFailed(e.to_string()))?; + Ok(Box::new(SqliteCatalogStore::new(pool)) as Box) + }) + }, + } +} + +use std::sync::Arc; + +use extenddb_storage::error::StorageError; +use engine::SqliteEngine; + +pub use engine::CATALOG_VERSION; + +mod engine; + +use extenddb_auth::BuiltinAuthProvider; +use extenddb_storage::hooks::{ServerRuntimeHooks, WorkerContext}; +use extenddb_storage::server_components::{ + BackendError, ServerComponents, ServerComponentsRegistration, +}; + +/// Backend-specific runtime hooks for SQLite. +struct SqliteRuntimeHooks { + engine: Arc, + control_plane_notify: Arc, +} + +#[async_trait::async_trait] +impl ServerRuntimeHooks for SqliteRuntimeHooks { + async fn spawn_workers(&self, ctx: &WorkerContext) { + // 1. Control plane transitions poller + let storage_for_poller = self.engine.clone(); + let cp_notify = self.control_plane_notify.clone(); + let catalog_store = ctx.catalog_store.clone(); + tokio::spawn(async move { + workers::poll_control_plane_transitions(storage_for_poller, cp_notify, catalog_store) + .await + }); + + // 2. Table size refresh worker + let storage_for_size = self.engine.clone(); + tokio::spawn(async move { workers::table_size_refresh_worker(storage_for_size).await }); + + // 3. Stream record cleanup worker + let storage_for_stream = self.engine.clone(); + let metrics = ctx.metrics.clone(); + tokio::spawn(async move { + workers::stream_record_cleanup_worker(storage_for_stream, metrics).await + }); + + // 4. Idempotency token cleanup worker + let storage_for_token = self.engine.clone(); + let metrics = ctx.metrics.clone(); + tokio::spawn(async move { + workers::idempotency_token_cleanup_worker(storage_for_token, metrics).await + }); + } + + fn backend_info(&self) -> Option { + Some("backend=sqlite".to_owned()) + } +} + +// Register the SQLite backend factory +inventory::submit! { + ServerComponentsRegistration { + backend: "sqlite", + factory: |config, region| { + let path = config.connection_config().to_string(); + let pool_size = config.max_connections(); + let region = region.to_string(); + Box::pin(async move { + let conn_str = if path == ":memory:" { + "sqlite::memory:".to_owned() + } else { + format!("sqlite://{}?mode=rwc", path) + }; + + let sqlite_config = engine::SqliteConfig { + connection_string: conn_str.clone(), + pool_size, + max_item_size_bytes: 400_000, + }; + + let engine = SqliteEngine::new(&sqlite_config, ®ion) + .await + .map_err(|e| BackendError::ConnectionFailed { + backend: "sqlite".to_string(), + details: e.to_string(), + })?; + + engine.check_catalog_version().await.map_err(|e| match e { + StorageError::CatalogVersionMismatch { expected, found } => { + BackendError::CatalogVersionMismatch { expected, found } + } + _ => BackendError::InitializationFailed(e.to_string()), + })?; + + // Recover pending control-plane transitions at startup. + match engine.process_control_plane_transitions().await { + Ok(ref t) if t.is_empty() => {} + Ok(transitions) => { + for (name, transition) in &transitions { + tracing::info!("Recovered table '{name}': {transition}"); + } + } + Err(e) => tracing::error!("Failed to recover control plane transitions: {e}"), + } + + let control_plane_notify = engine.control_plane_notify.clone(); + + let engine = Arc::new(engine); + + // Build the catalog pool (same DB, separate pool for catalog ops). + use sqlx::sqlite::SqlitePoolOptions; + let catalog_pool = SqlitePoolOptions::new() + .max_connections(pool_size) + .min_connections(2) + .after_connect(|conn, _| { + Box::pin(async move { + use sqlx::Executor; + conn.execute("PRAGMA journal_mode=WAL").await?; + conn.execute("PRAGMA foreign_keys=ON").await?; + conn.execute("PRAGMA synchronous=NORMAL").await?; + conn.execute("PRAGMA busy_timeout=5000").await?; + Ok(()) + }) + }) + .connect(&conn_str) + .await + .map_err(|e| BackendError::ConnectionFailed { + backend: "sqlite".to_string(), + details: format!("Failed to create catalog pool: {e}"), + })?; + + // Load encryption key + let enc_key: Option = + sqlx::query_scalar("SELECT value FROM settings WHERE key = 'encryption_key'") + .fetch_optional(&catalog_pool) + .await + .map_err(|e| BackendError::InitializationFailed(format!("Failed to fetch encryption key: {e}")))?; + + let catalog_store = Arc::new(match enc_key { + Some(k) => SqliteCatalogStore::with_encryption_key(catalog_pool.clone(), k), + None => return Err(BackendError::MissingEncryptionKey), + }) as Arc; + + // Create auth provider + let enc_key = extenddb_storage::CatalogStore::cached_encryption_key(&*catalog_store) + .ok_or(BackendError::MissingEncryptionKey)?; + let cred_store = SqliteCredentialStore::new(catalog_pool.clone(), enc_key); + let auth_provider = Arc::new(BuiltinAuthProvider::new(cred_store)); + + let runtime_hooks = Box::new(SqliteRuntimeHooks { + engine: engine.clone(), + control_plane_notify, + }); + + Ok(ServerComponents { + engine, + catalog_store, + auth_provider, + runtime_hooks: Some(runtime_hooks), + }) + }) + }, + } +} diff --git a/crates/storage-sqlite/src/management_store/access_keys.rs b/crates/storage-sqlite/src/management_store/access_keys.rs new file mode 100644 index 00000000..54b676db --- /dev/null +++ b/crates/storage-sqlite/src/management_store/access_keys.rs @@ -0,0 +1,228 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Access key and session operations for `SqliteCatalogStore`. + +use extenddb_storage::management_store::{AccessKeyCreated, OpError, OpResult}; + +use crate::catalog_store::SqliteCatalogStore; +use crate::sqlite_util::{is_fk_violation, is_unique_violation}; + +impl SqliteCatalogStore { + // ── Access keys ──────────────────────────────────────────────── + + pub(crate) async fn create_access_key_impl( + &self, + account_id: &str, + user_name: &str, + ) -> OpResult { + let enc_key: String = if let Some(cached) = self.encryption_key() { + cached.to_string() + } else { + let row: Option<(String,)> = + sqlx::query_as("SELECT value FROM settings WHERE key = 'encryption_key'") + .fetch_optional(self.pool()) + .await + .map_err(|e| { + tracing::error!("create_access_key fetch encryption key: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + row.map(|(v,)| v) + .ok_or_else(|| OpError::Internal("Encryption key not configured".to_owned()))? + }; + + let access_key_id = generate_access_key_id(); + let secret_key = generate_secret_key(); + let encrypted = encrypt_secret(&secret_key, &enc_key, &access_key_id).map_err(|e| { + tracing::error!("create_access_key encryption: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + sqlx::query( + "INSERT INTO access_keys (access_key_id, account_id, user_name, secret_key_encrypted) \ + VALUES (?, ?, ?, ?)", + ) + .bind(&access_key_id) + .bind(account_id) + .bind(user_name) + .bind(&encrypted) + .execute(self.pool()) + .await + .map_err(|e| { + if is_fk_violation(&e) { + OpError::NotFound("User not found".to_owned()) + } else { + tracing::error!("create_access_key failed: {e}"); + OpError::Internal("Database error".to_owned()) + } + })?; + + Ok(AccessKeyCreated { + access_key_id, + secret_access_key: secret_key, + }) + } + + pub(crate) async fn delete_access_key_impl( + &self, + account_id: &str, + user_name: &str, + key_id: &str, + ) -> OpResult<()> { + let result = sqlx::query( + "DELETE FROM access_keys \ + WHERE access_key_id = ? AND account_id = ? AND user_name = ?", + ) + .bind(key_id) + .bind(account_id) + .bind(user_name) + .execute(self.pool()) + .await; + match result { + Ok(r) if r.rows_affected() == 0 => { + Err(OpError::NotFound("Access key not found".to_owned())) + } + Ok(_) => Ok(()), + Err(e) => { + tracing::error!("delete_access_key failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + } + + pub(crate) async fn list_access_keys_impl( + &self, + account_id: &str, + user_name: &str, + ) -> OpResult> { + let rows: Vec<(String, bool, String)> = sqlx::query_as( + "SELECT access_key_id, is_active, created_at FROM access_keys \ + WHERE account_id = ? AND user_name = ? ORDER BY created_at", + ) + .bind(account_id) + .bind(user_name) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("list_access_keys: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + rows.into_iter() + .map(|(kid, active, ts)| { + let created_at = + crate::sqlite_util::parse_timestamp(&ts).map_err(|e| { + tracing::error!("list_access_keys parse_timestamp: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok((kid, active, created_at)) + }) + .collect() + } + + pub(crate) async fn import_access_key_impl( + &self, + account_id: &str, + user_name: &str, + access_key_id: &str, + secret_access_key: &str, + ) -> OpResult<()> { + let enc_key: String = if let Some(cached) = self.encryption_key() { + cached.to_string() + } else { + let row: Option<(String,)> = + sqlx::query_as("SELECT value FROM settings WHERE key = 'encryption_key'") + .fetch_optional(self.pool()) + .await + .map_err(|e| { + tracing::error!("import_access_key fetch encryption key: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + row.map(|(v,)| v) + .ok_or_else(|| OpError::Internal("Encryption key not configured".to_owned()))? + }; + + let encrypted = + encrypt_secret(secret_access_key, &enc_key, access_key_id).map_err(|e| { + tracing::error!("import_access_key encryption: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let result = sqlx::query( + "INSERT INTO access_keys \ + (access_key_id, secret_key_encrypted, account_id, user_name) \ + VALUES (?, ?, ?, ?)", + ) + .bind(access_key_id) + .bind(&encrypted) + .bind(account_id) + .bind(user_name) + .execute(self.pool()) + .await; + match result { + Ok(_) => Ok(()), + Err(e) if is_fk_violation(&e) => { + Err(OpError::NotFound("IAM user not found".to_owned())) + } + Err(e) if is_unique_violation(&e) => Err(OpError::AlreadyExists( + "Access key ID already exists".to_owned(), + )), + Err(e) => { + tracing::error!("import_access_key failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + } +} + +// ── Crypto helpers ────────────────────────────────────────────────────────── + +fn generate_access_key_id() -> String { + const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + let mut rng = rand::rng(); + let suffix: String = (0..8) + .map(|_| CHARSET[rand::Rng::random_range(&mut rng, 0..CHARSET.len())] as char) + .collect(); + format!("AKIAEXTENDDB{suffix}") +} + +fn generate_secret_key() -> String { + const CHARSET: &[u8] = + b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut rng = rand::rng(); + let suffix: String = (0..32) + .map(|_| CHARSET[rand::Rng::random_range(&mut rng, 0..CHARSET.len())] as char) + .collect(); + format!("extenddb{suffix}") +} + +fn encrypt_secret(plaintext: &str, key_b64: &str, aad: &str) -> Result, String> { + use aes_gcm::Aes256Gcm; + use aes_gcm::KeyInit; + use aes_gcm::aead::Aead; + use aes_gcm::aead::Payload; + use base64::Engine; + + let key_bytes = base64::engine::general_purpose::STANDARD + .decode(key_b64) + .map_err(|e| format!("decode encryption key: {e}"))?; + + let key = aes_gcm::Key::::from_slice(&key_bytes); + let cipher = Aes256Gcm::new(key); + + let nonce_bytes: [u8; 12] = rand::random(); + let nonce = aes_gcm::Nonce::from_slice(&nonce_bytes); + + let payload = Payload { + msg: plaintext.as_bytes(), + aad: aad.as_bytes(), + }; + let ciphertext = cipher + .encrypt(nonce, payload) + .map_err(|e| format!("encrypt: {e}"))?; + + let mut result = Vec::with_capacity(12 + ciphertext.len()); + result.extend_from_slice(&nonce_bytes); + result.extend_from_slice(&ciphertext); + Ok(result) +} diff --git a/crates/storage-sqlite/src/management_store/accounts.rs b/crates/storage-sqlite/src/management_store/accounts.rs new file mode 100644 index 00000000..65805eb4 --- /dev/null +++ b/crates/storage-sqlite/src/management_store/accounts.rs @@ -0,0 +1,219 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Account management operations for `SqliteCatalogStore`. + +use extenddb_storage::management_store::{AccountDetail, OpError, OpResult}; + +use crate::catalog_store::SqliteCatalogStore; +use crate::sqlite_util::is_unique_violation; + +impl SqliteCatalogStore { + pub(crate) async fn create_account_impl( + &self, + account_id: &str, + account_name: &str, + ) -> OpResult<()> { + let result = + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, ?)") + .bind(account_id) + .bind(account_name) + .execute(self.pool()) + .await; + match result { + Ok(_) => Ok(()), + Err(e) if is_unique_violation(&e) => { + Err(OpError::AlreadyExists("Account already exists".to_owned())) + } + Err(e) => { + tracing::error!("create_account failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + } + + pub(crate) async fn delete_account_impl(&self, account_id: &str) -> OpResult<()> { + let mut tx = self.pool().begin().await.map_err(|e| { + tracing::error!("delete_account begin transaction: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let exists: Option<(String,)> = + sqlx::query_as("SELECT account_id FROM accounts WHERE account_id = ?") + .bind(account_id) + .fetch_optional(&mut *tx) + .await + .map_err(|e| { + tracing::error!("delete_account check account: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + if exists.is_none() { + return Err(OpError::NotFound("Account not found".to_owned())); + } + + let (has_tables,): (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM tables WHERE account_id = ?") + .bind(account_id) + .fetch_one(&mut *tx) + .await + .map_err(|e| { + tracing::error!("delete_account check tables: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + if has_tables > 0 { + return Err(OpError::HasDependents( + "Cannot delete account with existing tables. Delete all tables first.".to_owned(), + )); + } + + let r = sqlx::query("DELETE FROM accounts WHERE account_id = ?") + .bind(account_id) + .execute(&mut *tx) + .await + .map_err(|e| { + tracing::error!("delete_account delete: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + if r.rows_affected() == 0 { + return Err(OpError::NotFound("Account not found".to_owned())); + } + + tx.commit().await.map_err(|e| { + tracing::error!("delete_account commit: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + Ok(()) + } + + pub(crate) async fn list_all_accounts_impl(&self) -> OpResult> { + sqlx::query_as("SELECT account_id, account_name FROM accounts ORDER BY account_id") + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("list_all_accounts: {e}"); + OpError::Internal("Database error".to_owned()) + }) + } + + pub(crate) async fn list_all_accounts_full_impl( + &self, + ) -> OpResult> { + let rows: Vec<(String, String, String)> = sqlx::query_as( + "SELECT account_id, account_name, created_at FROM accounts ORDER BY account_id", + ) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("list_all_accounts_full: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + rows.into_iter() + .map(|(id, name, ts)| { + let created_at = + crate::sqlite_util::parse_timestamp(&ts).map_err(|e| { + tracing::error!("list_all_accounts_full parse_timestamp: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok((id, name, created_at)) + }) + .collect() + } + + pub(crate) async fn list_accounts_for_impl( + &self, + account_id: &str, + ) -> OpResult> { + sqlx::query_as("SELECT account_id, account_name FROM accounts WHERE account_id = ?") + .bind(account_id) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("list_accounts_for: {e}"); + OpError::Internal("Database error".to_owned()) + }) + } + + pub(crate) async fn get_account_detail_impl( + &self, + account_id: &str, + ) -> OpResult> { + let acct: Option<(String,)> = + sqlx::query_as("SELECT account_name FROM accounts WHERE account_id = ?") + .bind(account_id) + .fetch_optional(self.pool()) + .await + .map_err(|e| { + tracing::error!("get_account_detail name: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let Some((account_name,)) = acct else { + return Ok(None); + }; + + let users: Vec<(String,)> = sqlx::query_as( + "SELECT user_name FROM iam_users WHERE account_id = ? ORDER BY user_name", + ) + .bind(account_id) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("get_account_detail users: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let groups: Vec<(String,)> = sqlx::query_as( + "SELECT group_name FROM iam_groups WHERE account_id = ? ORDER BY group_name", + ) + .bind(account_id) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("get_account_detail groups: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let roles: Vec<(String,)> = sqlx::query_as( + "SELECT role_name FROM iam_roles WHERE account_id = ? ORDER BY role_name", + ) + .bind(account_id) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("get_account_detail roles: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + Ok(Some(AccountDetail { + account_name, + users: users.into_iter().map(|(n,)| n).collect(), + groups: groups.into_iter().map(|(n,)| n).collect(), + roles: roles.into_iter().map(|(n,)| n).collect(), + })) + } + + pub(crate) async fn dashboard_counts_impl(&self) -> OpResult<(i64, i64)> { + let (account_count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM accounts") + .fetch_one(self.pool()) + .await + .map_err(|e| { + tracing::error!("dashboard_counts accounts: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let (admin_count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM admin_users") + .fetch_one(self.pool()) + .await + .map_err(|e| { + tracing::error!("dashboard_counts admins: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + Ok((account_count, admin_count)) + } +} diff --git a/crates/storage-sqlite/src/management_store/groups.rs b/crates/storage-sqlite/src/management_store/groups.rs new file mode 100644 index 00000000..e4537467 --- /dev/null +++ b/crates/storage-sqlite/src/management_store/groups.rs @@ -0,0 +1,212 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Group management operations for `SqliteCatalogStore`. + +use extenddb_storage::management_store::{GroupDetail, GroupListEntry, OpError, OpResult}; + +use crate::catalog_store::SqliteCatalogStore; +use crate::sqlite_util::{is_fk_violation, is_unique_violation}; + +impl SqliteCatalogStore { + pub(crate) async fn create_group_impl( + &self, + account_id: &str, + group_name: &str, + ) -> OpResult<()> { + let group_arn = format!("arn:aws:iam::{account_id}:group/{group_name}"); + let result = sqlx::query( + "INSERT INTO iam_groups (account_id, group_name, group_arn) VALUES (?, ?, ?)", + ) + .bind(account_id) + .bind(group_name) + .bind(&group_arn) + .execute(self.pool()) + .await; + match result { + Ok(_) => Ok(()), + Err(e) if is_unique_violation(&e) => Err(OpError::AlreadyExists( + "IAM group already exists".to_owned(), + )), + Err(e) if is_fk_violation(&e) => { + Err(OpError::NotFound("Account not found".to_owned())) + } + Err(e) => { + tracing::error!("create_group failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + } + + pub(crate) async fn delete_group_impl( + &self, + account_id: &str, + group_name: &str, + ) -> OpResult<()> { + let result = + sqlx::query("DELETE FROM iam_groups WHERE account_id = ? AND group_name = ?") + .bind(account_id) + .bind(group_name) + .execute(self.pool()) + .await; + match result { + Ok(r) if r.rows_affected() == 0 => { + Err(OpError::NotFound("IAM group not found".to_owned())) + } + Ok(_) => Ok(()), + Err(e) => { + tracing::error!("delete_group failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + } + + pub(crate) async fn list_groups_impl( + &self, + account_id: &str, + ) -> OpResult> { + let rows: Vec<(String, String, String, String)> = sqlx::query_as( + "SELECT account_id, group_name, group_arn, created_at \ + FROM iam_groups WHERE account_id = ? ORDER BY group_name", + ) + .bind(account_id) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("list_groups: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + rows.into_iter() + .map(|(aid, gn, arn, ts)| { + let created_at = + crate::sqlite_util::parse_timestamp(&ts).map_err(|e| { + tracing::error!("list_groups parse_timestamp: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok((aid, gn, arn, created_at)) + }) + .collect() + } + + pub(crate) async fn get_group_detail_impl( + &self, + account_id: &str, + group_name: &str, + ) -> OpResult> { + let exists: Option<(String,)> = sqlx::query_as( + "SELECT group_name FROM iam_groups WHERE account_id = ? AND group_name = ?", + ) + .bind(account_id) + .bind(group_name) + .fetch_optional(self.pool()) + .await + .map_err(|e| { + tracing::error!("get_group_detail exists: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + if exists.is_none() { + return Ok(None); + } + + let members: Vec<(String,)> = sqlx::query_as( + "SELECT user_name FROM iam_group_members \ + WHERE account_id = ? AND group_name = ? ORDER BY user_name", + ) + .bind(account_id) + .bind(group_name) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("get_group_detail members: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let policies: Vec<(String,)> = sqlx::query_as( + "SELECT policy_name FROM iam_policies \ + WHERE account_id = ? AND principal_type = 'group' AND principal_name = ? \ + ORDER BY policy_name", + ) + .bind(account_id) + .bind(group_name) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("get_group_detail policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let all_users: Vec<(String,)> = + sqlx::query_as("SELECT user_name FROM iam_users WHERE account_id = ? ORDER BY user_name") + .bind(account_id) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("get_group_detail all_users: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + Ok(Some(GroupDetail { + members: members.into_iter().map(|(n,)| n).collect(), + policies: policies.into_iter().map(|(n,)| n).collect(), + all_users: all_users.into_iter().map(|(n,)| n).collect(), + })) + } + + pub(crate) async fn add_group_member_impl( + &self, + account_id: &str, + group_name: &str, + user_name: &str, + ) -> OpResult<()> { + let result = sqlx::query( + "INSERT INTO iam_group_members (account_id, group_name, user_name) VALUES (?, ?, ?)", + ) + .bind(account_id) + .bind(group_name) + .bind(user_name) + .execute(self.pool()) + .await; + match result { + Ok(_) => Ok(()), + Err(e) if is_unique_violation(&e) => Err(OpError::AlreadyExists( + "User is already a member of this group".to_owned(), + )), + Err(e) if is_fk_violation(&e) => { + Err(OpError::NotFound("Group or user not found".to_owned())) + } + Err(e) => { + tracing::error!("add_group_member failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + } + + pub(crate) async fn remove_group_member_impl( + &self, + account_id: &str, + group_name: &str, + user_name: &str, + ) -> OpResult<()> { + let result = sqlx::query( + "DELETE FROM iam_group_members \ + WHERE account_id = ? AND group_name = ? AND user_name = ?", + ) + .bind(account_id) + .bind(group_name) + .bind(user_name) + .execute(self.pool()) + .await; + match result { + Ok(r) if r.rows_affected() == 0 => { + Err(OpError::NotFound("Membership not found".to_owned())) + } + Ok(_) => Ok(()), + Err(e) => { + tracing::error!("remove_group_member failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + } +} diff --git a/crates/storage-sqlite/src/management_store/mod.rs b/crates/storage-sqlite/src/management_store/mod.rs new file mode 100644 index 00000000..b84223c8 --- /dev/null +++ b/crates/storage-sqlite/src/management_store/mod.rs @@ -0,0 +1,582 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! IAM management store trait implementation for `SqliteCatalogStore`. + +mod access_keys; +mod accounts; +mod groups; +mod policies; +mod roles; +mod users; + +use extenddb_storage::management_store::{ + AccessKeyCreated, AccountDetail, GroupDetail, ManagementStore, OpError, OpResult, RoleDetail, + UserDetail, GroupListEntry, RoleListEntry, UserListEntry, +}; +use futures::future::BoxFuture; + +use crate::catalog_store::SqliteCatalogStore; + +impl ManagementStore for SqliteCatalogStore { + // ── Accounts ─────────────────────────────────────────────────── + + fn create_account(&self, account_id: &str, account_name: &str) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let account_name = account_name.to_owned(); + Box::pin(async move { self.create_account_impl(&account_id, &account_name).await }) + } + + fn delete_account(&self, account_id: &str) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + Box::pin(async move { self.delete_account_impl(&account_id).await }) + } + + fn list_all_accounts(&self) -> BoxFuture<'_, OpResult>> { + Box::pin(async move { self.list_all_accounts_impl().await }) + } + + fn list_all_accounts_full( + &self, + ) -> BoxFuture<'_, OpResult>> { + Box::pin(async move { self.list_all_accounts_full_impl().await }) + } + + fn list_accounts_for( + &self, + account_id: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + Box::pin(async move { self.list_accounts_for_impl(&account_id).await }) + } + + fn get_account_detail( + &self, + account_id: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + Box::pin(async move { self.get_account_detail_impl(&account_id).await }) + } + + fn dashboard_counts(&self) -> BoxFuture<'_, OpResult<(i64, i64)>> { + Box::pin(async move { self.dashboard_counts_impl().await }) + } + + // ── Users ────────────────────────────────────────────────────── + + fn create_user( + &self, + account_id: &str, + user_name: &str, + password_hash: Option<&str>, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let password_hash = password_hash.map(|s| s.to_owned()); + Box::pin( + async move { self.create_user_impl(&account_id, &user_name, password_hash.as_deref()).await }, + ) + } + + fn delete_user(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { self.delete_user_impl(&account_id, &user_name).await }) + } + + fn list_users(&self, account_id: &str) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + Box::pin(async move { self.list_users_impl(&account_id).await }) + } + + fn get_user_detail( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { self.get_user_detail_impl(&account_id, &user_name).await }) + } + + fn verify_iam_user_password( + &self, + account_id: &str, + user_name: &str, + password: &str, + ) -> BoxFuture<'_, OpResult> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let password = password.to_owned(); + Box::pin( + async move { self.verify_iam_user_password_impl(&account_id, &user_name, &password).await }, + ) + } + + fn change_user_password( + &self, + account_id: &str, + user_name: &str, + password_hash: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let password_hash = password_hash.to_owned(); + Box::pin(async move { + self.change_user_password_impl(&account_id, &user_name, &password_hash) + .await + }) + } + + // ── User tags ────────────────────────────────────────────────── + + fn tag_user( + &self, + account_id: &str, + user_name: &str, + tags: &[(String, String)], + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let tags = tags.to_vec(); + Box::pin(async move { self.tag_user_impl(&account_id, &user_name, &tags).await }) + } + + fn untag_user( + &self, + account_id: &str, + user_name: &str, + tag_keys: &[String], + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let tag_keys = tag_keys.to_vec(); + Box::pin(async move { self.untag_user_impl(&account_id, &user_name, &tag_keys).await }) + } + + fn list_user_tags( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { self.list_user_tags_impl(&account_id, &user_name).await }) + } + + // ── Groups ───────────────────────────────────────────────────── + + fn create_group(&self, account_id: &str, group_name: &str) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let group_name = group_name.to_owned(); + Box::pin(async move { self.create_group_impl(&account_id, &group_name).await }) + } + + fn delete_group(&self, account_id: &str, group_name: &str) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let group_name = group_name.to_owned(); + Box::pin(async move { self.delete_group_impl(&account_id, &group_name).await }) + } + + fn list_groups(&self, account_id: &str) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + Box::pin(async move { self.list_groups_impl(&account_id).await }) + } + + fn get_group_detail( + &self, + account_id: &str, + group_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let group_name = group_name.to_owned(); + Box::pin(async move { self.get_group_detail_impl(&account_id, &group_name).await }) + } + + fn add_group_member( + &self, + account_id: &str, + group_name: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let group_name = group_name.to_owned(); + let user_name = user_name.to_owned(); + Box::pin( + async move { self.add_group_member_impl(&account_id, &group_name, &user_name).await }, + ) + } + + fn remove_group_member( + &self, + account_id: &str, + group_name: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let group_name = group_name.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + self.remove_group_member_impl(&account_id, &group_name, &user_name) + .await + }) + } + + // ── Roles ────────────────────────────────────────────────────── + + fn create_role( + &self, + account_id: &str, + role_name: &str, + trust_policy: &serde_json::Value, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let trust_policy = trust_policy.clone(); + Box::pin( + async move { self.create_role_impl(&account_id, &role_name, &trust_policy).await }, + ) + } + + fn delete_role(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { self.delete_role_impl(&account_id, &role_name).await }) + } + + fn list_roles(&self, account_id: &str) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + Box::pin(async move { self.list_roles_impl(&account_id).await }) + } + + fn get_role_detail( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { self.get_role_detail_impl(&account_id, &role_name).await }) + } + + fn get_role_trust_policy( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { self.get_role_trust_policy_impl(&account_id, &role_name).await }) + } + + // ── Role tags ────────────────────────────────────────────────── + + fn tag_role( + &self, + account_id: &str, + role_name: &str, + tags: &[(String, String)], + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let tags = tags.to_vec(); + Box::pin(async move { self.tag_role_impl(&account_id, &role_name, &tags).await }) + } + + fn untag_role( + &self, + account_id: &str, + role_name: &str, + tag_keys: &[String], + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let tag_keys = tag_keys.to_vec(); + Box::pin(async move { self.untag_role_impl(&account_id, &role_name, &tag_keys).await }) + } + + fn list_role_tags( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { self.list_role_tags_impl(&account_id, &role_name).await }) + } + + // ── Policies ─────────────────────────────────────────────────── + + fn put_policy( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + policy_name: &str, + document: &serde_json::Value, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let principal_type = principal_type.to_owned(); + let principal_name = principal_name.to_owned(); + let policy_name = policy_name.to_owned(); + let document = document.clone(); + Box::pin(async move { + self.put_policy_impl(&account_id, &principal_type, &principal_name, &policy_name, &document) + .await + }) + } + + fn delete_policy( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + policy_name: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let principal_type = principal_type.to_owned(); + let principal_name = principal_name.to_owned(); + let policy_name = policy_name.to_owned(); + Box::pin(async move { + self.delete_policy_impl(&account_id, &principal_type, &principal_name, &policy_name) + .await + }) + } + + fn list_policies( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let principal_type = principal_type.to_owned(); + let principal_name = principal_name.to_owned(); + Box::pin(async move { + self.list_policies_impl(&account_id, &principal_type, &principal_name) + .await + }) + } + + // ── Permissions boundaries ───────────────────────────────────── + + fn set_user_boundary( + &self, + account_id: &str, + user_name: &str, + document: &serde_json::Value, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let document = document.clone(); + Box::pin( + async move { self.set_user_boundary_impl(&account_id, &user_name, &document).await }, + ) + } + + fn get_user_boundary( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { self.get_user_boundary_impl(&account_id, &user_name).await }) + } + + fn delete_user_boundary( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { self.delete_user_boundary_impl(&account_id, &user_name).await }) + } + + fn set_role_boundary( + &self, + account_id: &str, + role_name: &str, + document: &serde_json::Value, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let document = document.clone(); + Box::pin( + async move { self.set_role_boundary_impl(&account_id, &role_name, &document).await }, + ) + } + + fn get_role_boundary( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { self.get_role_boundary_impl(&account_id, &role_name).await }) + } + + fn delete_role_boundary( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { self.delete_role_boundary_impl(&account_id, &role_name).await }) + } + + // ── Access keys ──────────────────────────────────────────────── + + fn create_access_key( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { self.create_access_key_impl(&account_id, &user_name).await }) + } + + fn delete_access_key( + &self, + account_id: &str, + user_name: &str, + key_id: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let key_id = key_id.to_owned(); + Box::pin( + async move { self.delete_access_key_impl(&account_id, &user_name, &key_id).await }, + ) + } + + fn list_access_keys( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { self.list_access_keys_impl(&account_id, &user_name).await }) + } + + fn import_access_key( + &self, + account_id: &str, + user_name: &str, + access_key_id: &str, + secret_access_key: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let access_key_id = access_key_id.to_owned(); + let secret_access_key = secret_access_key.to_owned(); + Box::pin(async move { + self.import_access_key_impl(&account_id, &user_name, &access_key_id, &secret_access_key) + .await + }) + } + + // ── Sessions ─────────────────────────────────────────────────── + + #[allow(clippy::too_many_arguments)] + fn store_session( + &self, + session_token: &str, + access_key_id: &str, + secret_key_encrypted: &[u8], + account_id: &str, + role_name: &str, + session_name: &str, + session_tags: &Option, + session_policy: &Option, + expires_at: time::OffsetDateTime, + ) -> BoxFuture<'_, OpResult<()>> { + let session_token = session_token.to_owned(); + let access_key_id = access_key_id.to_owned(); + let secret_key_encrypted = secret_key_encrypted.to_vec(); + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let session_name = session_name.to_owned(); + let session_tags = session_tags.clone(); + let session_policy = session_policy.clone(); + let pool = self.pool().clone(); + Box::pin(async move { + let expires_str = crate::sqlite_util::format_timestamp(expires_at); + let tags_json = session_tags + .as_ref() + .map(|v| serde_json::to_string(v).unwrap_or_default()); + let policy_json = session_policy + .as_ref() + .map(|v| serde_json::to_string(v).unwrap_or_default()); + + sqlx::query( + "INSERT INTO iam_sessions \ + (session_token, access_key_id, secret_key_encrypted, account_id, role_name, \ + session_name, session_tags, session_policy, expires_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(&session_token) + .bind(&access_key_id) + .bind(&secret_key_encrypted) + .bind(&account_id) + .bind(&role_name) + .bind(&session_name) + .bind(tags_json.as_deref()) + .bind(policy_json.as_deref()) + .bind(&expires_str) + .execute(&pool) + .await + .map_err(|e| { + tracing::error!("store_session: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(()) + }) + } + + // ── Caller tags ──────────────────────────────────────────────── + + fn fetch_caller_tags( + &self, + account_id: &str, + resource: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let resource = resource.to_owned(); + let pool = self.pool().clone(); + Box::pin(async move { + // resource can be a user ARN or role ARN + let user_tags: Vec<(String, String)> = sqlx::query_as( + "SELECT tag_key, tag_value FROM iam_user_tags \ + WHERE account_id = ? AND user_name = ? ORDER BY tag_key", + ) + .bind(&account_id) + .bind(&resource) + .fetch_all(&pool) + .await + .unwrap_or_default(); + + if !user_tags.is_empty() { + return Ok(user_tags); + } + + sqlx::query_as( + "SELECT tag_key, tag_value FROM iam_role_tags \ + WHERE account_id = ? AND role_name = ? ORDER BY tag_key", + ) + .bind(&account_id) + .bind(&resource) + .fetch_all(&pool) + .await + .map_err(|e| { + tracing::error!("fetch_caller_tags: {e}"); + OpError::Internal("Database error".to_owned()) + }) + }) + } +} diff --git a/crates/storage-sqlite/src/management_store/policies.rs b/crates/storage-sqlite/src/management_store/policies.rs new file mode 100644 index 00000000..6d18d816 --- /dev/null +++ b/crates/storage-sqlite/src/management_store/policies.rs @@ -0,0 +1,248 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Policy and permissions-boundary operations for `SqliteCatalogStore`. + +use extenddb_storage::management_store::{OpError, OpResult}; + +use crate::catalog_store::SqliteCatalogStore; + +impl SqliteCatalogStore { + // ── Policies ─────────────────────────────────────────────────── + + pub(crate) async fn put_policy_impl( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + policy_name: &str, + document: &serde_json::Value, + ) -> OpResult<()> { + let doc_str = serde_json::to_string(document).unwrap_or_else(|_| "{}".to_owned()); + let result = sqlx::query( + "INSERT INTO iam_policies \ + (account_id, principal_type, principal_name, policy_name, policy_document) \ + VALUES (?, ?, ?, ?, ?) \ + ON CONFLICT (account_id, principal_type, principal_name, policy_name) \ + DO UPDATE SET policy_document = excluded.policy_document", + ) + .bind(account_id) + .bind(principal_type) + .bind(principal_name) + .bind(policy_name) + .bind(&doc_str) + .execute(self.pool()) + .await; + match result { + Ok(_) => Ok(()), + Err(e) => { + tracing::error!("put_policy failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + } + + pub(crate) async fn delete_policy_impl( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + policy_name: &str, + ) -> OpResult<()> { + let result = sqlx::query( + "DELETE FROM iam_policies \ + WHERE account_id = ? AND principal_type = ? AND principal_name = ? \ + AND policy_name = ?", + ) + .bind(account_id) + .bind(principal_type) + .bind(principal_name) + .bind(policy_name) + .execute(self.pool()) + .await; + match result { + Ok(r) if r.rows_affected() == 0 => { + Err(OpError::NotFound("Policy not found".to_owned())) + } + Ok(_) => Ok(()), + Err(e) => { + tracing::error!("delete_policy failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + } + + pub(crate) async fn list_policies_impl( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + ) -> OpResult> { + let rows: Vec<(String, String, String)> = sqlx::query_as( + "SELECT policy_name, policy_document, created_at FROM iam_policies \ + WHERE account_id = ? AND principal_type = ? AND principal_name = ? \ + ORDER BY policy_name", + ) + .bind(account_id) + .bind(principal_type) + .bind(principal_name) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("list_policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + rows.into_iter() + .map(|(name, doc_str, ts)| { + let document = serde_json::from_str::(&doc_str) + .unwrap_or(serde_json::Value::Object(Default::default())); + let created_at = + crate::sqlite_util::parse_timestamp(&ts).map_err(|e| { + tracing::error!("list_policies parse_timestamp: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok((name, document, created_at)) + }) + .collect() + } + + // ── Permissions boundaries ───────────────────────────────────── + + pub(crate) async fn set_user_boundary_impl( + &self, + account_id: &str, + user_name: &str, + document: &serde_json::Value, + ) -> OpResult<()> { + self.set_boundary_impl(account_id, "user", user_name, document) + .await + } + + pub(crate) async fn get_user_boundary_impl( + &self, + account_id: &str, + user_name: &str, + ) -> OpResult> { + self.get_boundary_impl(account_id, "user", user_name).await + } + + pub(crate) async fn delete_user_boundary_impl( + &self, + account_id: &str, + user_name: &str, + ) -> OpResult<()> { + self.delete_boundary_impl(account_id, "user", user_name) + .await + } + + pub(crate) async fn set_role_boundary_impl( + &self, + account_id: &str, + role_name: &str, + document: &serde_json::Value, + ) -> OpResult<()> { + self.set_boundary_impl(account_id, "role", role_name, document) + .await + } + + pub(crate) async fn get_role_boundary_impl( + &self, + account_id: &str, + role_name: &str, + ) -> OpResult> { + self.get_boundary_impl(account_id, "role", role_name).await + } + + pub(crate) async fn delete_role_boundary_impl( + &self, + account_id: &str, + role_name: &str, + ) -> OpResult<()> { + self.delete_boundary_impl(account_id, "role", role_name) + .await + } + + async fn set_boundary_impl( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + document: &serde_json::Value, + ) -> OpResult<()> { + let doc_str = serde_json::to_string(document).unwrap_or_else(|_| "{}".to_owned()); + let result = sqlx::query( + "INSERT INTO iam_permissions_boundaries \ + (account_id, principal_type, principal_name, policy_document) \ + VALUES (?, ?, ?, ?) \ + ON CONFLICT (account_id, principal_type, principal_name) \ + DO UPDATE SET policy_document = excluded.policy_document", + ) + .bind(account_id) + .bind(principal_type) + .bind(principal_name) + .bind(&doc_str) + .execute(self.pool()) + .await; + match result { + Ok(_) => Ok(()), + Err(e) => { + tracing::error!("set_boundary failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + } + + async fn get_boundary_impl( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + ) -> OpResult> { + let row: Option<(String,)> = sqlx::query_as( + "SELECT policy_document FROM iam_permissions_boundaries \ + WHERE account_id = ? AND principal_type = ? AND principal_name = ?", + ) + .bind(account_id) + .bind(principal_type) + .bind(principal_name) + .fetch_optional(self.pool()) + .await + .map_err(|e| { + tracing::error!("get_boundary: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + Ok(row.map(|(doc_str,)| { + serde_json::from_str::(&doc_str) + .unwrap_or(serde_json::Value::Object(Default::default())) + })) + } + + async fn delete_boundary_impl( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + ) -> OpResult<()> { + let result = sqlx::query( + "DELETE FROM iam_permissions_boundaries \ + WHERE account_id = ? AND principal_type = ? AND principal_name = ?", + ) + .bind(account_id) + .bind(principal_type) + .bind(principal_name) + .execute(self.pool()) + .await; + match result { + Ok(r) if r.rows_affected() == 0 => { + Err(OpError::NotFound("Permissions boundary not set".to_owned())) + } + Ok(_) => Ok(()), + Err(e) => { + tracing::error!("delete_boundary failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + } +} diff --git a/crates/storage-sqlite/src/management_store/roles.rs b/crates/storage-sqlite/src/management_store/roles.rs new file mode 100644 index 00000000..f4b69344 --- /dev/null +++ b/crates/storage-sqlite/src/management_store/roles.rs @@ -0,0 +1,272 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Role management operations for `SqliteCatalogStore`. + +use extenddb_storage::management_store::{OpError, OpResult, RoleDetail, RoleListEntry}; + +use crate::catalog_store::SqliteCatalogStore; +use crate::sqlite_util::{is_fk_violation, is_unique_violation}; + +impl SqliteCatalogStore { + pub(crate) async fn create_role_impl( + &self, + account_id: &str, + role_name: &str, + trust_policy: &serde_json::Value, + ) -> OpResult<()> { + let role_arn = format!("arn:aws:iam::{account_id}:role/{role_name}"); + let trust_policy_str = + serde_json::to_string(trust_policy).unwrap_or_else(|_| "{}".to_owned()); + let result = sqlx::query( + "INSERT INTO iam_roles (account_id, role_name, role_arn, trust_policy) \ + VALUES (?, ?, ?, ?)", + ) + .bind(account_id) + .bind(role_name) + .bind(&role_arn) + .bind(&trust_policy_str) + .execute(self.pool()) + .await; + match result { + Ok(_) => Ok(()), + Err(e) if is_unique_violation(&e) => { + Err(OpError::AlreadyExists("IAM role already exists".to_owned())) + } + Err(e) if is_fk_violation(&e) => { + Err(OpError::NotFound("Account not found".to_owned())) + } + Err(e) => { + tracing::error!("create_role failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + } + + pub(crate) async fn delete_role_impl( + &self, + account_id: &str, + role_name: &str, + ) -> OpResult<()> { + let result = + sqlx::query("DELETE FROM iam_roles WHERE account_id = ? AND role_name = ?") + .bind(account_id) + .bind(role_name) + .execute(self.pool()) + .await; + match result { + Ok(r) if r.rows_affected() == 0 => { + Err(OpError::NotFound("IAM role not found".to_owned())) + } + Ok(_) => Ok(()), + Err(e) => { + tracing::error!("delete_role failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + } + + pub(crate) async fn list_roles_impl( + &self, + account_id: &str, + ) -> OpResult> { + let rows: Vec<(String, String, String, String, String)> = sqlx::query_as( + "SELECT account_id, role_name, role_arn, trust_policy, created_at \ + FROM iam_roles WHERE account_id = ? ORDER BY role_name", + ) + .bind(account_id) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("list_roles: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + rows.into_iter() + .map(|(aid, rn, arn, tp_str, ts)| { + let trust_policy = serde_json::from_str::(&tp_str) + .unwrap_or(serde_json::Value::Object(Default::default())); + let created_at = + crate::sqlite_util::parse_timestamp(&ts).map_err(|e| { + tracing::error!("list_roles parse_timestamp: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok((aid, rn, arn, trust_policy, created_at)) + }) + .collect() + } + + pub(crate) async fn get_role_detail_impl( + &self, + account_id: &str, + role_name: &str, + ) -> OpResult> { + let role: Option<(String, String)> = sqlx::query_as( + "SELECT role_name, trust_policy FROM iam_roles \ + WHERE account_id = ? AND role_name = ?", + ) + .bind(account_id) + .bind(role_name) + .fetch_optional(self.pool()) + .await + .map_err(|e| { + tracing::error!("get_role_detail role: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let Some((_, tp_str)) = role else { + return Ok(None); + }; + + let trust_policy = serde_json::from_str::(&tp_str) + .unwrap_or(serde_json::Value::Object(Default::default())); + + let policies: Vec<(String,)> = sqlx::query_as( + "SELECT policy_name FROM iam_policies \ + WHERE account_id = ? AND principal_type = 'role' AND principal_name = ? \ + ORDER BY policy_name", + ) + .bind(account_id) + .bind(role_name) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("get_role_detail policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let tags: Vec<(String, String)> = sqlx::query_as( + "SELECT tag_key, tag_value FROM iam_role_tags \ + WHERE account_id = ? AND role_name = ? ORDER BY tag_key", + ) + .bind(account_id) + .bind(role_name) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("get_role_detail tags: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + Ok(Some(RoleDetail { + trust_policy, + policies: policies.into_iter().map(|(n,)| n).collect(), + tags, + })) + } + + pub(crate) async fn get_role_trust_policy_impl( + &self, + account_id: &str, + role_name: &str, + ) -> OpResult> { + let row: Option<(String,)> = sqlx::query_as( + "SELECT trust_policy FROM iam_roles WHERE account_id = ? AND role_name = ?", + ) + .bind(account_id) + .bind(role_name) + .fetch_optional(self.pool()) + .await + .map_err(|e| { + tracing::error!("get_role_trust_policy: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + Ok(row.map(|(tp_str,)| { + serde_json::from_str::(&tp_str) + .unwrap_or(serde_json::Value::Object(Default::default())) + })) + } + + // ── Role tags ────────────────────────────────────────────────── + + pub(crate) async fn tag_role_impl( + &self, + account_id: &str, + role_name: &str, + tags: &[(String, String)], + ) -> OpResult<()> { + let mut tx = self.pool().begin().await.map_err(|e| { + tracing::error!("tag_role begin: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + for (key, value) in tags { + let result = sqlx::query( + "INSERT INTO iam_role_tags (account_id, role_name, tag_key, tag_value) \ + VALUES (?, ?, ?, ?) \ + ON CONFLICT (account_id, role_name, tag_key) \ + DO UPDATE SET tag_value = excluded.tag_value", + ) + .bind(account_id) + .bind(role_name) + .bind(key) + .bind(value) + .execute(&mut *tx) + .await; + match result { + Ok(_) => {} + Err(e) if is_fk_violation(&e) => { + return Err(OpError::NotFound("IAM role not found".to_owned())); + } + Err(e) => { + tracing::error!("tag_role failed: {e}"); + return Err(OpError::Internal("Database error".to_owned())); + } + } + } + tx.commit().await.map_err(|e| { + tracing::error!("tag_role commit: {e}"); + OpError::Internal("Database error".to_owned()) + }) + } + + pub(crate) async fn untag_role_impl( + &self, + account_id: &str, + role_name: &str, + tag_keys: &[String], + ) -> OpResult<()> { + let mut tx = self.pool().begin().await.map_err(|e| { + tracing::error!("untag_role begin: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + for key in tag_keys { + sqlx::query( + "DELETE FROM iam_role_tags \ + WHERE account_id = ? AND role_name = ? AND tag_key = ?", + ) + .bind(account_id) + .bind(role_name) + .bind(key) + .execute(&mut *tx) + .await + .map_err(|e| { + tracing::error!("untag_role failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + } + tx.commit().await.map_err(|e| { + tracing::error!("untag_role commit: {e}"); + OpError::Internal("Database error".to_owned()) + }) + } + + pub(crate) async fn list_role_tags_impl( + &self, + account_id: &str, + role_name: &str, + ) -> OpResult> { + sqlx::query_as( + "SELECT tag_key, tag_value FROM iam_role_tags \ + WHERE account_id = ? AND role_name = ? ORDER BY tag_key", + ) + .bind(account_id) + .bind(role_name) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("list_role_tags: {e}"); + OpError::Internal("Database error".to_owned()) + }) + } +} diff --git a/crates/storage-sqlite/src/management_store/users.rs b/crates/storage-sqlite/src/management_store/users.rs new file mode 100644 index 00000000..6722193e --- /dev/null +++ b/crates/storage-sqlite/src/management_store/users.rs @@ -0,0 +1,370 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! User management operations for `SqliteCatalogStore`. + +use extenddb_storage::management_store::{OpError, OpResult, UserDetail, UserListEntry}; + +use crate::catalog_store::SqliteCatalogStore; +use crate::sqlite_util::{is_fk_violation, is_unique_violation}; + +impl SqliteCatalogStore { + pub(crate) async fn create_user_impl( + &self, + account_id: &str, + user_name: &str, + password_hash: Option<&str>, + ) -> OpResult<()> { + let user_arn = format!("arn:aws:iam::{account_id}:user/{user_name}"); + + let mut tx = self.pool().begin().await.map_err(|e| { + tracing::error!("create_user begin transaction: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let result = sqlx::query( + "INSERT INTO iam_users (account_id, user_name, user_arn, password_hash) \ + VALUES (?, ?, ?, ?)", + ) + .bind(account_id) + .bind(user_name) + .bind(&user_arn) + .bind(password_hash) + .execute(&mut *tx) + .await; + + match result { + Ok(_) => {} + Err(e) if is_unique_violation(&e) => { + return Err(OpError::AlreadyExists("IAM user already exists".to_owned())); + } + Err(e) if is_fk_violation(&e) => { + return Err(OpError::NotFound("Account not found".to_owned())); + } + Err(e) => { + tracing::error!("create_user failed: {e}"); + return Err(OpError::Internal("Database error".to_owned())); + } + } + + let self_service_policy = serde_json::json!({ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": [ + "iam:CreateAccessKey", + "iam:DeleteAccessKey", + "iam:ListAccessKeys", + "iam:ChangePassword" + ], + "Resource": format!("arn:aws:iam::{account_id}:user/{user_name}") + }] + }); + let policy_doc_str = + serde_json::to_string(&self_service_policy).unwrap_or_else(|_| "{}".to_owned()); + + if let Err(e) = sqlx::query( + "INSERT INTO iam_policies \ + (account_id, principal_type, principal_name, policy_name, policy_document) \ + VALUES (?, 'user', ?, 'SelfServicePolicy', ?) ON CONFLICT DO NOTHING", + ) + .bind(account_id) + .bind(user_name) + .bind(&policy_doc_str) + .execute(&mut *tx) + .await + { + tracing::error!("seed self-service policy failed: {e}"); + return Err(OpError::Internal("Database error".to_owned())); + } + + tx.commit().await.map_err(|e| { + tracing::error!("create_user commit: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + Ok(()) + } + + pub(crate) async fn delete_user_impl( + &self, + account_id: &str, + user_name: &str, + ) -> OpResult<()> { + let result = + sqlx::query("DELETE FROM iam_users WHERE account_id = ? AND user_name = ?") + .bind(account_id) + .bind(user_name) + .execute(self.pool()) + .await; + match result { + Ok(r) if r.rows_affected() == 0 => { + Err(OpError::NotFound("IAM user not found".to_owned())) + } + Ok(_) => Ok(()), + Err(e) => { + tracing::error!("delete_user failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + } + + pub(crate) async fn list_users_impl( + &self, + account_id: &str, + ) -> OpResult> { + let rows: Vec<(String, String, String, Option, String)> = sqlx::query_as( + "SELECT account_id, user_name, user_arn, password_hash, created_at \ + FROM iam_users WHERE account_id = ? ORDER BY user_name", + ) + .bind(account_id) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("list_users: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + rows.into_iter() + .map(|(aid, un, arn, pw, ts)| { + let created_at = + crate::sqlite_util::parse_timestamp(&ts).map_err(|e| { + tracing::error!("list_users parse_timestamp: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok((aid, un, arn, pw.is_some(), created_at)) + }) + .collect() + } + + pub(crate) async fn get_user_detail_impl( + &self, + account_id: &str, + user_name: &str, + ) -> OpResult> { + let exists: Option<(String,)> = sqlx::query_as( + "SELECT user_name FROM iam_users WHERE account_id = ? AND user_name = ?", + ) + .bind(account_id) + .bind(user_name) + .fetch_optional(self.pool()) + .await + .map_err(|e| { + tracing::error!("get_user_detail exists: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + if exists.is_none() { + return Ok(None); + } + + let keys: Vec<(String, bool)> = sqlx::query_as( + "SELECT access_key_id, is_active FROM access_keys \ + WHERE account_id = ? AND user_name = ? ORDER BY access_key_id", + ) + .bind(account_id) + .bind(user_name) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("get_user_detail keys: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let policies: Vec<(String,)> = sqlx::query_as( + "SELECT policy_name FROM iam_policies \ + WHERE account_id = ? AND principal_type = 'user' AND principal_name = ? \ + ORDER BY policy_name", + ) + .bind(account_id) + .bind(user_name) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("get_user_detail policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let tags: Vec<(String, String)> = sqlx::query_as( + "SELECT tag_key, tag_value FROM iam_user_tags \ + WHERE account_id = ? AND user_name = ? ORDER BY tag_key", + ) + .bind(account_id) + .bind(user_name) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("get_user_detail tags: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let groups: Vec<(String,)> = sqlx::query_as( + "SELECT group_name FROM iam_group_members \ + WHERE account_id = ? AND user_name = ? ORDER BY group_name", + ) + .bind(account_id) + .bind(user_name) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("get_user_detail groups: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + Ok(Some(UserDetail { + keys, + policies: policies.into_iter().map(|(n,)| n).collect(), + tags, + groups: groups.into_iter().map(|(n,)| n).collect(), + })) + } + + pub(crate) async fn verify_iam_user_password_impl( + &self, + account_id: &str, + user_name: &str, + password: &str, + ) -> OpResult { + let row: Option<(String,)> = sqlx::query_as( + "SELECT password_hash FROM iam_users \ + WHERE account_id = ? AND user_name = ? AND password_hash IS NOT NULL", + ) + .bind(account_id) + .bind(user_name) + .fetch_optional(self.pool()) + .await + .map_err(|e| { + tracing::error!("verify_iam_user_password: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let Some((hash,)) = row else { + return Ok(false); + }; + + let pw = password.to_owned(); + Ok( + tokio::task::spawn_blocking(move || bcrypt::verify(pw, &hash).unwrap_or(false)) + .await + .unwrap_or(false), + ) + } + + pub(crate) async fn change_user_password_impl( + &self, + account_id: &str, + user_name: &str, + password_hash: &str, + ) -> OpResult<()> { + let result = sqlx::query( + "UPDATE iam_users SET password_hash = ? WHERE account_id = ? AND user_name = ?", + ) + .bind(password_hash) + .bind(account_id) + .bind(user_name) + .execute(self.pool()) + .await; + match result { + Ok(r) if r.rows_affected() == 0 => { + Err(OpError::NotFound("IAM user not found".to_owned())) + } + Ok(_) => Ok(()), + Err(e) => { + tracing::error!("change_user_password failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + } + + // ── User tags ────────────────────────────────────────────────── + + pub(crate) async fn tag_user_impl( + &self, + account_id: &str, + user_name: &str, + tags: &[(String, String)], + ) -> OpResult<()> { + let mut tx = self.pool().begin().await.map_err(|e| { + tracing::error!("tag_user begin: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + for (key, value) in tags { + let result = sqlx::query( + "INSERT INTO iam_user_tags (account_id, user_name, tag_key, tag_value) \ + VALUES (?, ?, ?, ?) \ + ON CONFLICT (account_id, user_name, tag_key) \ + DO UPDATE SET tag_value = excluded.tag_value", + ) + .bind(account_id) + .bind(user_name) + .bind(key) + .bind(value) + .execute(&mut *tx) + .await; + match result { + Ok(_) => {} + Err(e) if is_fk_violation(&e) => { + return Err(OpError::NotFound("IAM user not found".to_owned())); + } + Err(e) => { + tracing::error!("tag_user failed: {e}"); + return Err(OpError::Internal("Database error".to_owned())); + } + } + } + tx.commit().await.map_err(|e| { + tracing::error!("tag_user commit: {e}"); + OpError::Internal("Database error".to_owned()) + }) + } + + pub(crate) async fn untag_user_impl( + &self, + account_id: &str, + user_name: &str, + tag_keys: &[String], + ) -> OpResult<()> { + let mut tx = self.pool().begin().await.map_err(|e| { + tracing::error!("untag_user begin: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + for key in tag_keys { + sqlx::query( + "DELETE FROM iam_user_tags \ + WHERE account_id = ? AND user_name = ? AND tag_key = ?", + ) + .bind(account_id) + .bind(user_name) + .bind(key) + .execute(&mut *tx) + .await + .map_err(|e| { + tracing::error!("untag_user failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + } + tx.commit().await.map_err(|e| { + tracing::error!("untag_user commit: {e}"); + OpError::Internal("Database error".to_owned()) + }) + } + + pub(crate) async fn list_user_tags_impl( + &self, + account_id: &str, + user_name: &str, + ) -> OpResult> { + sqlx::query_as( + "SELECT tag_key, tag_value FROM iam_user_tags \ + WHERE account_id = ? AND user_name = ? ORDER BY tag_key", + ) + .bind(account_id) + .bind(user_name) + .fetch_all(self.pool()) + .await + .map_err(|e| { + tracing::error!("list_user_tags: {e}"); + OpError::Internal("Database error".to_owned()) + }) + } +} diff --git a/crates/storage-sqlite/src/metadata_engine.rs b/crates/storage-sqlite/src/metadata_engine.rs new file mode 100644 index 00000000..d1a7d2b1 --- /dev/null +++ b/crates/storage-sqlite/src/metadata_engine.rs @@ -0,0 +1,448 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `MetadataEngine` trait implementation for `SqliteEngine`. + +use extenddb_core::types::{Item, Tag, TimeToLiveDescription, TimeToLiveStatus}; +use extenddb_storage::MetadataEngine; +use extenddb_storage::error::StorageError; +use futures::future::BoxFuture; + +use crate::data; +use crate::engine::SqliteEngine; + +impl MetadataEngine for SqliteEngine { + fn describe_ttl( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { + let row: Option<(Option,)> = sqlx::query_as( + "SELECT ttl_attribute FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (ttl_attr,) = row.ok_or_else(|| StorageError::TableNotFound(table_name.clone()))?; + + Ok(match ttl_attr { + Some(attr) => TimeToLiveDescription { + time_to_live_status: TimeToLiveStatus::Enabled, + attribute_name: Some(attr), + }, + None => TimeToLiveDescription { + time_to_live_status: TimeToLiveStatus::Disabled, + attribute_name: None, + }, + }) + }) + } + + fn update_ttl( + &self, + account_id: &str, + table_name: &str, + attribute_name: &str, + enabled: bool, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + let attribute_name = attribute_name.to_string(); + Box::pin(async move { + let ttl_val: Option<&str> = if enabled { Some(&attribute_name) } else { None }; + let index_ready = false; + + let result = sqlx::query( + "UPDATE tables SET ttl_attribute = ?, ttl_index_ready = ? \ + WHERE account_id = ? AND table_name = ? AND table_status = 'ACTIVE'", + ) + .bind(ttl_val) + .bind(index_ready) + .bind(&account_id) + .bind(&table_name) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if result.rows_affected() == 0 { + let exists: Option<(String,)> = sqlx::query_as( + "SELECT table_status FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + return match exists { + None => Err(StorageError::TableNotFound(table_name)), + Some(_) => Err(StorageError::TableNotActive(table_name)), + }; + } + + Ok(()) + }) + } + + fn tag_resource(&self, arn: &str, tags: &[Tag]) -> BoxFuture<'_, Result<(), StorageError>> { + let arn = arn.to_string(); + let tags = tags.to_vec(); + Box::pin(async move { + for tag in &tags { + sqlx::query( + "INSERT INTO tags (resource_arn, tag_key, tag_value) VALUES (?, ?, ?) \ + ON CONFLICT (resource_arn, tag_key) DO UPDATE SET tag_value = EXCLUDED.tag_value", + ) + .bind(&arn) + .bind(&tag.key) + .bind(&tag.value) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) + }) + } + + fn untag_resource( + &self, + arn: &str, + tag_keys: &[String], + ) -> BoxFuture<'_, Result<(), StorageError>> { + let arn = arn.to_string(); + let tag_keys = tag_keys.to_vec(); + Box::pin(async move { + for key in &tag_keys { + sqlx::query("DELETE FROM tags WHERE resource_arn = ? AND tag_key = ?") + .bind(&arn) + .bind(key) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) + }) + } + + fn list_tags(&self, arn: &str) -> BoxFuture<'_, Result, StorageError>> { + let arn = arn.to_string(); + Box::pin(async move { + let rows: Vec<(String, String)> = sqlx::query_as( + "SELECT tag_key, tag_value FROM tags WHERE resource_arn = ? ORDER BY tag_key", + ) + .bind(&arn) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(rows + .into_iter() + .map(|(key, value)| Tag { key, value }) + .collect()) + }) + } + + fn tables_with_ttl( + &self, + account_id: &str, + ) -> BoxFuture<'_, Result, StorageError>> { + let account_id = account_id.to_string(); + Box::pin(async move { + let rows: Vec<(String, String)> = sqlx::query_as( + "SELECT table_name, ttl_attribute FROM tables \ + WHERE account_id = ? AND ttl_attribute IS NOT NULL AND table_status = 'ACTIVE'", + ) + .bind(&account_id) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(rows) + }) + } + + fn refresh_table_size( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { + Self::validate_account_id(&account_id)?; + let row: Option<(String,)> = sqlx::query_as( + "SELECT table_id FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (table_id,) = match row { + Some(r) => r, + None => return Ok(()), + }; + + let data_table = data::data_table_name(&table_id); + let count_sql = format!("SELECT COUNT(*) FROM {data_table}"); + let item_count: i64 = match sqlx::query_scalar::<_, i64>(&count_sql) + .fetch_one(&self.pool) + .await + { + Ok(c) => c, + Err(_) => return Ok(()), + }; + + // Estimate size: each row is roughly 256 bytes average + let table_size = item_count * 256; + + sqlx::query( + "UPDATE tables SET item_count = ?, table_size_bytes = ? \ + WHERE account_id = ? AND table_name = ? AND table_status = 'ACTIVE'", + ) + .bind(item_count) + .bind(table_size) + .bind(&account_id) + .bind(&table_name) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(()) + }) + } + + fn list_active_table_names( + &self, + account_id: &str, + ) -> BoxFuture<'_, Result, StorageError>> { + let account_id = account_id.to_string(); + Box::pin(async move { + let rows: Vec<(String,)> = sqlx::query_as( + "SELECT table_name FROM tables \ + WHERE account_id = ? AND table_status = 'ACTIVE' ORDER BY table_name", + ) + .bind(&account_id) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(rows.into_iter().map(|(n,)| n).collect()) + }) + } + + fn all_tables_with_ttl( + &self, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let rows: Vec<(String, String, String)> = sqlx::query_as( + "SELECT account_id, table_name, ttl_attribute FROM tables \ + WHERE ttl_attribute IS NOT NULL AND table_status = 'ACTIVE'", + ) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(rows) + }) + } + + fn all_tables_with_ttl_index_ready( + &self, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let rows: Vec<(String, String, String)> = sqlx::query_as( + "SELECT account_id, table_name, ttl_attribute FROM tables \ + WHERE ttl_attribute IS NOT NULL AND ttl_index_ready = TRUE AND table_status = 'ACTIVE'", + ) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(rows) + }) + } + + fn create_ttl_index( + &self, + account_id: &str, + table_name: &str, + ttl_attribute: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + let ttl_attribute = ttl_attribute.to_string(); + Box::pin(async move { + Self::validate_account_id(&account_id)?; + let row: Option<(String,)> = sqlx::query_as( + "SELECT table_id FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (table_id,) = match row { + Some(r) => r, + None => return Err(StorageError::TableNotFound(table_name)), + }; + + let data_table = data::data_table_name(&table_id); + let bare_table = data_table.trim_matches('"'); + let index_name = format!("idx_ttl_{bare_table}"); + + // SQLite uses json_extract for JSON field access. + let sql = format!( + "CREATE INDEX IF NOT EXISTS \"{index_name}\" \ + ON {data_table} (CAST(json_extract(item_data, '$.{ttl_attribute}.N') AS INTEGER)) \ + WHERE json_extract(item_data, '$.{ttl_attribute}.N') IS NOT NULL" + ); + sqlx::query(&sql) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(format!("TTL index creation failed: {e}")))?; + + sqlx::query( + "UPDATE tables SET ttl_index_ready = TRUE \ + WHERE account_id = ? AND table_name = ?", + ) + .bind(&account_id) + .bind(&table_name) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(()) + }) + } + + fn drop_ttl_index( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { + Self::validate_account_id(&account_id)?; + let row: Option<(String,)> = sqlx::query_as( + "SELECT table_id FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (table_id,) = match row { + Some(r) => r, + None => return Err(StorageError::TableNotFound(table_name)), + }; + + let data_table = data::data_table_name(&table_id); + let bare_table = data_table.trim_matches('"'); + let index_name = format!("idx_ttl_{bare_table}"); + + sqlx::query( + "UPDATE tables SET ttl_index_ready = FALSE \ + WHERE account_id = ? AND table_name = ?", + ) + .bind(&account_id) + .bind(&table_name) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let sql = format!("DROP INDEX IF EXISTS \"{index_name}\""); + sqlx::query(&sql) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(format!("TTL index drop failed: {e}")))?; + + Ok(()) + }) + } + + fn find_expired_items_indexed( + &self, + account_id: &str, + table_name: &str, + ttl_attribute: &str, + limit: usize, + ) -> BoxFuture<'_, Result, StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + let ttl_attribute = ttl_attribute.to_string(); + Box::pin(async move { + Self::validate_account_id(&account_id)?; + let row: Option<(String,)> = sqlx::query_as( + "SELECT table_id FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (table_id,) = match row { + Some(r) => r, + None => return Err(StorageError::TableNotFound(table_name)), + }; + + let data_table = data::data_table_name(&table_id); + + let now_epoch = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + + let limit_i64 = i64::try_from(limit).unwrap_or(i64::MAX); + let now_i64 = i64::try_from(now_epoch).unwrap_or(i64::MAX); + + // Use json_extract with dynamic path bound as parameter. + let ttl_path = format!("$.{ttl_attribute}.N"); + let sql = format!( + "SELECT item_data FROM {data_table} \ + WHERE CAST(json_extract(item_data, ?) AS INTEGER) BETWEEN 1 AND ? \ + ORDER BY CAST(json_extract(item_data, ?) AS INTEGER) \ + LIMIT ?" + ); + let rows: Vec<(String,)> = sqlx::query_as(&sql) + .bind(&ttl_path) + .bind(now_i64) + .bind(&ttl_path) + .bind(limit_i64) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + rows.into_iter() + .map(|(s,)| { + serde_json::from_str(&s).map_err(|e| StorageError::Internal(e.to_string())) + }) + .collect() + }) + } + + fn all_active_tables(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let rows: Vec<(String, String)> = sqlx::query_as( + "SELECT account_id, table_name FROM tables \ + WHERE table_status = 'ACTIVE' ORDER BY account_id, table_name", + ) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(rows) + }) + } +} diff --git a/crates/storage-sqlite/src/migrations.rs b/crates/storage-sqlite/src/migrations.rs new file mode 100644 index 00000000..6cffb013 --- /dev/null +++ b/crates/storage-sqlite/src/migrations.rs @@ -0,0 +1,91 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! SQLite schema migration helpers. + +use extenddb_storage::management_store::{OpError, OpResult}; +use sqlx::SqlitePool; + +/// Embedded catalog/data migration files, applied in order. +pub(crate) const MIGRATIONS: &[(&str, &str)] = &[( + "001_schema.sql", + include_str!("../migrations/001_schema.sql"), +)]; + +/// Check if a table exists in the SQLite database. +pub(crate) async fn table_exists(pool: &SqlitePool, name: &str) -> OpResult { + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?)", + ) + .bind(name) + .fetch_one(pool) + .await + .map_err(|e| OpError::Internal(format!("Check table exists: {e}")))?; + Ok(exists) +} + +/// Run all migrations, skipping already-applied ones. +pub(crate) async fn run_migrations(pool: &SqlitePool) -> OpResult<()> { + // Create schema_history table if it doesn't exist yet (needed for tracking). + sqlx::query( + "CREATE TABLE IF NOT EXISTS schema_history ( + filename TEXT PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + )", + ) + .execute(pool) + .await + .map_err(|e| OpError::Internal(format!("Create schema_history: {e}")))?; + + println!("--- Running migrations..."); + for (filename, sql) in MIGRATIONS { + if is_migration_applied(pool, filename).await? { + println!(" {filename} — already applied, skipping."); + continue; + } + println!(" Applying {filename}..."); + // Execute statements one by one (SQLite doesn't support multi-statement raw_sql well). + for stmt in split_sql(sql) { + let stmt = stmt.trim(); + if stmt.is_empty() { + continue; + } + sqlx::query(stmt) + .execute(pool) + .await + .map_err(|e| OpError::Internal(format!("Migration {filename} failed: {e}\nSQL: {stmt}")))?; + } + record_migration(pool, filename).await?; + } + println!(" Migrations applied."); + Ok(()) +} + +/// Check if a migration has already been applied. +async fn is_migration_applied(pool: &SqlitePool, filename: &str) -> OpResult { + let applied: bool = + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM schema_history WHERE filename = ?)") + .bind(filename) + .fetch_one(pool) + .await + .map_err(|e| OpError::Internal(format!("Check migration: {e}")))?; + Ok(applied) +} + +/// Record a migration as applied. +async fn record_migration(pool: &SqlitePool, filename: &str) -> OpResult<()> { + sqlx::query( + "INSERT OR IGNORE INTO schema_history (filename) VALUES (?)", + ) + .bind(filename) + .execute(pool) + .await + .map_err(|e| OpError::Internal(format!("Record migration: {e}")))?; + Ok(()) +} + +/// Split a SQL script into individual statements by semicolon. +/// Handles basic cases — does not parse strings or comments. +fn split_sql(sql: &str) -> Vec<&str> { + sql.split(';').collect() +} diff --git a/crates/storage-sqlite/src/operations.rs b/crates/storage-sqlite/src/operations.rs new file mode 100644 index 00000000..a50370e9 --- /dev/null +++ b/crates/storage-sqlite/src/operations.rs @@ -0,0 +1,71 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! SQLite implementation of `OperationsEngine`. + +use extenddb_storage::error::StorageError; +use extenddb_storage::operations::{ConnectionParts, OperationsEngine}; + +/// SQLite operations engine for ExtendDB CLI commands. +pub struct SqliteOperationsEngine; + +impl OperationsEngine for SqliteOperationsEngine { + fn parse_connection_string(&self, s: &str) -> Result { + // SQLite connection strings are just file paths or sqlite:// URLs. + // We map them to ConnectionParts with dummy host/port/user/password. + let path = s + .strip_prefix("sqlite://") + .or_else(|| s.strip_prefix("sqlite:")) + .unwrap_or(s); + let path = path.trim_start_matches('/'); + Ok(ConnectionParts { + host: "localhost".to_owned(), + port: 0, + user: String::new(), + password: String::new(), + database: path.to_owned(), + }) + } + + fn redact_connection_string(&self, s: &str) -> String { + // SQLite paths don't contain passwords. + s.to_owned() + } + + fn validate_identifier(&self, name: &str, label: &str) -> Result<(), StorageError> { + if name.contains('"') { + return Err(StorageError::Internal(format!( + "{label} must not contain double quotes" + ))); + } + if name.contains('\0') { + return Err(StorageError::Internal(format!( + "{label} must not contain null bytes" + ))); + } + if !name.is_ascii() { + return Err(StorageError::Internal(format!( + "{label} must contain only ASCII characters" + ))); + } + Ok(()) + } + + fn catalog_version(&self) -> String { + crate::engine::CATALOG_VERSION.to_string() + } + + fn is_sensitive_key(&self, key: &str) -> bool { + let lower = key.to_lowercase(); + [ + "path", + "connection_string", + "password", + "secret", + "token", + "encryption_key", + ] + .iter() + .any(|pattern| lower.contains(pattern)) + } +} diff --git a/crates/storage-sqlite/src/sqlite_util.rs b/crates/storage-sqlite/src/sqlite_util.rs new file mode 100644 index 00000000..12972c76 --- /dev/null +++ b/crates/storage-sqlite/src/sqlite_util.rs @@ -0,0 +1,44 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Shared SQLite error classification helpers. + +/// Check if a sqlx error is a unique constraint violation. +pub(crate) fn is_unique_violation(e: &sqlx::Error) -> bool { + if let sqlx::Error::Database(db_err) = e { + let msg = db_err.message(); + return msg.contains("UNIQUE constraint failed"); + } + false +} + +/// Check if a sqlx error is a foreign key violation. +pub(crate) fn is_fk_violation(e: &sqlx::Error) -> bool { + if let sqlx::Error::Database(db_err) = e { + let msg = db_err.message(); + return msg.contains("FOREIGN KEY constraint failed"); + } + false +} + +/// Format a `time::OffsetDateTime` as RFC 3339 text for SQLite storage. +pub(crate) fn format_timestamp(dt: time::OffsetDateTime) -> String { + dt.format(&time::format_description::well_known::Rfc3339) + .unwrap_or_else(|_| dt.to_string()) +} + +/// Parse an RFC 3339 timestamp text from SQLite. +pub(crate) fn parse_timestamp( + s: &str, +) -> Result { + time::OffsetDateTime::parse(s, &time::format_description::well_known::Rfc3339) + .or_else(|_| { + // Fall back to SQLite's default CURRENT_TIMESTAMP format (no timezone). + let format = time::format_description::parse("[year]-[month]-[day] [hour]:[minute]:[second]") + .map_err(|e| e.to_string())?; + time::PrimitiveDateTime::parse(s, &format) + .map(|dt| dt.assume_utc()) + .map_err(|e| e.to_string()) + }) + .map_err(|e| extenddb_storage::error::StorageError::Internal(format!("Timestamp parse error: {e}"))) +} diff --git a/crates/storage-sqlite/src/stream_engine.rs b/crates/storage-sqlite/src/stream_engine.rs new file mode 100644 index 00000000..33738fcf --- /dev/null +++ b/crates/storage-sqlite/src/stream_engine.rs @@ -0,0 +1,491 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `StreamEngine` trait implementation for `SqliteEngine`. + +use extenddb_core::types::{ + SequenceNumberRange, Shard, StreamDescription, StreamRecord, StreamStatus, StreamSummary, + StreamViewType, +}; +use extenddb_storage::StreamEngine; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{parse_stream_arn, stream_arn}; +use futures::future::BoxFuture; + +use crate::engine::SqliteEngine; + +/// Number of fixed shards per stream. +const SHARDS_PER_STREAM: u32 = 4; + +impl SqliteEngine { + /// Initialize stream shards for a table and set the stream_label. + pub(crate) async fn init_stream_shards( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + account_id: &str, + table_name: &str, + table_id: &str, + ) -> Result { + let label: String = sqlx::query_scalar( + "UPDATE tables \ + SET stream_label = strftime('%Y-%m-%dT%H:%M:%S', 'now') \ + WHERE account_id = ? AND table_name = ? \ + RETURNING stream_label", + ) + .bind(account_id) + .bind(table_name) + .fetch_one(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + for i in 0..SHARDS_PER_STREAM { + let shard_id = format!("shardId-{table_name}-{i:012}"); + let start_seq = format!("{:021}", 0); + sqlx::query( + "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number) \ + VALUES (?, ?, ?)", + ) + .bind(&shard_id) + .bind(table_id) + .bind(&start_seq) + .execute(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + Ok(label) + } +} + +impl StreamEngine for SqliteEngine { + fn write_stream_record( + &self, + account_id: &str, + record: &StreamRecord, + shard_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let account_id = account_id.to_string(); + let record = record.clone(); + let shard_id = shard_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { + let record_json = serde_json::to_string(&record) + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let table_id: String = sqlx::query_scalar( + "SELECT table_id FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_one(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + sqlx::query( + "INSERT INTO stream_records \ + (sequence_number, shard_id, table_id, event_name, record_data) \ + VALUES (?, ?, ?, ?, ?)", + ) + .bind(&record.dynamodb.sequence_number) + .bind(&shard_id) + .bind(&table_id) + .bind(format!("{:?}", record.event_name)) + .bind(&record_json) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(()) + }) + } + + fn get_stream_records( + &self, + shard_id: &str, + after_sequence: Option<&str>, + limit: i64, + ) -> BoxFuture<'_, Result<(Vec, Option), StorageError>> { + let shard_id = shard_id.to_string(); + let after_sequence = after_sequence.map(|s| s.to_string()); + Box::pin(async move { + let rows: Vec<(String,)> = if let Some(after) = after_sequence { + sqlx::query_as( + "SELECT record_data FROM stream_records \ + WHERE shard_id = ? AND sequence_number > ? \ + ORDER BY sequence_number LIMIT ?", + ) + .bind(&shard_id) + .bind(&after) + .bind(limit) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + } else { + sqlx::query_as( + "SELECT record_data FROM stream_records \ + WHERE shard_id = ? \ + ORDER BY sequence_number LIMIT ?", + ) + .bind(&shard_id) + .bind(limit) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + }; + + let records: Vec = rows + .into_iter() + .map(|(data,)| { + serde_json::from_str(&data).map_err(|e| StorageError::Internal(e.to_string())) + }) + .collect::, _>>()?; + + let last_seq = records.last().map(|r| r.dynamodb.sequence_number.clone()); + Ok((records, last_seq)) + }) + } + + fn describe_stream( + &self, + account_id: &str, + input: &extenddb_core::types::DescribeStreamInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let stream_arn_val = input.stream_arn.clone(); + let limit = input.limit; + let exclusive_start_shard_id = input.exclusive_start_shard_id.clone(); + Box::pin(async move { + let (table_name, stream_label) = parse_stream_arn(&stream_arn_val)?; + + let row: Option<(String, String, Option, String, String)> = sqlx::query_as( + "SELECT key_schema, attribute_definitions, stream_specification, \ + table_status, table_id \ + FROM tables \ + WHERE account_id = ? AND table_name = ? AND stream_label = ?", + ) + .bind(&account_id) + .bind(&table_name) + .bind(&stream_label) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (ks_str, _ad_str, stream_spec_str, table_status, table_id) = + row.ok_or_else(|| { + StorageError::TableNotFound(format!( + "Requested resource not found: Stream: {arn} not found.", + arn = stream_arn_val + )) + })?; + + let key_schema = serde_json::from_str(&ks_str) + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let stream_view_type = stream_spec_str + .as_deref() + .and_then(|s| serde_json::from_str::(s).ok()) + .and_then(|v| { + v.get("StreamViewType") + .and_then(|sv| serde_json::from_value::(sv.clone()).ok()) + }) + .unwrap_or(StreamViewType::KeysOnly); + + let limit = limit.unwrap_or(100); + let shard_rows: Vec<(String, Option, String, Option)> = + if let Some(ref start) = exclusive_start_shard_id { + sqlx::query_as( + "SELECT shard_id, parent_shard_id, starting_sequence_number, \ + ending_sequence_number \ + FROM stream_shards WHERE table_id = ? AND shard_id > ? \ + ORDER BY shard_id LIMIT ?", + ) + .bind(&table_id) + .bind(start) + .bind(limit + 1) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + } else { + sqlx::query_as( + "SELECT shard_id, parent_shard_id, starting_sequence_number, \ + ending_sequence_number \ + FROM stream_shards WHERE table_id = ? \ + ORDER BY shard_id LIMIT ?", + ) + .bind(&table_id) + .bind(limit + 1) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + }; + + #[allow(clippy::cast_sign_loss)] + let limit_usize = limit as usize; + let last_shard = if shard_rows.len() > limit_usize { + Some(shard_rows[limit_usize - 1].0.clone()) + } else { + None + }; + + let shards: Vec = shard_rows + .into_iter() + .take(limit_usize) + .map(|(id, parent, start, end)| Shard { + shard_id: id, + parent_shard_id: parent, + sequence_number_range: SequenceNumberRange { + starting_sequence_number: start, + ending_sequence_number: end, + }, + }) + .collect(); + + let stream_status = if table_status == "DELETING" { + StreamStatus::Disabling + } else { + StreamStatus::Enabled + }; + + Ok(StreamDescription { + stream_arn: stream_arn_val, + stream_label, + stream_status, + stream_view_type, + table_name, + key_schema, + shards, + last_evaluated_shard_id: last_shard, + }) + }) + } + + fn list_streams( + &self, + account_id: &str, + table_name: Option<&str>, + limit: i64, + exclusive_start_stream_arn: Option<&str>, + ) -> BoxFuture<'_, Result<(Vec, Option), StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.map(|s| s.to_string()); + let exclusive_start_stream_arn = exclusive_start_stream_arn.map(|s| s.to_string()); + Box::pin(async move { + let rows: Vec<(String, String, String)> = match ( + table_name.as_deref(), + exclusive_start_stream_arn.as_deref(), + ) { + (Some(tn), Some(start_arn)) => { + let (_, start_label) = parse_stream_arn(start_arn)?; + sqlx::query_as( + "SELECT table_name, table_arn, stream_label FROM tables \ + WHERE account_id = ? AND stream_label IS NOT NULL \ + AND table_name = ? AND stream_label > ? \ + ORDER BY stream_label LIMIT ?", + ) + .bind(&account_id) + .bind(tn) + .bind(&start_label) + .bind(limit + 1) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + } + (Some(tn), None) => sqlx::query_as( + "SELECT table_name, table_arn, stream_label FROM tables \ + WHERE account_id = ? AND stream_label IS NOT NULL AND table_name = ? \ + ORDER BY stream_label LIMIT ?", + ) + .bind(&account_id) + .bind(tn) + .bind(limit + 1) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?, + (None, Some(start_arn)) => { + // SQLite doesn't support tuple comparison; expand it. + let (start_table, start_label) = parse_stream_arn(start_arn)?; + sqlx::query_as( + "SELECT table_name, table_arn, stream_label FROM tables \ + WHERE account_id = ? AND stream_label IS NOT NULL \ + AND (table_name > ? OR (table_name = ? AND stream_label > ?)) \ + ORDER BY table_name, stream_label LIMIT ?", + ) + .bind(&account_id) + .bind(&start_table) + .bind(&start_table) + .bind(&start_label) + .bind(limit + 1) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + } + (None, None) => sqlx::query_as( + "SELECT table_name, table_arn, stream_label FROM tables \ + WHERE account_id = ? AND stream_label IS NOT NULL \ + ORDER BY table_name, stream_label LIMIT ?", + ) + .bind(&account_id) + .bind(limit + 1) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?, + }; + + #[allow(clippy::cast_sign_loss)] + let limit_usize = limit as usize; + + let summaries: Vec = rows + .iter() + .take(limit_usize) + .map(|(tn, _table_arn, label)| StreamSummary { + stream_arn: stream_arn(&self.region, &account_id, tn, label), + stream_label: label.clone(), + table_name: tn.clone(), + }) + .collect(); + + let last_arn = if rows.len() > limit_usize { + summaries.last().map(|s| s.stream_arn.clone()) + } else { + None + }; + + Ok((summaries, last_arn)) + }) + } + + fn cleanup_expired_stream_records( + &self, + retention_hours: i64, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { + let result = sqlx::query( + "DELETE FROM stream_records \ + WHERE created_at < datetime('now', '-' || ? || ' hours')", + ) + .bind(retention_hours) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(result.rows_affected()) + }) + } + + fn assign_shard( + &self, + account_id: &str, + table_name: &str, + partition_key: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + let partition_key = partition_key.to_string(); + Box::pin(async move { + let table_id: String = sqlx::query_scalar( + "SELECT table_id FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(&account_id) + .bind(&table_name) + .fetch_one(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let shards: Vec<(String,)> = sqlx::query_as( + "SELECT shard_id FROM stream_shards \ + WHERE table_id = ? \ + ORDER BY shard_id", + ) + .bind(&table_id) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if shards.is_empty() { + return Err(StorageError::Internal(format!( + "No stream shards for table {table_name}" + ))); + } + + let hash = crc32fast::hash(partition_key.as_bytes()); + #[allow(clippy::cast_possible_truncation)] + let idx = (hash as usize) % shards.len(); + Ok(shards[idx].0.clone()) + }) + } + + fn next_sequence_number(&self, _shard_id: &str) -> BoxFuture<'_, Result> { + Box::pin(async move { + let (seq_val,): (i64,) = sqlx::query_as( + "UPDATE seq_counters SET value = value + 1 WHERE name = 'stream' RETURNING value", + ) + .fetch_one(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(format!("{seq_val:021}")) + }) + } + + fn validate_shard( + &self, + account_id: &str, + stream_arn: &str, + shard_id: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let account_id = account_id.to_string(); + let stream_arn = stream_arn.to_string(); + let shard_id = shard_id.to_string(); + Box::pin(async move { + let (table_name, stream_label) = parse_stream_arn(&stream_arn)?; + + let table_id: Option = sqlx::query_scalar( + "SELECT table_id FROM tables \ + WHERE account_id = ? AND table_name = ? AND stream_label = ?", + ) + .bind(&account_id) + .bind(&table_name) + .bind(&stream_label) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let Some(table_id) = table_id else { + return Err(StorageError::TableNotFound(format!( + "Requested resource not found: Stream: {stream_arn} not found." + ))); + }; + + let exists: Option<(i32,)> = sqlx::query_as( + "SELECT 1 FROM stream_shards WHERE shard_id = ? AND table_id = ?", + ) + .bind(&shard_id) + .bind(&table_id) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if exists.is_none() { + return Err(StorageError::TableNotFound(format!( + "Requested resource not found: Stream: {stream_arn} not found." + ))); + } + Ok(()) + }) + } + + fn latest_sequence_number( + &self, + shard_id: &str, + ) -> BoxFuture<'_, Result, StorageError>> { + let shard_id = shard_id.to_string(); + Box::pin(async move { + let row: Option<(String,)> = sqlx::query_as( + "SELECT sequence_number FROM stream_records \ + WHERE shard_id = ? ORDER BY sequence_number DESC LIMIT 1", + ) + .bind(&shard_id) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(row.map(|(s,)| s)) + }) + } +} diff --git a/crates/storage-sqlite/src/table_engine.rs b/crates/storage-sqlite/src/table_engine.rs new file mode 100644 index 00000000..c0860b0a --- /dev/null +++ b/crates/storage-sqlite/src/table_engine.rs @@ -0,0 +1,146 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `TableEngine` trait implementation for `SqliteEngine`. + +use futures::future::BoxFuture; + +use extenddb_core::types::{ + CreateTableInput, DeleteTableInput, DescribeTableInput, IndexInfo, ListTablesInput, + ListTablesOutput, TableDescription, TableKeyInfo, +}; +use extenddb_storage::TableEngine; +use extenddb_storage::error::StorageError; + +use crate::engine::SqliteEngine; + +impl TableEngine for SqliteEngine { + fn create_table( + &self, + account_id: &str, + input: CreateTableInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + Box::pin(async move { self.create_table_impl(&account_id, input).await }) + } + + fn delete_table( + &self, + account_id: &str, + input: DeleteTableInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + Box::pin(async move { self.delete_table_impl(&account_id, input).await }) + } + + fn describe_table( + &self, + account_id: &str, + input: DescribeTableInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + Box::pin(async move { + self.build_table_description(&account_id, &input.table_name) + .await + }) + } + + fn list_tables( + &self, + account_id: &str, + input: ListTablesInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + Box::pin(async move { + let limit = i64::from(input.limit.unwrap_or(100)); + + let rows: Vec<(String,)> = if let Some(ref start) = input.exclusive_start_table_name { + sqlx::query_as( + "SELECT table_name FROM tables \ + WHERE account_id = ? AND table_name > ? \ + ORDER BY table_name LIMIT ?", + ) + .bind(&account_id) + .bind(start) + .bind(limit + 1) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + } else { + sqlx::query_as( + "SELECT table_name FROM tables \ + WHERE account_id = ? \ + ORDER BY table_name LIMIT ?", + ) + .bind(&account_id) + .bind(limit + 1) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + }; + + let names: Vec = rows.into_iter().map(|(n,)| n).collect(); + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + let limit_usize = limit as usize; + + if names.len() > limit_usize { + Ok(ListTablesOutput { + last_evaluated_table_name: Some(names[limit_usize - 1].clone()), + table_names: names[..limit_usize].to_vec(), + }) + } else { + Ok(ListTablesOutput { + table_names: names, + last_evaluated_table_name: None, + }) + } + }) + } + + fn table_key_info( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { self.fetch_table_key_info(&account_id, &table_name).await }) + } + + fn index_info( + &self, + account_id: &str, + table_name: &str, + index_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + let index_name = index_name.to_string(); + Box::pin(async move { + self.fetch_index_info(&account_id, &table_name, &index_name) + .await + }) + } + + fn index_info_by_table_id( + &self, + table_id: &str, + index_name: &str, + ) -> BoxFuture<'_, Result> { + let table_id = table_id.to_string(); + let index_name = index_name.to_string(); + Box::pin(async move { + self.fetch_index_info_by_table_id(&table_id, &index_name) + .await + }) + } + + fn update_table( + &self, + account_id: &str, + input: extenddb_core::types::UpdateTableInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + Box::pin(async move { self.update_table_impl(&account_id, input).await }) + } +} diff --git a/crates/storage-sqlite/src/table_helpers.rs b/crates/storage-sqlite/src/table_helpers.rs new file mode 100644 index 00000000..07d89430 --- /dev/null +++ b/crates/storage-sqlite/src/table_helpers.rs @@ -0,0 +1,322 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Helper types and methods for `TableEngine` operations (SQLite backend). + +use extenddb_core::types::{ + AttributeDefinition, BillingMode, BillingModeSummary, GsiDescription, KeySchemaElement, + LsiDescription, Projection, ProvisionedThroughputDescription, TableDescription, TableStatus, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{index_arn, stream_arn}; + +use crate::data; +use crate::engine::SqliteEngine; + +/// Row type for table metadata queries (SQLite version with TEXT timestamps). +#[derive(sqlx::FromRow)] +pub(crate) struct TableRow { + pub table_name: String, + pub key_schema: String, + pub attribute_definitions: String, + pub billing_mode: String, + pub provisioned_throughput: Option, + pub stream_specification: Option, + pub table_status: String, + pub creation_date_time: Option, + pub table_size_bytes: i64, + pub item_count: i64, + pub table_arn: String, + pub table_id: String, + pub deletion_protection_enabled: bool, + pub stream_label: Option, +} + +/// Row type for index metadata queries (SQLite version with TEXT JSON). +#[derive(sqlx::FromRow)] +pub(crate) struct IndexRow { + pub index_name: String, + pub index_id: String, + pub index_type: String, + pub key_schema: String, + pub projection: String, + pub index_status: String, + pub provisioned_throughput: Option, +} + +impl SqliteEngine { + /// SQL table name for a GSI data table (for use outside `data` module). + pub(crate) fn index_table_name_static(index_id: &str) -> String { + data::index_table_name(index_id) + } + + /// Backfill existing items from the base table into a newly created GSI. + #[allow(clippy::too_many_arguments)] + pub(crate) async fn backfill_gsi( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, + index_id: &str, + index_key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + base_key_schema: &[KeySchemaElement], + base_attr_defs: &[AttributeDefinition], + projection: &Projection, + ) -> Result<(), StorageError> { + const BATCH_SIZE: i64 = 500; + + let base_table = data::data_table_name(table_id); + let idx_table = data::index_table_name(index_id); + + let idx_sks = data::all_sort_key_info(index_key_schema, attr_defs); + let base_sks = data::all_sort_key_info(base_key_schema, base_attr_defs); + + let sql = format!( + "SELECT item_data FROM {base_table} ORDER BY pk, sk_s, sk_n, sk_b LIMIT ? OFFSET ?" + ); + let mut offset: i64 = 0; + loop { + let rows: Vec<(serde_json::Value,)> = sqlx::query_as(&sql) + .bind(BATCH_SIZE) + .bind(offset) + .fetch_all(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if rows.is_empty() { + break; + } + + let batch_len = i64::from(u16::try_from(rows.len()).unwrap_or(u16::MAX)); + + for (item_json,) in rows { + let item = data::json_to_item(item_json)?; + + let has_all_keys = index_key_schema + .iter() + .all(|ks| item.contains_key(&ks.attribute_name)); + if !has_all_keys { + continue; + } + + let projected = data::project_item_for_index( + &item, + index_key_schema, + base_key_schema, + projection, + ); + + data::insert_index_row_multi( + tx, + &idx_table, + &item, + &projected, + index_key_schema, + base_key_schema, + attr_defs, + &idx_sks, + &base_sks, + ) + .await?; + } + + if batch_len < BATCH_SIZE { + break; + } + offset += batch_len; + } + + Ok(()) + } + + pub(crate) async fn build_table_description( + &self, + account_id: &str, + table_name: &str, + ) -> Result { + let row: Option = sqlx::query_as( + "SELECT table_name, key_schema, attribute_definitions, billing_mode, \ + provisioned_throughput, stream_specification, table_status, \ + creation_date_time, \ + table_size_bytes, item_count, table_arn, table_id, \ + deletion_protection_enabled, stream_label \ + FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let row = row.ok_or_else(|| StorageError::TableNotFound(table_name.to_owned()))?; + + let index_rows: Vec = sqlx::query_as( + "SELECT index_name, index_id, index_type, key_schema, projection, \ + index_status, provisioned_throughput \ + FROM indexes WHERE table_id = ?", + ) + .bind(&row.table_id) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + self.build_table_description_from_row(account_id, row, index_rows) + } + + pub(crate) fn build_table_description_from_row( + &self, + account_id: &str, + row: TableRow, + index_rows: Vec, + ) -> Result { + let mut gsis: Vec = Vec::new(); + let mut lsis: Vec = Vec::new(); + + for idx in index_rows { + let ks: Vec = serde_json::from_str(&idx.key_schema) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let proj: Projection = serde_json::from_str(&idx.projection) + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if idx.index_type == "GSI" { + let pt: Option = idx + .provisioned_throughput + .as_deref() + .map(|s| { + serde_json::from_str::(s) + .or_else(|_| { + let old: extenddb_core::types::ProvisionedThroughput = + serde_json::from_str(s)?; + Ok(ProvisionedThroughputDescription { + read_capacity_units: old.read_capacity_units, + write_capacity_units: old.write_capacity_units, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }) + }) + }) + .transpose() + .map_err(|e: serde_json::Error| StorageError::Internal(e.to_string()))?; + + gsis.push(GsiDescription { + index_name: idx.index_name.clone(), + key_schema: ks, + projection: proj, + index_status: idx.index_status, + provisioned_throughput: pt.or(Some(ProvisionedThroughputDescription { + read_capacity_units: 0, + write_capacity_units: 0, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + })), + index_size_bytes: 0, + item_count: 0, + index_arn: index_arn( + &self.region, + account_id, + &row.table_name, + &idx.index_name, + ), + }); + } else { + lsis.push(LsiDescription { + index_name: idx.index_name.clone(), + key_schema: ks, + projection: proj, + index_size_bytes: 0, + item_count: 0, + index_arn: index_arn( + &self.region, + account_id, + &row.table_name, + &idx.index_name, + ), + }); + } + } + + let key_schema: Vec = serde_json::from_str(&row.key_schema) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let attr_defs: Vec = + serde_json::from_str(&row.attribute_definitions) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let stream_spec = row + .stream_specification + .as_deref() + .map(serde_json::from_str) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (rcu, wcu) = match &row.provisioned_throughput { + Some(s) => { + let pt: extenddb_core::types::ProvisionedThroughput = serde_json::from_str(s) + .map_err(|e| StorageError::Internal(e.to_string()))?; + (pt.read_capacity_units, pt.write_capacity_units) + } + None => (0, 0), + }; + + let table_status = match row.table_status.as_str() { + "ACTIVE" => TableStatus::Active, + "CREATING" => TableStatus::Creating, + "DELETING" => TableStatus::Deleting, + "UPDATING" => TableStatus::Updating, + other => { + return Err(StorageError::Internal(format!( + "unknown table status in database: {other}" + ))); + } + }; + + // Compute creation epoch from stored text timestamp + let creation_epoch = row + .creation_date_time + .as_deref() + .and_then(|s| crate::sqlite_util::parse_timestamp(s).ok()) + .map(|dt| dt.unix_timestamp() as f64) + .unwrap_or(0.0); + + let billing_mode_summary = if row.billing_mode == "PAY_PER_REQUEST" { + Some(BillingModeSummary { + billing_mode: BillingMode::PayPerRequest, + last_update_to_pay_per_request_date_time: Some(creation_epoch), + }) + } else { + None + }; + + let latest_stream_arn = row + .stream_label + .as_ref() + .map(|label| stream_arn(&self.region, account_id, &row.table_name, label)); + + Ok(TableDescription { + table_name: row.table_name, + key_schema, + attribute_definitions: attr_defs, + table_status, + creation_date_time: creation_epoch, + table_size_bytes: row.table_size_bytes, + item_count: row.item_count, + table_arn: row.table_arn, + table_id: row.table_id, + provisioned_throughput: ProvisionedThroughputDescription { + read_capacity_units: rcu, + write_capacity_units: wcu, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }, + billing_mode_summary, + global_secondary_indexes: if gsis.is_empty() { None } else { Some(gsis) }, + local_secondary_indexes: if lsis.is_empty() { None } else { Some(lsis) }, + stream_specification: stream_spec, + latest_stream_arn, + latest_stream_label: row.stream_label, + deletion_protection_enabled: row.deletion_protection_enabled, + sse_description: None, + table_class_summary: None, + }) + } +} diff --git a/crates/storage-sqlite/src/update_table.rs b/crates/storage-sqlite/src/update_table.rs new file mode 100644 index 00000000..150be815 --- /dev/null +++ b/crates/storage-sqlite/src/update_table.rs @@ -0,0 +1,385 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `update_table` implementation for `SqliteEngine`. + +use extenddb_core::types::{ + AttributeDefinition, BillingMode, KeySchemaElement, TableDescription, UpdateTableInput, +}; +use extenddb_storage::error::StorageError; + +use crate::engine::SqliteEngine; + +impl SqliteEngine { + pub(crate) async fn update_table_impl( + &self, + account_id: &str, + input: UpdateTableInput, + ) -> Result { + Self::validate_account_id(account_id)?; + let mut tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let row: Option<(String, String, String, String)> = sqlx::query_as( + "SELECT table_status, table_id, key_schema, attribute_definitions \ + FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(account_id) + .bind(&input.table_name) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (status, table_id, ks_str, ad_str) = + row.ok_or_else(|| StorageError::TableNotFound(input.table_name.clone()))?; + if status != "ACTIVE" { + return Err(StorageError::TableNotActive(input.table_name.clone())); + } + + // No-op rejection for same PROVISIONED billing mode/throughput. + if matches!(input.billing_mode, Some(BillingMode::Provisioned)) { + if let Some(ref pt) = input.provisioned_throughput { + let current_row: Option<(Option, Option)> = sqlx::query_as( + "SELECT billing_mode, provisioned_throughput FROM tables \ + WHERE account_id = ? AND table_name = ?", + ) + .bind(account_id) + .bind(&input.table_name) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if let Some((current_bm, current_pt_opt)) = current_row { + let current_pt_val: serde_json::Value = current_pt_opt + .as_deref() + .and_then(|s| serde_json::from_str(s).ok()) + .unwrap_or(serde_json::Value::Object(Default::default())); + let is_provisioned = + current_bm.as_deref() == Some("PROVISIONED") || current_bm.is_none(); + let current_rcu = current_pt_val + .get("ReadCapacityUnits") + .or_else(|| current_pt_val.get("read_capacity_units")) + .and_then(|v| v.as_i64()) + .unwrap_or(0); + let current_wcu = current_pt_val + .get("WriteCapacityUnits") + .or_else(|| current_pt_val.get("write_capacity_units")) + .and_then(|v| v.as_i64()) + .unwrap_or(0); + + if is_provisioned + && current_rcu == pt.read_capacity_units + && current_wcu == pt.write_capacity_units + { + return Err(StorageError::NoOpUpdate(format!( + "The provisioned throughput for the table will not change. \ + The requested value equals the current value. \ + Current ReadCapacityUnits provisioned for the table: {}. \ + Requested ReadCapacityUnits: {}. \ + Current WriteCapacityUnits provisioned for the table: {}. \ + Requested WriteCapacityUnits: {}.", + current_rcu, + pt.read_capacity_units, + current_wcu, + pt.write_capacity_units + ))); + } + } + } + } + + // Apply billing mode change. + if let Some(bm) = &input.billing_mode { + let bm_str = match bm { + BillingMode::Provisioned => "PROVISIONED", + BillingMode::PayPerRequest => "PAY_PER_REQUEST", + }; + sqlx::query( + "UPDATE tables SET billing_mode = ? WHERE account_id = ? AND table_name = ?", + ) + .bind(bm_str) + .bind(account_id) + .bind(&input.table_name) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + // Apply provisioned throughput change. + if let Some(pt) = &input.provisioned_throughput { + let pt_json = serde_json::to_string(pt) + .map_err(|e| StorageError::Internal(e.to_string()))?; + sqlx::query( + "UPDATE tables SET provisioned_throughput = ? \ + WHERE account_id = ? AND table_name = ?", + ) + .bind(&pt_json) + .bind(account_id) + .bind(&input.table_name) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + // Apply deletion protection change. + if let Some(dp) = input.deletion_protection_enabled { + sqlx::query( + "UPDATE tables SET deletion_protection_enabled = ? \ + WHERE account_id = ? AND table_name = ?", + ) + .bind(dp) + .bind(account_id) + .bind(&input.table_name) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + // Apply stream specification change. + if let Some(spec) = &input.stream_specification { + let spec_json = serde_json::to_string(spec) + .map_err(|e| StorageError::Internal(e.to_string()))?; + sqlx::query( + "UPDATE tables SET stream_specification = ? \ + WHERE account_id = ? AND table_name = ?", + ) + .bind(&spec_json) + .bind(account_id) + .bind(&input.table_name) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if spec.stream_enabled { + let existing: Option<(String,)> = sqlx::query_as( + "SELECT shard_id FROM stream_shards WHERE table_id = ? LIMIT 1", + ) + .bind(&table_id) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if existing.is_none() { + Self::init_stream_shards( + &mut tx, + account_id, + &input.table_name, + &table_id, + ) + .await?; + } else { + let current_label: Option = sqlx::query_scalar( + "SELECT stream_label FROM tables \ + WHERE account_id = ? AND table_name = ?", + ) + .bind(account_id) + .bind(&input.table_name) + .fetch_one(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if current_label.is_none() { + sqlx::query( + "UPDATE tables \ + SET stream_label = strftime('%Y-%m-%dT%H:%M:%S', 'now') \ + WHERE account_id = ? AND table_name = ?", + ) + .bind(account_id) + .bind(&input.table_name) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + } + } + } + + // Apply GSI updates (create/delete). + let mut created_index_ids: Vec = Vec::new(); + let mut deleted_index_ids: Vec = Vec::new(); + if let Some(updates) = &input.global_secondary_index_updates { + for update in updates { + if let Some(create) = &update.create { + let existing: Option<(String,)> = sqlx::query_as( + "SELECT index_name FROM indexes WHERE table_id = ? AND index_name = ?", + ) + .bind(&table_id) + .bind(&create.index_name) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if existing.is_some() { + return Err(StorageError::IndexAlreadyExists(create.index_name.clone())); + } + + let gsi_ks = serde_json::to_string(&create.key_schema) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let gsi_proj = serde_json::to_string(&create.projection) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let gsi_pt = create + .provisioned_throughput + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let index_id = uuid::Uuid::new_v4().to_string(); + sqlx::query( + "INSERT INTO indexes \ + (table_id, index_name, index_id, index_type, key_schema, projection, \ + index_status, provisioned_throughput) \ + VALUES (?, ?, ?, 'GSI', ?, ?, 'ACTIVE', ?)", + ) + .bind(&table_id) + .bind(&create.index_name) + .bind(&index_id) + .bind(&gsi_ks) + .bind(&gsi_proj) + .bind(&gsi_pt) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + created_index_ids.push(index_id); + } + + if let Some(delete) = &update.delete { + let existing: Option<(String, String)> = sqlx::query_as( + "SELECT index_name, index_id FROM indexes \ + WHERE table_id = ? AND index_name = ?", + ) + .bind(&table_id) + .bind(&delete.index_name) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let (_, del_index_id) = existing + .ok_or_else(|| StorageError::IndexNotFound(delete.index_name.clone()))?; + + sqlx::query("DELETE FROM indexes WHERE table_id = ? AND index_name = ?") + .bind(&table_id) + .bind(&delete.index_name) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + deleted_index_ids.push(del_index_id); + } + } + + // Update attribute_definitions if new ones provided. + if let Some(new_attr_defs) = &input.attribute_definitions { + let ad_json = serde_json::to_string(new_attr_defs) + .map_err(|e| StorageError::Internal(e.to_string()))?; + sqlx::query( + "UPDATE tables SET attribute_definitions = ? \ + WHERE account_id = ? AND table_name = ?", + ) + .bind(&ad_json) + .bind(account_id) + .bind(&input.table_name) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + } + + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Execute data DDL after catalog commit. + if let Some(updates) = &input.global_secondary_index_updates { + let base_key_schema: Vec = serde_json::from_str(&ks_str) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let base_attr_defs: Vec = serde_json::from_str(&ad_str) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let effective_attr_defs = input + .attribute_definitions + .as_deref() + .unwrap_or(&base_attr_defs); + + let mut create_idx = 0usize; + let mut delete_idx = 0usize; + for update in updates { + if let Some(create) = &update.create { + let idx_id = &created_index_ids[create_idx]; + create_idx += 1; + let data_result = async { + let mut data_tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Self::create_index_data_table( + &mut data_tx, + idx_id, + &create.key_schema, + effective_attr_defs, + &base_key_schema, + &base_attr_defs, + ) + .await?; + + Self::backfill_gsi( + &mut data_tx, + &table_id, + idx_id, + &create.key_schema, + effective_attr_defs, + &base_key_schema, + &base_attr_defs, + &create.projection, + ) + .await?; + + data_tx + .commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok::<(), StorageError>(()) + } + .await; + + if let Err(e) = data_result { + tracing::error!( + "Failed to create data table for GSI '{}' on '{}': {e}", + create.index_name, + input.table_name, + ); + let _ = sqlx::query( + "DELETE FROM indexes WHERE table_id = ? AND index_name = ?", + ) + .bind(&table_id) + .bind(&create.index_name) + .execute(&self.pool) + .await; + return Err(e); + } + } + + if update.delete.is_some() { + let idx_id = &deleted_index_ids[delete_idx]; + delete_idx += 1; + let idx_table = Self::index_table_name_static(idx_id); + if let Err(e) = sqlx::query(&format!("DROP TABLE IF EXISTS {idx_table}")) + .execute(&self.pool) + .await + { + tracing::warn!( + "Failed to drop data table for deleted GSI on '{}': {e}", + input.table_name, + ); + } + } + } + } + + self.build_table_description(account_id, &input.table_name) + .await + } +} diff --git a/crates/storage-sqlite/src/worker_store.rs b/crates/storage-sqlite/src/worker_store.rs new file mode 100644 index 00000000..e177a228 --- /dev/null +++ b/crates/storage-sqlite/src/worker_store.rs @@ -0,0 +1,104 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `WorkerStore` trait implementation and control plane transition processing. + +use futures::future::BoxFuture; + +use extenddb_storage::WorkerStore; +use extenddb_storage::error::StorageError; + +use crate::engine::SqliteEngine; + +impl WorkerStore for SqliteEngine { + fn process_control_plane_transitions( + &self, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { Self::process_control_plane_transitions(self).await }) + } +} + +impl SqliteEngine { + /// Process pending control plane transitions (H-5). + /// + /// Tables in CREATING state whose `status_transition_at` has passed are + /// moved to ACTIVE. Tables in DELETING state whose transition time has + /// passed are removed. + pub async fn process_control_plane_transitions( + &self, + ) -> Result, StorageError> { + let mut transitions = Vec::new(); + + // CREATING → ACTIVE + let activated: Vec<(String,)> = sqlx::query_as( + "UPDATE tables SET table_status = 'ACTIVE', status_transition_at = NULL \ + WHERE table_status = 'CREATING' AND status_transition_at <= datetime('now') \ + RETURNING table_name", + ) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + for (name,) in activated { + transitions.push((name, "CREATING → active")); + } + + // DELETING → remove. Collect index ids before deletion since CASCADE + // removes them when the table row is deleted. + let candidates: Vec<(String, String, String, String)> = sqlx::query_as( + "SELECT account_id, table_name, table_arn, table_id FROM tables \ + WHERE table_status = 'DELETING' AND status_transition_at <= datetime('now')", + ) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut drop_info: Vec<(String, Vec)> = Vec::new(); + + for (_acct_id, name, arn, table_id) in &candidates { + let index_ids: Vec<(String,)> = + sqlx::query_as("SELECT index_id FROM indexes WHERE table_id = ?") + .bind(table_id) + .fetch_all(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + sqlx::query("DELETE FROM tags WHERE resource_arn = ?") + .bind(arn) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + sqlx::query("DELETE FROM tables WHERE table_id = ?") + .bind(table_id) + .execute(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + drop_info.push(( + table_id.clone(), + index_ids.into_iter().map(|(n,)| n).collect(), + )); + + transitions.push((name.clone(), "DELETING → deleted")); + } + + // Drop data tables after catalog rows deleted. + for (table_id, index_ids) in &drop_info { + let mut data_tx = self + .pool + .begin() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + for idx_id in index_ids { + Self::drop_index_data_table(&mut data_tx, idx_id).await?; + } + Self::drop_data_table(&mut data_tx, table_id).await?; + data_tx + .commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + Ok(transitions) + } +} diff --git a/crates/storage-sqlite/src/workers.rs b/crates/storage-sqlite/src/workers.rs new file mode 100644 index 00000000..8d312268 --- /dev/null +++ b/crates/storage-sqlite/src/workers.rs @@ -0,0 +1,151 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! SQLite-specific background workers. + +use std::sync::Arc; +use std::time::Duration; + +use extenddb_core::metrics::MetricsCollector; +use extenddb_storage::management_store::SettingsStore; +use extenddb_storage::{DataEngine, MetadataEngine, StreamEngine}; + +use crate::engine::SqliteEngine; + +pub(crate) async fn poll_control_plane_transitions( + storage: Arc, + notify: Arc, + settings: Arc, +) { + const ACTIVE_POLL: Duration = Duration::from_secs(1); + const IDLE_TIMEOUT: Duration = Duration::from_secs(60); + const MARGIN_SECS: f64 = 5.0; + + loop { + let _ = tokio::time::timeout(IDLE_TIMEOUT, notify.notified()).await; + + let delay_secs = read_control_plane_delay(&*settings).await; + let active_window = Duration::from_secs_f64(delay_secs + MARGIN_SECS); + + let deadline = tokio::time::Instant::now() + active_window; + loop { + match storage.process_control_plane_transitions().await { + Ok(ref t) if t.is_empty() => {} + Ok(transitions) => { + for (name, transition) in &transitions { + tracing::info!("Table '{name}': {transition}"); + } + } + Err(e) => { + tracing::warn!("Control plane transition poll failed: {e}"); + break; + } + } + if tokio::time::Instant::now() >= deadline { + break; + } + tokio::time::sleep(ACTIVE_POLL).await; + } + } +} + +async fn read_control_plane_delay(store: &S) -> f64 { + store + .get_setting("control_plane_delay_seconds") + .await + .ok() + .flatten() + .and_then(|v| v.parse::().ok()) + .filter(|&v| v >= 0.0) + .unwrap_or(0.25) +} + +pub(crate) async fn table_size_refresh_worker(storage: Arc) { + const REFRESH_INTERVAL: Duration = Duration::from_secs(300); + + loop { + tokio::time::sleep(REFRESH_INTERVAL).await; + + let tables = match MetadataEngine::all_active_tables(&*storage).await { + Ok(t) => t, + Err(e) => { + tracing::warn!("Size refresh worker: failed to list tables: {e}"); + continue; + } + }; + + for (account_id, table_name) in &tables { + if let Err(e) = + MetadataEngine::refresh_table_size(&*storage, account_id, table_name).await + { + tracing::warn!("Size refresh worker: failed for {table_name}: {e}"); + } + } + } +} + +pub(crate) async fn stream_record_cleanup_worker( + storage: Arc, + metrics: Arc, +) { + use extenddb_core::metrics::QuerySource; + + const CLEANUP_INTERVAL: Duration = Duration::from_secs(3600); + const RETENTION_HOURS: i64 = 24; + + loop { + tokio::time::sleep(CLEANUP_INTERVAL).await; + let cycle_start = std::time::Instant::now(); + + match StreamEngine::cleanup_expired_stream_records(&*storage, RETENTION_HOURS).await { + Ok(0) => { + #[allow(clippy::cast_precision_loss)] + let cycle_us = cycle_start.elapsed().as_micros() as f64; + metrics.record_worker_success(QuerySource::StreamCleanup, cycle_us); + } + Ok(n) => { + tracing::info!("Stream cleanup worker: deleted {n} expired record(s)"); + #[allow(clippy::cast_precision_loss)] + let cycle_us = cycle_start.elapsed().as_micros() as f64; + metrics.record_worker_success(QuerySource::StreamCleanup, cycle_us); + } + Err(e) => { + tracing::error!("Stream record cleanup failed: {e}"); + metrics.record_worker_error(QuerySource::StreamCleanup); + } + } + } +} + +pub(crate) async fn idempotency_token_cleanup_worker( + storage: Arc, + metrics: Arc, +) { + use extenddb_core::metrics::QuerySource; + + const CLEANUP_INTERVAL: Duration = Duration::from_secs(600); + const MAX_AGE_SECONDS: i64 = 600; + + loop { + tokio::time::sleep(CLEANUP_INTERVAL).await; + let cycle_start = std::time::Instant::now(); + + match DataEngine::cleanup_expired_idempotency_tokens(&*storage, MAX_AGE_SECONDS).await { + Ok(0) => { + #[allow(clippy::cast_precision_loss)] + let cycle_us = cycle_start.elapsed().as_micros() as f64; + metrics.record_worker_success(QuerySource::IdempotencyCleanup, cycle_us); + } + Ok(n) => { + tracing::info!("Idempotency cleanup worker: deleted {n} expired token(s)"); + #[allow(clippy::cast_precision_loss)] + let cycle_us = cycle_start.elapsed().as_micros() as f64; + metrics.record_worker_success(QuerySource::IdempotencyCleanup, cycle_us); + } + Err(e) => { + tracing::error!("Idempotency token cleanup failed: {e}"); + metrics.record_worker_error(QuerySource::IdempotencyCleanup); + } + } + } +}