diff --git a/AGENTS.md b/AGENTS.md index ee53cbea..adc5707c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,7 +10,7 @@ by AWS engineers. It is not a fork of DynamoDB and contains no DynamoDB source c protocol: any AWS SDK, CLI, or tool that works with DynamoDB works with ExtendDB, unchanged. - **Language:** Rust (edition 2024, MSRV 1.88+) -- **Storage backend:** PostgreSQL 14+ +- **Storage backends:** PostgreSQL 14+ (default), MongoDB 7.0+ (feature flag `mongodb`) - **Architecture:** Async (tokio), trait-based storage abstraction - **Authentication:** Mandatory SigV4 with built-in IAM (users, groups, roles, policies) - **TLS:** Mandatory (self-signed cert generated by default) @@ -51,7 +51,8 @@ extenddb/ The `TableEngine` trait in `crates/storage/src/lib.rs` defines the storage interface. All storage backends implement this trait: -- **Current:** `storage-postgres` (PostgreSQL) +- `storage-postgres` (PostgreSQL) — default backend +- `storage-mongodb` (MongoDB) — feature flag `mongodb` The trait uses RPITIT (return-position impl Trait in traits) for async methods — no `#[async_trait]` macro. @@ -66,12 +67,14 @@ extenddb (bin) │ ├─> extenddb-core (pure sync, no async) │ └─> extenddb-storage (trait definitions) ├─> extenddb-auth - └─> extenddb-storage-postgres + ├─> extenddb-storage-postgres (feature: postgres) + └─> extenddb-storage-mongodb (feature: mongodb) ``` - **extenddb-core:** Pure synchronous Rust. No async, no I/O. Types, validation, expression parsing. - **extenddb-storage:** Trait definitions only. No implementation. - **extenddb-storage-postgres:** Concrete PostgreSQL implementation. +- **extenddb-storage-mongodb:** Concrete MongoDB implementation. - **extenddb-engine:** Operation handlers that call storage traits. - **extenddb-server:** HTTP server, management API, web console. - **extenddb-auth:** SigV4 signature verification, IAM policy evaluation. @@ -82,13 +85,22 @@ extenddb (bin) ### Prerequisites - Rust 1.88+ (`rustup update`) -- PostgreSQL 14+ running locally (see `docs/local-postgres-setup.md`) +- Storage backend (one of): + - PostgreSQL 14+ running locally (see `docs/local-postgres-setup.md`) + - MongoDB 7.0+ with replica set (see `docs/local-mongodb-setup.md`) - Python 3.10+ for tests (`python3 -m venv ~/venvs/extenddb-venv && source ~/venvs/extenddb-venv/bin/activate && pip install -r requirements.txt`) ### Build ```bash +# PostgreSQL backend (default) cargo build --release + +# MongoDB backend +cargo build --release --features mongodb + +# Both backends +cargo build --release --features postgres,mongodb ``` Binary: `target/release/extenddb` @@ -109,11 +121,15 @@ cargo clippy --all-targets -- -D warnings ### Initialize (first time only) ```bash +# PostgreSQL (default) ./target/release/extenddb init --config extenddb.toml + +# MongoDB +./target/release/extenddb init --backend mongodb --config extenddb.toml ``` This creates: -- PostgreSQL databases (`extenddb_catalog`, `extenddb_account_`) +- Databases (`extenddb_catalog`, `extenddb_data`) - Admin user credentials (printed to stdout — save the password!) - Self-signed TLS certificate at `~/.extenddb/tls/cert.pem` - Config file `extenddb.toml` @@ -357,6 +373,9 @@ Expression parsing lives in `crates/core/src/expression/`. This is pure sync Rus | Differences from DynamoDB | `docs/differences-from-dynamodb.md` | Behavioral differences | | Troubleshooting | `docs/troubleshooting.md` | Common errors and solutions | | Storage Component Design | `docs/design/04-component-storage.md` | Storage trait design | +| MongoDB Design | `docs/design/13-storage-mongodb.md` | MongoDB backend architecture | +| Local PostgreSQL Setup | `docs/local-postgres-setup.md` | PostgreSQL installation and config | +| Local MongoDB Setup | `docs/local-mongodb-setup.md` | MongoDB installation and config | | Testing Design | `docs/design/09-testing.md` | Test strategy and infrastructure | | Extending Storage | `12-backend-plugin-architecture.md` | Guide to implementing storage backends | @@ -448,9 +467,10 @@ Activate when the user asks about installing, configuring, running, or debugging 1. **TLS is mandatory** — server refuses to start without it. Use `AWS_CA_BUNDLE` for self-signed certs. 2. **Auth is mandatory** — all requests must be SigV4-signed. Use `extenddb manage` or web console to create credentials. 3. **Account isolation** — all operations are scoped to `account_id`. Different accounts can have tables with the same name. -4. **PostgreSQL must be running** — `extenddb init` and `extenddb serve` require a running PostgreSQL instance. -5. **Python venv** — activate the venv before running tests: `source ~/venvs/extenddb-venv/bin/activate` -6. **Test credentials** — run `devtools/provision-test-credentials` before pytest to create test users and keys. +4. **Storage backend must be running** — `extenddb init` and `extenddb serve` require a running PostgreSQL or MongoDB instance. +5. **MongoDB requires a replica set** — even single-node MongoDB must be configured with `--replSet` for transactions and streams. +6. **Python venv** — activate the venv before running tests: `source ~/venvs/extenddb-venv/bin/activate` +7. **Test credentials** — run `devtools/provision-test-credentials` before pytest to create test users and keys. ## Getting Help diff --git a/Cargo.lock b/Cargo.lock index 78e4d0d5..574474ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -26,7 +26,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -50,6 +50,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -122,15 +123,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arc-swap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" dependencies = [ "rustversion", ] @@ -165,7 +166,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] @@ -177,7 +178,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -205,13 +206,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.2", ] [[package]] @@ -237,9 +238,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.17.0" +version = "1.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" dependencies = [ "aws-lc-sys", "zeroize", @@ -247,14 +248,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.41.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -318,7 +320,7 @@ checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -388,6 +390,21 @@ dependencies = [ "serde", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec 0.8.0", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bit-vec" version = "0.9.1" @@ -399,13 +416,25 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.11.1" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" dependencies = [ "serde_core", ] +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -415,6 +444,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "blowfish" version = "0.9.1" @@ -425,6 +463,29 @@ dependencies = [ "cipher", ] +[[package]] +name = "bson" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969a9ba84b0ff843813e7249eed1678d9b6607ce5a3b8f0a47af3fcf7978e6e" +dependencies = [ + "ahash", + "base64 0.22.1", + "bitvec", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hex", + "indexmap", + "js-sys", + "once_cell", + "rand 0.9.5", + "serde", + "serde_bytes", + "serde_json", + "time", + "uuid", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -439,15 +500,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.2.62" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" dependencies = [ "find-msvc-tools", "jobserver", @@ -461,21 +522,32 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "cipher" version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] [[package]] name = "clap" -version = "4.6.1" +version = "4.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "0fb99565819980999fb7b4a1796046a5c949e6d4ff132cf5fadf5a641e20d776" dependencies = [ "clap_builder", "clap_derive", @@ -483,9 +555,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" dependencies = [ "anstream", "anstyle", @@ -495,14 +567,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "32f2392eae7f16557a3d727ef3a12e57b2b2ca6f98566a5f4fb41ffe305df077" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -520,12 +592,28 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "compression-codecs" version = "0.4.38" @@ -559,7 +647,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68578f196d2a33ff61b27fae256c3164f65e36382648e30666dde05b8cc9dfdf" dependencies = [ "async-trait", - "convert_case", + "convert_case 0.6.0", "json5", "nom", "pathdiff", @@ -577,6 +665,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -606,6 +700,31 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -615,6 +734,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.4.0" @@ -639,38 +767,44 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-channel" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crunchy" @@ -689,6 +823,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "ctr" version = "0.9.2" @@ -698,6 +841,15 @@ dependencies = [ "cipher", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "daemonize" version = "0.5.0" @@ -707,6 +859,54 @@ dependencies = [ "libc", ] +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data-encoding" version = "2.11.0" @@ -719,7 +919,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "pem-rfc7468", "zeroize", ] @@ -744,31 +944,87 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "powerfmt", "serde_core", ] +[[package]] +name = "derive-syn-parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive-where" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case 0.10.0", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", + "unicode-xid", +] + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -865,6 +1121,7 @@ dependencies = [ "anyhow", "extenddb-app", "extenddb-storage", + "extenddb-storage-mongodb", "extenddb-storage-postgres", ] @@ -907,9 +1164,9 @@ dependencies = [ "extenddb-core", "futures", "hex", - "hmac", + "hmac 0.12.1", "serde_json", - "sha2", + "sha2 0.10.9", "thiserror", "time", "tokio", @@ -964,10 +1221,10 @@ dependencies = [ "extenddb-core", "extenddb-storage", "hex", - "hmac", + "hmac 0.12.1", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "tokio", "tracing", "uuid", @@ -995,7 +1252,7 @@ dependencies = [ "hyper", "libc", "metrics", - "rand 0.9.4", + "rand 0.9.5", "rustls", "serde", "serde_json", @@ -1021,7 +1278,7 @@ dependencies = [ "extenddb-auth", "extenddb-core", "futures", - "rand 0.9.4", + "rand 0.9.5", "serde", "serde_json", "thiserror", @@ -1033,6 +1290,36 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "extenddb-storage-mongodb" +version = "0.1.2" +dependencies = [ + "aes-gcm", + "anyhow", + "async-trait", + "base64 0.22.1", + "bcrypt", + "bson", + "crc32fast", + "dashmap", + "extenddb-auth", + "extenddb-core", + "extenddb-storage", + "futures", + "mongodb", + "proptest", + "rand 0.9.5", + "serde", + "serde_json", + "thiserror", + "time", + "tokio", + "toml", + "tracing", + "uuid", + "zeroize", +] + [[package]] name = "extenddb-storage-postgres" version = "0.1.2" @@ -1047,7 +1334,7 @@ dependencies = [ "extenddb-core", "extenddb-storage", "futures", - "rand 0.9.4", + "rand 0.9.5", "serde", "serde_json", "sqlx", @@ -1060,6 +1347,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1110,9 +1403,9 @@ dependencies = [ [[package]] name = "fs-err" -version = "3.3.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fde052dbfc920003cfd2c8e2c6e6d4cc7c1091538c3a24226cec0665ab08c0" +checksum = "b91aa448ca50d7e79433bdf3ee8d99215430d2ec02ade5aefab2a073a1822e8a" dependencies = [ "autocfg", "tokio", @@ -1124,11 +1417,17 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" dependencies = [ "futures-channel", "futures-core", @@ -1141,9 +1440,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", "futures-sink", @@ -1151,15 +1450,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" dependencies = [ "futures-core", "futures-task", @@ -1179,38 +1478,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-channel", "futures-core", @@ -1240,8 +1539,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -1251,22 +1552,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 5.3.0", "wasip2", + "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", - "wasip2", - "wasip3", + "rand_core 0.10.1", ] [[package]] @@ -1281,9 +1583,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" dependencies = [ "atomic-waker", "bytes", @@ -1355,13 +1657,83 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-net" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "futures-channel", + "futures-io", + "futures-util", + "hickory-proto", + "idna", + "ipnet", + "jni", + "rand 0.10.2", + "thiserror", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "prefix-trie", + "rand 0.10.2", + "ring", + "thiserror", + "tinyvec", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-net", + "hickory-proto", + "ipconfig", + "ipnet", + "jni", + "moka", + "ndk-context", + "once_cell", + "parking_lot", + "rand 0.10.2", + "resolv-conf", + "smallvec", + "system-configuration", + "thiserror", + "tokio", + "tracing", +] + [[package]] name = "hkdf" version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", ] [[package]] @@ -1370,7 +1742,16 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", ] [[package]] @@ -1384,9 +1765,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -1394,9 +1775,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -1404,9 +1785,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", @@ -1427,11 +1808,20 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" -version = "1.9.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" dependencies = [ "atomic-waker", "bytes", @@ -1546,10 +1936,10 @@ dependencies = [ ] [[package]] -name = "id-arena" -version = "2.3.0" +name = "ident_case" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" @@ -1580,8 +1970,6 @@ checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", - "serde", - "serde_core", ] [[package]] @@ -1593,6 +1981,28 @@ dependencies = [ "generic-array", ] +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2", + "widestring", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +dependencies = [ + "serde", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -1605,25 +2015,73 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "jobserver" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.3", "libc", ] [[package]] name = "js-sys" -version = "0.3.99" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -1647,17 +2105,11 @@ dependencies = [ "spin", ] -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "libc" -version = "0.2.186" +version = "0.2.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "22053b6a34f84abc97f9129e61334f40174659a1b9bd18c970b83db6a9a6348b" [[package]] name = "libm" @@ -1667,14 +2119,14 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.16" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" +checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" dependencies = [ "bitflags", "libc", "plain", - "redox_syscall 0.7.5", + "redox_syscall 0.9.0", ] [[package]] @@ -1687,6 +2139,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.2" @@ -1704,9 +2162,57 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "macro_magic" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc33f9f0351468d26fbc53d9ce00a096c8522ecb42f19b50f34f2c422f76d21d" +dependencies = [ + "macro_magic_core", + "macro_magic_macros", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "macro_magic_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1687dc887e42f352865a393acae7cf79d98fab6351cde1f58e9e057da89bf150" +dependencies = [ + "const-random", + "derive-syn-parse", + "macro_magic_core_macros", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "macro_magic_core_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "macro_magic_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" +dependencies = [ + "macro_magic_core", + "quote", + "syn 2.0.119", +] [[package]] name = "matchers" @@ -1730,14 +2236,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", ] [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "metrics" @@ -1773,9 +2289,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.0" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi", @@ -1802,6 +2318,88 @@ dependencies = [ "uuid", ] +[[package]] +name = "mongocrypt" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8426a875ded61430d4a811dbfda7633b6b8af0225c547fc6c28b8b0aa7d79a13" +dependencies = [ + "bson", + "mongocrypt-sys", + "once_cell", + "serde", +] + +[[package]] +name = "mongocrypt-sys" +version = "0.1.6+1.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851fac73f7fe22f6a3ab87f720ce509cae7c9fd08e7dd27866cc232dee07ccf4" + +[[package]] +name = "mongodb" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b814038f367d212f55de0a630cb35102a9b8ca23785a86955d62c0087c93846d" +dependencies = [ + "base64 0.22.1", + "bitflags", + "bson", + "derive-where", + "derive_more", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hickory-net", + "hickory-proto", + "hickory-resolver", + "hmac 0.13.0", + "macro_magic", + "md-5 0.11.0", + "mongocrypt", + "mongodb-internal-macros", + "pbkdf2", + "percent-encoding", + "rand 0.9.5", + "rustc_version_runtime", + "rustls", + "serde", + "serde_bytes", + "serde_with", + "sha1 0.11.0", + "sha2 0.11.0", + "socket2", + "stringprep", + "strsim", + "take_mut", + "thiserror", + "tokio", + "tokio-rustls", + "tokio-util", + "typed-builder", + "uuid", + "webpki-roots 1.0.9", +] + +[[package]] +name = "mongodb-internal-macros" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f736d2fbc56e0011a341fbb9172bd822fda75c5f93b82fae1c7aab1e2613c810" +dependencies = [ + "macro_magic", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "nom" version = "7.1.3" @@ -1823,9 +2421,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.6" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", @@ -1842,7 +2440,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.6", + "rand 0.8.7", "smallvec", "zeroize", ] @@ -1864,11 +2462,10 @@ dependencies = [ [[package]] name = "num-iter" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" dependencies = [ - "autocfg", "num-integer", "num-traits", ] @@ -1897,6 +2494,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -1955,6 +2556,15 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "pbkdf2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "pem" version = "3.0.6" @@ -1982,9 +2592,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0848c601009d37dfa3430c4666e147e49cdcf1b92ecd3e63657d8a5f19da662" +checksum = "47627dd7305c6a2d6c8c6bcd24c5a4c17dbbf425f4f9c5313e724b38fc9782e9" dependencies = [ "memchr", "ucd-trie", @@ -1992,9 +2602,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11f486f1ea21e6c10ed15d5a7c77165d0ee443402f0780849d1768e7d9d6fe77" +checksum = "4b4254325ecad416ab689e27ba51da03ba01a9632bc6e108f5fe7c3c4ad29d58" dependencies = [ "pest", "pest_generator", @@ -2002,25 +2612,24 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8040c4647b13b210a963c1ed407c1ff4fdfa01c31d6d2a098218702e6664f94f" +checksum = "6c4c0e91ead7a8f7acecbca6f003fc2e8282b1dbe2dd9c9d2f16aba42995e0a7" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "pest_meta" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" +checksum = "f9744bc48116fee06334924bb5f2bad41eed5e89bd26e29b0b799f9a3f82c210" dependencies = [ "pest", - "sha2", ] [[package]] @@ -2069,16 +2678,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] [[package]] name = "portable-atomic" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" [[package]] name = "potential_utf" @@ -2105,29 +2714,55 @@ dependencies = [ ] [[package]] -name = "prettyplease" -version = "0.2.37" +name = "prefix-trie" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" dependencies = [ - "proc-macro2", - "syn", + "either", + "ipnet", + "num-traits", ] [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set", + "bit-vec 0.8.0", + "bitflags", + "num-traits", + "rand 0.9.5", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -2144,11 +2779,17 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -2157,14 +2798,25 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -2203,11 +2855,26 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "rapidhash" -version = "4.4.1" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e48930979c155e2f33aa36ab3119b5ee81332beb6482199a8ecd6029b80b59" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" dependencies = [ "rustversion", ] @@ -2237,18 +2904,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.7.5" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4666a1a60d8412eab19d94f6d13dcc9cea0a5ef4fdf6a5db306537413c661b1b" +checksum = "c5102a6aaa05aa011a238e178e6bca86d2cb56fc9f586d37cb80f5bca6e07759" dependencies = [ "bitflags", ] [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -2257,9 +2924,15 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "ring" @@ -2293,8 +2966,8 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", - "digest", + "const-oid 0.9.6", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", @@ -2317,20 +2990,52 @@ dependencies = [ "ordered-multimap", ] +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustc_version_runtime" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" +dependencies = [ + "rustc_version", + "semver", +] + [[package]] name = "rusticata-macros" version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" dependencies = [ - "nom", + "nom", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "aws-lc-rs", "log", @@ -2353,9 +3058,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" dependencies = [ "zeroize", ] @@ -2374,9 +3079,21 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "rusty-fork" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] [[package]] name = "ryu" @@ -2384,6 +3101,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2398,40 +3124,51 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.2", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap", "itoa", "memchr", "serde", @@ -2471,15 +3208,48 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "serde_core", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "sha1" -version = "0.10.6" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -2489,8 +3259,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -2504,9 +3285,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signal-hook-registry" @@ -2524,15 +3305,31 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", ] [[package]] name = "simd-adler32" -version = "0.3.9" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "slab" @@ -2542,18 +3339,18 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" dependencies = [ "serde", ] [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -2561,9 +3358,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" dependencies = [ "lock_api", ] @@ -2618,7 +3415,7 @@ dependencies = [ "rustls", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "thiserror", "time", @@ -2640,7 +3437,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn", + "syn 2.0.119", ] [[package]] @@ -2658,12 +3455,12 @@ dependencies = [ "quote", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "sqlx-core", "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn", + "syn 2.0.119", "tokio", "url", ] @@ -2681,7 +3478,7 @@ dependencies = [ "byteorder", "bytes", "crc", - "digest", + "digest 0.10.7", "dotenvy", "either", "futures-channel", @@ -2691,18 +3488,18 @@ dependencies = [ "generic-array", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", "percent-encoding", - "rand 0.8.6", + "rand 0.8.7", "rsa", "serde", - "sha1", - "sha2", + "sha1 0.10.7", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", @@ -2732,18 +3529,18 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "home", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "num-bigint", "once_cell", - "rand 0.8.6", + "rand 0.8.7", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", @@ -2811,9 +3608,20 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.117" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" dependencies = [ "proc-macro2", "quote", @@ -2834,7 +3642,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2848,49 +3656,94 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tagptr" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.2", ] [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ "cfg-if", ] [[package]] name = "time" -version = "0.3.47" +version = "0.3.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" dependencies = [ "deranged", - "itoa", "num-conv", "powerfmt", "serde_core", @@ -2900,15 +3753,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.27" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" dependencies = [ "num-conv", "time-core", @@ -2935,9 +3788,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" dependencies = [ "tinyvec_macros", ] @@ -2950,9 +3803,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -2967,13 +3820,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2999,13 +3852,16 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", + "futures-util", + "libc", "pin-project-lite", "tokio", ] @@ -3118,7 +3974,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3173,11 +4029,31 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "typed-builder" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "398a3a3c918c96de527dc11e6e846cd549d4508030b8a33e1da12789c856b81a" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e48cea23f68d1f78eb7bc092881b6bb88d3d6b5b7e6234f6f9c911da1ffb221" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "typenum" -version = "1.20.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "ucd-trie" @@ -3185,6 +4061,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicode-bidi" version = "0.3.18" @@ -3214,9 +4096,9 @@ checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" [[package]] name = "unicode-segmentation" -version = "1.13.2" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-xid" @@ -3230,7 +4112,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] @@ -3272,12 +4154,13 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.1" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ - "getrandom 0.4.2", + "getrandom 0.4.3", "js-sys", + "serde_core", "wasm-bindgen", ] @@ -3300,27 +4183,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" +name = "wait-timeout" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] [[package]] -name = "wasip2" -version = "1.0.3+wasi-0.2.9" +name = "walkdir" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ - "wit-bindgen 0.57.1", + "same-file", + "winapi-util", ] [[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ - "wit-bindgen 0.51.0", + "wit-bindgen", ] [[package]] @@ -3331,9 +4224,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -3344,9 +4237,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3354,93 +4247,103 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.119", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] [[package]] -name = "wasm-encoder" -version = "0.244.0" +name = "webpki-roots" +version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "leb128fmt", - "wasmparser", + "webpki-roots 1.0.9", ] [[package]] -name = "wasm-metadata" -version = "0.244.0" +name = "webpki-roots" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" dependencies = [ - "anyhow", - "indexmap", - "wasm-encoder", - "wasmparser", + "rustls-pki-types", ] [[package]] -name = "wasmparser" -version = "0.244.0" +name = "whoami" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" dependencies = [ - "bitflags", - "hashbrown 0.15.5", - "indexmap", - "semver", + "libredox", + "wasite", ] [[package]] -name = "webpki-roots" -version = "0.26.11" +name = "widestring" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "webpki-roots 1.0.7", + "windows-sys 0.61.2", ] [[package]] -name = "webpki-roots" -version = "1.0.7" +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" dependencies = [ - "rustls-pki-types", + "windows-link", + "windows-result", + "windows-strings", ] [[package]] -name = "whoami" -version = "1.6.1" +name = "windows-result" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "libredox", - "wasite", + "windows-link", ] [[package]] -name = "windows-link" -version = "0.2.1" +name = "windows-strings" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] [[package]] name = "windows-sys" @@ -3599,15 +4502,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - [[package]] name = "wit-bindgen" version = "0.57.1" @@ -3615,90 +4509,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap", - "prettyplease", - "syn", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" +name = "writeable" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] -name = "wit-parser" -version = "0.244.0" +name = "wyz" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", + "tap", ] -[[package]] -name = "writeable" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" - [[package]] name = "x509-parser" version = "0.18.1" @@ -3734,15 +4558,15 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" dependencies = [ - "bit-vec", + "bit-vec 0.9.1", "time", ] [[package]] name = "yoke" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -3757,28 +4581,28 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3798,28 +4622,28 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" dependencies = [ "zeroize_derive", ] [[package]] name = "zeroize_derive" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3852,11 +4676,11 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index 76f909b6..7b3faece 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "crates/storage", "crates/config", "crates/storage-postgres", + "crates/storage-mongodb", "crates/auth", "crates/server", "crates/app", @@ -29,6 +30,7 @@ extenddb-engine = { path = "crates/engine" } extenddb-storage = { path = "crates/storage" } extenddb-config = { path = "crates/config" } extenddb-storage-postgres = { path = "crates/storage-postgres" } +extenddb-storage-mongodb = { path = "crates/storage-mongodb" } extenddb-auth = { path = "crates/auth" } extenddb-server = { path = "crates/server" } extenddb-app = { path = "crates/app" } @@ -60,6 +62,9 @@ moka = { version = "0.12", features = ["future"] } # Database sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres", "json", "time", "uuid", "bigdecimal"] } +mongodb = "3" +bson = "2.13" +dashmap = "6" # Crypto & checksums crc32fast = "1" diff --git a/crates/bin/Cargo.toml b/crates/bin/Cargo.toml index d5b148ab..c44a6217 100755 --- a/crates/bin/Cargo.toml +++ b/crates/bin/Cargo.toml @@ -11,8 +11,14 @@ license.workspace = true name = "extenddb" path = "src/main.rs" +[features] +default = ["postgres"] +postgres = ["dep:extenddb-storage-postgres"] +mongodb = ["dep:extenddb-storage-mongodb"] + [dependencies] extenddb-app = { workspace = true } extenddb-storage = { workspace = true } -extenddb-storage-postgres = { workspace = true } +extenddb-storage-postgres = { workspace = true, optional = true } +extenddb-storage-mongodb = { workspace = true, optional = true } anyhow = { workspace = true } diff --git a/crates/bin/src/main.rs b/crates/bin/src/main.rs index 5d71c9e7..3d6431a3 100755 --- a/crates/bin/src/main.rs +++ b/crates/bin/src/main.rs @@ -1,18 +1,26 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! extenddb — the PostgreSQL-backed ExtendDB server binary. +//! extenddb — the ExtendDB server binary. //! //! This is the reference thin bin for the per-backend packaging model: it //! installs exactly one backend and hands off to the shared `extenddb-app` CLI. //! A third-party backend author copies this file, swaps the `backend()` call for //! their crate, and ships their own `extenddb-` image — with no edits to //! any ExtendDB core crate. +//! +//! This fork's bin compiles the PostgreSQL backend by default and the MongoDB +//! backend under `--features mongodb`, selecting the one to install at compile +//! time so a single bin serves both while the reviewer's per-backend model is +//! adopted. fn main() -> anyhow::Result<()> { // Install the compiled-in backend before dispatch. The compiler checks this // call; there is no link-time auto-registration and no name to resolve, so a // missing or mistyped backend cannot become a runtime error. + #[cfg(feature = "mongodb")] + extenddb_storage::set_backend(extenddb_storage_mongodb::backend())?; + #[cfg(not(feature = "mongodb"))] extenddb_storage::set_backend(extenddb_storage_postgres::backend())?; extenddb_app::run(extenddb_app::BuildInfo { diff --git a/crates/engine/src/create_table.rs b/crates/engine/src/create_table.rs index b6f18f7b..c9d00179 100755 --- a/crates/engine/src/create_table.rs +++ b/crates/engine/src/create_table.rs @@ -132,6 +132,13 @@ pub(crate) fn storage_err_to_dynamo(e: extenddb_storage::error::StorageError) -> tracing::error!("Unexpected idempotency error in generic error handler"); DynamoDbError::InternalServerError("Internal server error".to_owned()) } + StorageError::TransactionConflict(msg) => { + // Single-item write raced an in-flight TransactWriteItems on + // the same item and the backend couldn't serialize them + // through internal retries. RFC-0003 §4.3 requires + // TransactionConflictException here — never InternalServerError. + DynamoDbError::TransactionConflictException(msg) + } StorageError::Internal(msg) => { // Log the raw message for debugging but do not expose storage // backend details (e.g. PostgreSQL error text) to the client. diff --git a/crates/storage-mongodb/Cargo.toml b/crates/storage-mongodb/Cargo.toml new file mode 100644 index 00000000..54f6d56e --- /dev/null +++ b/crates/storage-mongodb/Cargo.toml @@ -0,0 +1,58 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "extenddb-storage-mongodb" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +# Internal crates +extenddb-core.workspace = true +extenddb-storage.workspace = true +extenddb-auth.workspace = true + +# Serialization +serde.workspace = true +serde_json.workspace = true +toml.workspace = true +bson.workspace = true + +# Error handling +thiserror.workspace = true +anyhow.workspace = true + +# Async +tokio.workspace = true +async-trait.workspace = true +futures.workspace = true + +# MongoDB driver +mongodb.workspace = true + +# Crypto & checksums +uuid.workspace = true +base64.workspace = true +bcrypt.workspace = true +aes-gcm.workspace = true +rand.workspace = true +zeroize.workspace = true + +# Time +time.workspace = true + +# Logging +tracing.workspace = true + +# Checksums +crc32fast.workspace = true + +# In-process GSI existence cache +dashmap.workspace = true + +[dev-dependencies] +# Property-based testing for the filter-pushdown parity harness +# (crates/storage-mongodb/tests/pushdown_parity.rs). +proptest = "1" diff --git a/crates/storage-mongodb/src/admin_store.rs b/crates/storage-mongodb/src/admin_store.rs new file mode 100644 index 00000000..aa24f8fd --- /dev/null +++ b/crates/storage-mongodb/src/admin_store.rs @@ -0,0 +1,4 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Admin store stub module (implemented in `management_store.rs`). diff --git a/crates/storage-mongodb/src/authorization_store.rs b/crates/storage-mongodb/src/authorization_store.rs new file mode 100644 index 00000000..6f712808 --- /dev/null +++ b/crates/storage-mongodb/src/authorization_store.rs @@ -0,0 +1,360 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `AuthorizationStore` trait implementation for `MongoDB`. + +use futures::TryStreamExt; +use futures::future::BoxFuture; +use mongodb::bson::{self, Document, doc}; + +use extenddb_storage::authorization_store::{AuthorizationStore, SessionData}; +use extenddb_storage::management_store::{OpError, OpResult}; + +use crate::catalog_store::MongoCatalogStore; + +impl AuthorizationStore for MongoCatalogStore { + fn fetch_user_policies( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_policies"); + let cursor = coll + .find(doc! { + "account_id": &account_id, + "principal_type": "user", + "principal_name": &user_name, + }) + .await + .map_err(|e| { + tracing::error!("fetch_user_policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_user_policies cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + let bson_val = d.get("policy_document")?; + let json_val: serde_json::Value = bson::from_bson(bson_val.clone()).ok()?; + Some(json_val.to_string()) + }) + .collect()) + }) + } + + fn fetch_user_group_policies( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + // First get the groups this user belongs to + let members_coll = self + .catalog_db() + .collection::("iam_group_members"); + let members_cursor = members_coll + .find(doc! { "account_id": &account_id, "user_name": &user_name }) + .await + .map_err(|e| { + tracing::error!("fetch_user_group_policies members: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let member_docs: Vec = members_cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_user_group_policies members cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let group_names: Vec<&str> = member_docs + .iter() + .filter_map(|d| d.get_str("group_name").ok()) + .collect(); + + if group_names.is_empty() { + return Ok(Vec::new()); + } + + // Now get all policies for those groups + let policies_coll = self.catalog_db().collection::("iam_policies"); + let cursor = policies_coll + .find(doc! { + "account_id": &account_id, + "principal_type": "group", + "principal_name": { "$in": &group_names }, + }) + .await + .map_err(|e| { + tracing::error!("fetch_user_group_policies policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_user_group_policies policies cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + let bson_val = d.get("policy_document")?; + let json_val: serde_json::Value = bson::from_bson(bson_val.clone()).ok()?; + Some(json_val.to_string()) + }) + .collect()) + }) + } + + fn fetch_user_boundary( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_permissions_boundaries"); + let doc = coll + .find_one(doc! { + "account_id": &account_id, + "principal_type": "user", + "principal_name": &user_name, + }) + .await + .map_err(|e| { + tracing::error!("fetch_user_boundary: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(doc.and_then(|d| { + let bson_val = d.get("policy_document")?; + let json_val: serde_json::Value = bson::from_bson(bson_val.clone()).ok()?; + Some(json_val.to_string()) + })) + }) + } + + fn fetch_role_policies( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_policies"); + let cursor = coll + .find(doc! { + "account_id": &account_id, + "principal_type": "role", + "principal_name": &role_name, + }) + .await + .map_err(|e| { + tracing::error!("fetch_role_policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_role_policies cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + let bson_val = d.get("policy_document")?; + let json_val: serde_json::Value = bson::from_bson(bson_val.clone()).ok()?; + Some(json_val.to_string()) + }) + .collect()) + }) + } + + fn fetch_role_boundary( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_permissions_boundaries"); + let doc = coll + .find_one(doc! { + "account_id": &account_id, + "principal_type": "role", + "principal_name": &role_name, + }) + .await + .map_err(|e| { + tracing::error!("fetch_role_boundary: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(doc.and_then(|d| { + let bson_val = d.get("policy_document")?; + let json_val: serde_json::Value = bson::from_bson(bson_val.clone()).ok()?; + Some(json_val.to_string()) + })) + }) + } + + fn fetch_session_data( + &self, + account_id: &str, + role_name: &str, + session_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let session_name = session_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_sessions"); + let now_bson = mongodb::bson::DateTime::now(); + let doc = coll + .find_one(doc! { + "account_id": &account_id, + "role_name": &role_name, + "session_name": &session_name, + "expires_at": { "$gt": now_bson }, + }) + .await + .map_err(|e| { + tracing::error!("fetch_session_data: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let Some(session_doc) = doc else { + return Ok(None); + }; + + let session_policy = session_doc.get("session_policy").and_then(|b| { + let json_val: serde_json::Value = bson::from_bson(b.clone()).ok()?; + Some(json_val.to_string()) + }); + + let mut session_tags = Vec::new(); + if let Some(tags_bson) = session_doc.get("session_tags") + && let Ok(tags_val) = bson::from_bson::(tags_bson.clone()) + { + if let Some(arr) = tags_val.as_array() { + for tag in arr { + if let (Some(k), Some(v)) = ( + tag.get("Key").and_then(|k| k.as_str()), + tag.get("Value").and_then(|v| v.as_str()), + ) { + session_tags.push((k.to_owned(), v.to_owned())); + } + } + } else if let Some(obj) = tags_val.as_object() { + for (k, v) in obj { + if let Some(v_str) = v.as_str() { + session_tags.push((k.clone(), v_str.to_owned())); + } + } + } + } + + Ok(Some(SessionData { + session_policy, + session_tags, + })) + }) + } + + fn fetch_user_tags( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_user_tags"); + let cursor = coll + .find(doc! { "account_id": &account_id, "user_name": &user_name }) + .await + .map_err(|e| { + tracing::error!("fetch_user_tags: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_user_tags cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect()) + }) + } + + fn fetch_role_tags( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_role_tags"); + let cursor = coll + .find(doc! { "account_id": &account_id, "role_name": &role_name }) + .await + .map_err(|e| { + tracing::error!("fetch_role_tags: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_role_tags cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect()) + }) + } + + fn fetch_resource_tags(&self, arn: &str) -> BoxFuture<'_, OpResult>> { + let arn = arn.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("tags"); + let cursor = coll + .find(doc! { "resource_arn": &arn }) + .await + .map_err(|e| { + tracing::error!("fetch_resource_tags: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_resource_tags cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect()) + }) + } +} diff --git a/crates/storage-mongodb/src/backup_engine.rs b/crates/storage-mongodb/src/backup_engine.rs new file mode 100644 index 00000000..3c200b46 --- /dev/null +++ b/crates/storage-mongodb/src/backup_engine.rs @@ -0,0 +1,650 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `BackupEngine` implementation for `MongoDB`. +//! +//! Backups are stored as one MongoDB collection per backup, plus a `backups` +//! metadata collection in the catalog. `CreateBackup` uses MongoDB's +//! server-side aggregation `$out` stage to clone the source data collection +//! into `_backup_` in the data database — no per-item traffic +//! between the driver and the server. `RestoreTableFromBackup` uses the same +//! stage in reverse. `DeleteBackup` drops the collection. +//! +//! Backup metadata carries a `backup_id` UUID; the collection name is derived +//! from that id so the `backup_arn` (which contains slashes and colons) never +//! appears in a collection name. + +use futures::TryStreamExt; +use futures::future::BoxFuture; +use mongodb::bson::{Document, doc}; + +use extenddb_core::types::{ + BackupDescription, BackupDetails, BackupSummary, ContinuousBackupsDescription, + KeySchemaElement, PointInTimeRecoveryDescription, SourceTableDetails, TableDescription, +}; +use extenddb_storage::BackupEngine; +use extenddb_storage::TableEngine; +use extenddb_storage::error::StorageError; + +use crate::MongoEngine; +use crate::data::data_collection_name; + +fn epoch_millis() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() +} + +/// Return the MongoDB collection name that holds items for a given backup. +/// +/// The collection lives in the data database. The name is derived from the +/// backup's UUID so it is safe for MongoDB (no colons, slashes, or dots) and +/// bounded in length regardless of how long the source `backup_arn` is. +fn backup_collection_name(backup_id: &str) -> String { + format!("_backup_{backup_id}") +} + +#[allow(clippy::cast_precision_loss)] +fn now_epoch_secs() -> f64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as f64 +} + +impl BackupEngine for MongoEngine { + fn create_backup( + &self, + account_id: &str, + table_name: &str, + backup_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + let backup_name = backup_name.to_string(); + Box::pin(async move { + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { + "_id": { "account_id": &account_id, "table_name": &table_name }, + "table_status": "ACTIVE", + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(table_name.clone()))?; + + let table_id = table_doc + .get_str("table_id") + .map_err(|_| StorageError::Internal("missing table_id".to_string()))? + .to_owned(); + let table_arn = table_doc + .get_str("table_arn") + .unwrap_or_default() + .to_owned(); + let key_schema_bson = table_doc + .get_array("key_schema") + .map_err(|_| StorageError::Internal("missing key_schema".to_string()))? + .clone(); + let attr_defs_bson = table_doc + .get_array("attribute_definitions") + .map_err(|_| StorageError::Internal("missing attribute_definitions".to_string()))? + .clone(); + let billing_mode = table_doc + .get_str("billing_mode") + .unwrap_or("PAY_PER_REQUEST") + .to_owned(); + let table_size = table_doc.get_i64("table_size_bytes").unwrap_or(0); + let _item_count = table_doc.get_i64("item_count").unwrap_or(0); + + // Preserve TableClass / SSESpecification / OnDemandThroughput so + // RestoreTableFromBackup can recreate the table with the same + // configuration. + let table_class_bson = table_doc + .get("table_class") + .cloned() + .unwrap_or(mongodb::bson::Bson::Null); + let sse_spec_bson = table_doc + .get("sse_specification") + .cloned() + .unwrap_or(mongodb::bson::Bson::Null); + let on_demand_bson = table_doc + .get("on_demand_throughput") + .cloned() + .unwrap_or(mongodb::bson::Bson::Null); + + // The trailing backup-id component is a timestamp plus an 8-hex-char + // random suffix, so a backup ARN (which is a capability) is not + // guessable from the creation time alone. Matches the postgres + // backend. + let arn_suffix: u32 = { + use rand::Rng; + rand::rng().random() + }; + let backup_arn = format!( + "arn:aws:dynamodb:{region}:{account_id}:table/{table_name}/backup/{ts}-{arn_suffix:08x}", + region = self.region, + ts = epoch_millis() + ); + let backup_id = uuid::Uuid::new_v4().to_string(); + + // Snapshot items from the data collection using a server-side + // `$out` aggregation. Items are copied directly between + // collections in MongoDB — no per-item traffic to the driver. + let src_coll_name = data_collection_name(&table_id); + let dst_coll_name = backup_collection_name(&backup_id); + let data_coll = self.data_db.collection::(&src_coll_name); + + let pipeline = vec![doc! { "$out": &dst_coll_name }]; + let out_cursor = data_coll + .aggregate(pipeline) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + // `$out` writes to the target collection and returns an empty + // cursor; consume it to ensure the stage has fully completed + // before we count. + let _drained: Vec = out_cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let dst_coll = self.data_db.collection::(&dst_coll_name); + let actual_count = dst_coll + .count_documents(doc! {}) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + as i64; + + let created_at = now_epoch_secs(); + + // Store backup metadata. `backup_id` is what maps to the + // physical collection; `backup_arn` remains the caller-visible + // handle and stays the `_id` for compatibility with existing + // describe/list callers. + let backups_coll = self.catalog_db.collection::("backups"); + let backup_meta = doc! { + "_id": &backup_arn, + "backup_id": &backup_id, + "backup_name": &backup_name, + "backup_status": "AVAILABLE", + "backup_type": "USER", + "table_id": &table_id, + "table_name": &table_name, + "table_arn": &table_arn, + "account_id": &account_id, + "backup_size_bytes": table_size, + "item_count": actual_count, + "key_schema": key_schema_bson, + "attribute_definitions": attr_defs_bson, + "billing_mode": &billing_mode, + "created_at": mongodb::bson::DateTime::now(), + "table_creation_date_time": created_at, + "table_class": table_class_bson, + "sse_specification": sse_spec_bson, + "on_demand_throughput": on_demand_bson, + }; + + backups_coll + .insert_one(backup_meta) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(BackupDetails { + backup_arn, + backup_name, + backup_status: "AVAILABLE".to_owned(), + backup_type: "USER".to_owned(), + backup_size_bytes: table_size, + backup_creation_date_time: created_at, + }) + }) + } + + fn describe_backup( + &self, + account_id: &str, + backup_arn: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let backup_arn = backup_arn.to_string(); + Box::pin(async move { + let backups_coll = self.catalog_db.collection::("backups"); + // Scope the lookup to the calling account so a backup ARN cannot be + // read cross-account, and exclude DELETED backups so a deleted + // backup reads as BackupNotFoundException. Matches the postgres + // backend. + let backup_doc = backups_coll + .find_one(doc! { + "_id": &backup_arn, + "account_id": &account_id, + "backup_status": { "$ne": "DELETED" }, + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + StorageError::Validation(format!("Backup not found: {backup_arn}")) + })?; + + let name = backup_doc + .get_str("backup_name") + .unwrap_or_default() + .to_owned(); + let status = backup_doc + .get_str("backup_status") + .unwrap_or("AVAILABLE") + .to_owned(); + let table_id = backup_doc + .get_str("table_id") + .unwrap_or_default() + .to_owned(); + let table_name = backup_doc + .get_str("table_name") + .unwrap_or_default() + .to_owned(); + let table_arn = backup_doc + .get_str("table_arn") + .unwrap_or_default() + .to_owned(); + let size = backup_doc.get_i64("backup_size_bytes").unwrap_or(0); + let count = backup_doc.get_i64("item_count").unwrap_or(0); + let billing = backup_doc + .get_str("billing_mode") + .unwrap_or("PAY_PER_REQUEST") + .to_owned(); + + let created_at = backup_doc + .get_datetime("created_at") + .map(|dt| dt.timestamp_millis() as f64 / 1000.0) + .unwrap_or(0.0); + let table_created = backup_doc + .get_f64("table_creation_date_time") + .unwrap_or(created_at); + + let key_schema_bson = backup_doc + .get_array("key_schema") + .map_err(|_| StorageError::Internal("missing key_schema in backup".to_string()))?; + let key_schema_json = serde_json::to_value(key_schema_bson) + .map_err(|e| StorageError::Internal(format!("serialize key_schema: {e}")))?; + let key_schema: Vec = serde_json::from_value(key_schema_json) + .map_err(|e| StorageError::Internal(format!("parse key_schema: {e}")))?; + + Ok(BackupDescription { + backup_details: BackupDetails { + backup_arn: backup_arn.clone(), + backup_name: name, + backup_status: status, + backup_type: "USER".to_owned(), + backup_size_bytes: size, + backup_creation_date_time: created_at, + }, + source_table_details: SourceTableDetails { + table_name, + table_id, + table_arn, + key_schema, + item_count: count, + table_size_bytes: size, + billing_mode: Some(billing), + table_creation_date_time: table_created, + }, + }) + }) + } + + fn list_backups( + &self, + account_id: &str, + table_name: Option<&str>, + ) -> BoxFuture<'_, Result, StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.map(std::string::ToString::to_string); + Box::pin(async move { + let backups_coll = self.catalog_db.collection::("backups"); + + let mut filter = doc! { + "account_id": &account_id, + "backup_status": { "$ne": "DELETED" }, + }; + if let Some(tn) = &table_name { + filter.insert("table_name", tn.as_str()); + } + + let mut cursor = backups_coll + .find(filter) + .sort(doc! { "created_at": -1 }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut results = Vec::new(); + while let Some(doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let arn = doc.get_str("_id").unwrap_or_default().to_owned(); + let name = doc.get_str("backup_name").unwrap_or_default().to_owned(); + let tn = doc.get_str("table_name").unwrap_or_default().to_owned(); + let table_arn = doc.get_str("table_arn").unwrap_or_default().to_owned(); + let status = doc + .get_str("backup_status") + .unwrap_or("AVAILABLE") + .to_owned(); + let size = doc.get_i64("backup_size_bytes").unwrap_or(0); + let created_at = doc + .get_datetime("created_at") + .map(|dt| dt.timestamp_millis() as f64 / 1000.0) + .unwrap_or(0.0); + + results.push(BackupSummary { + backup_arn: arn, + backup_name: name, + table_name: tn, + table_arn, + backup_status: status, + backup_type: "USER".to_owned(), + backup_size_bytes: size, + backup_creation_date_time: created_at, + }); + } + Ok(results) + }) + } + + fn delete_backup( + &self, + account_id: &str, + backup_arn: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let backup_arn = backup_arn.to_string(); + Box::pin(async move { + let desc = self.describe_backup(&account_id, &backup_arn).await?; + + // Look up the physical collection name from metadata (account-scoped). + let backups_coll = self.catalog_db.collection::("backups"); + let meta = backups_coll + .find_one(doc! { "_id": &backup_arn, "account_id": &account_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + StorageError::Validation(format!("Backup not found: {backup_arn}")) + })?; + + // Drop the backup collection. If backup_id is absent (e.g., a + // pre-`$out` backup on an old catalog) we skip — nothing to drop + // at the collection level in that case. + if let Ok(backup_id) = meta.get_str("backup_id") { + let coll_name = backup_collection_name(backup_id); + let coll = self.data_db.collection::(&coll_name); + coll.drop() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + // Mark backup as deleted (account-scoped) + backups_coll + .update_one( + doc! { "_id": &backup_arn, "account_id": &account_id }, + doc! { "$set": { "backup_status": "DELETED" } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(BackupDescription { + backup_details: BackupDetails { + backup_status: "DELETED".to_owned(), + ..desc.backup_details + }, + source_table_details: desc.source_table_details, + }) + }) + } + + fn restore_table_from_backup( + &self, + account_id: &str, + target_table_name: &str, + backup_arn: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let target_table_name = target_table_name.to_string(); + let backup_arn = backup_arn.to_string(); + Box::pin(async move { + let backups_coll = self.catalog_db.collection::("backups"); + let backup_doc = backups_coll + .find_one(doc! { "_id": &backup_arn, "backup_status": "AVAILABLE" }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + StorageError::Validation(format!("Backup not found: {backup_arn}")) + })?; + + let key_schema_bson = backup_doc + .get_array("key_schema") + .map_err(|_| StorageError::Internal("missing key_schema".to_string()))?; + let attr_defs_bson = backup_doc + .get_array("attribute_definitions") + .map_err(|_| StorageError::Internal("missing attribute_definitions".to_string()))?; + let billing = backup_doc + .get_str("billing_mode") + .unwrap_or("PAY_PER_REQUEST"); + + let ks_json = serde_json::to_value(key_schema_bson) + .map_err(|e| StorageError::Internal(format!("serialize key_schema: {e}")))?; + let ad_json = serde_json::to_value(attr_defs_bson) + .map_err(|e| StorageError::Internal(format!("serialize attr_defs: {e}")))?; + + let key_schema: Vec = + serde_json::from_value(ks_json) + .map_err(|e| StorageError::Internal(format!("parse key_schema: {e}")))?; + let attr_defs: Vec = + serde_json::from_value(ad_json) + .map_err(|e| StorageError::Internal(format!("parse attr_defs: {e}")))?; + + let billing_mode = if billing == "PAY_PER_REQUEST" { + Some(extenddb_core::types::BillingMode::PayPerRequest) + } else { + Some(extenddb_core::types::BillingMode::Provisioned) + }; + + // Preserve the source table's TableClass / SSESpecification / + // OnDemandThroughput settings when recreating. + let table_class = backup_doc.get_str("table_class").ok().map(str::to_owned); + let sse_specification: Option = + backup_doc.get("sse_specification").and_then(|b| { + if matches!(b, mongodb::bson::Bson::Null) { + None + } else { + bson::from_bson(b.clone()).ok() + } + }); + let on_demand_throughput: Option = + backup_doc.get("on_demand_throughput").and_then(|b| { + if matches!(b, mongodb::bson::Bson::Null) { + None + } else { + bson::from_bson(b.clone()).ok() + } + }); + + let create_input = extenddb_core::types::CreateTableInput { + table_name: target_table_name.clone(), + key_schema, + attribute_definitions: attr_defs, + billing_mode, + provisioned_throughput: Some(extenddb_core::types::ProvisionedThroughput { + read_capacity_units: 5, + write_capacity_units: 5, + }), + global_secondary_indexes: None, + local_secondary_indexes: None, + stream_specification: None, + tags: None, + deletion_protection_enabled: None, + sse_specification, + table_class, + on_demand_throughput, + }; + + let desc = self.create_table(&account_id, create_input).await?; + + // Restore items from the backup collection using server-side `$out`. + // The backup collection was written by `create_backup` in the same + // document shape as the source data collection, so this is a + // direct clone — no per-item transformation needed. + let backup_id = backup_doc + .get_str("backup_id") + .map_err(|_| { + StorageError::Internal("backup metadata missing backup_id".to_string()) + })? + .to_owned(); + let src_coll_name = backup_collection_name(&backup_id); + let src_coll = self.data_db.collection::(&src_coll_name); + let new_coll_name = data_collection_name(&desc.table_id); + + let pipeline = vec![doc! { "$out": &new_coll_name }]; + let out_cursor = src_coll + .aggregate(pipeline) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let _drained: Vec = out_cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let new_data_coll = self.data_db.collection::(&new_coll_name); + let item_count = new_data_coll + .count_documents(doc! {}) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + as i64; + + // Update the item count. The table was created via `create_table`, + // so it is already in CREATING (with a scheduled transition) when + // control_plane_delay_seconds > 0, or ACTIVE when it is 0; the + // control_plane_worker flips CREATING -> ACTIVE once the window + // passes. The data was just copied above, so it is in place before + // the table becomes ACTIVE. `desc` (returned to the caller) already + // carries the CREATING status from create_table, matching DynamoDB. + let tables_coll = self.catalog_db.collection::("tables"); + tables_coll + .update_one( + doc! { "_id": { "account_id": &account_id, "table_name": &target_table_name } }, + doc! { "$set": { "item_count": item_count } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(desc) + }) + } + + fn describe_continuous_backups( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { + let tables_coll = self.catalog_db.collection::("tables"); + let exists = tables_coll + .find_one(doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if exists.is_none() { + return Err(StorageError::TableNotFound(table_name)); + } + + let cb_coll = self.catalog_db.collection::("continuous_backups"); + let pitr_doc = cb_coll + .find_one(doc! { "account_id": &account_id, "table_name": &table_name }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let pitr_enabled = pitr_doc + .as_ref() + .and_then(|d| d.get_bool("pitr_enabled").ok()) + .unwrap_or(false); + + let now_epoch = now_epoch_secs(); + + Ok(ContinuousBackupsDescription { + continuous_backups_status: "ENABLED".to_owned(), + point_in_time_recovery_description: Some(PointInTimeRecoveryDescription { + point_in_time_recovery_status: if pitr_enabled { + "ENABLED".to_owned() + } else { + "DISABLED".to_owned() + }, + earliest_restorable_date_time: if pitr_enabled { + Some(now_epoch - 35.0 * 24.0 * 3600.0) + } else { + None + }, + latest_restorable_date_time: if pitr_enabled { Some(now_epoch) } else { None }, + }), + }) + }) + } + + fn update_continuous_backups( + &self, + account_id: &str, + table_name: &str, + pitr_enabled: bool, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { + let tables_coll = self.catalog_db.collection::("tables"); + let exists = tables_coll + .find_one(doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if exists.is_none() { + return Err(StorageError::TableNotFound(table_name.clone())); + } + + let cb_coll = self.catalog_db.collection::("continuous_backups"); + cb_coll + .update_one( + doc! { "account_id": &account_id, "table_name": &table_name }, + doc! { "$set": { + "account_id": &account_id, + "table_name": &table_name, + "pitr_enabled": pitr_enabled, + }}, + ) + .upsert(true) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + self.describe_continuous_backups(&account_id, &table_name) + .await + }) + } + + fn restore_table_to_point_in_time( + &self, + account_id: &str, + source_table_name: &str, + target_table_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let source_table_name = source_table_name.to_string(); + let target_table_name = target_table_name.to_string(); + Box::pin(async move { + let backup = self + .create_backup(&account_id, &source_table_name, "__pitr_restore__") + .await?; + let desc = self + .restore_table_from_backup(&account_id, &target_table_name, &backup.backup_arn) + .await?; + let _ = self.delete_backup(&account_id, &backup.backup_arn).await; + Ok(desc) + }) + } +} diff --git a/crates/storage-mongodb/src/bootstrapper.rs b/crates/storage-mongodb/src/bootstrapper.rs new file mode 100644 index 00000000..21196e3f --- /dev/null +++ b/crates/storage-mongodb/src/bootstrapper.rs @@ -0,0 +1,705 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Bootstrapper implementation for `MongoDB`. + +use async_trait::async_trait; +use bson::{Document, doc}; +use mongodb::IndexModel; +use mongodb::options::IndexOptions; + +use extenddb_storage::bootstrapper::{AdminBootstrapResult, Bootstrapper}; +use extenddb_storage::error::StorageError; +use extenddb_storage::management_store::{OpError, OpResult}; + +/// `MongoDB` bootstrapper for init/destroy/migrate operations. +pub struct MongoBootstrapper { + client: mongodb::Client, + connection_string: String, +} + +impl MongoBootstrapper { + pub async fn from_config( + config_path: &str, + _cli_args: &[String], + ) -> Result { + // Read the config file to get the connection string + let config_content = std::fs::read_to_string(config_path).map_err(|e| { + StorageError::Internal(format!("Cannot read config file '{config_path}': {e}")) + })?; + + let config: toml::Value = config_content + .parse() + .map_err(|e| StorageError::Internal(format!("Cannot parse config: {e}")))?; + + let connection_string = config + .get("storage") + .and_then(|s| s.get("mongodb")) + .and_then(|m| m.get("connection_string")) + .and_then(|v| v.as_str()) + .unwrap_or("mongodb://localhost:27017") + .to_string(); + + let client = mongodb::Client::with_uri_str(&connection_string) + .await + .map_err(|e| StorageError::Connection(e.to_string()))?; + + Ok(Self { + client, + connection_string, + }) + } + + fn catalog_db(&self) -> mongodb::Database { + self.client.database("extenddb_catalog") + } + + fn data_db(&self) -> mongodb::Database { + self.client.database("extenddb_data") + } +} + +#[async_trait] +impl Bootstrapper for MongoBootstrapper { + async fn ensure_app_user(&self) -> OpResult<()> { + // MongoDB uses connection-level auth; no separate app user needed + Ok(()) + } + + async fn grant_app_role_to_admin(&self) -> OpResult<()> { + // Not applicable for MongoDB + Ok(()) + } + + async fn create_catalog_db(&self) -> OpResult<()> { + // MongoDB creates databases implicitly on first write. + // We'll create a sentinel collection to materialize the database. + let db = self.catalog_db(); + db.create_collection("schema_history") + .await + .map_err(|e| OpError::Internal(format!("Failed to create catalog db: {e}")))?; + Ok(()) + } + + async fn create_data_db(&self) -> OpResult<()> { + // MongoDB creates databases implicitly on first write. + let db = self.data_db(); + db.create_collection("idempotency_tokens") + .await + .map_err(|e| OpError::Internal(format!("Failed to create data db: {e}")))?; + + let coll = db.collection::("idempotency_tokens"); + + // DDB spec: `ClientRequestToken` dedups retries within a 10-minute + // window. MongoDB's TTL monitor runs on a ~60s cadence, so an + // `expireAfterSeconds = 600` index deletes rows anywhere from + // 10:00 to ~11:00 minutes after `created_at` — retention drifts + // above the spec. Shrink to 540s (9 min) so worst-case retention + // is ≤10 min. The data-plane read path also filters on `created_at` + // to enforce the boundary strictly, independent of TTL monitor + // cadence (see `transact_write_items_impl`). + let ttl_index = IndexModel::builder() + .keys(doc! { "created_at": 1 }) + .options( + IndexOptions::builder() + .expire_after(std::time::Duration::from_secs(540)) + .build(), + ) + .build(); + coll.create_index(ttl_index) + .await + .map_err(|e| OpError::Internal(format!("Failed to create TTL index: {e}")))?; + + // Unique compound index on (account_id, token). Without this, + // two concurrent TransactWriteItems calls with the same token + // both do a snapshot read that misses the other's uncommitted + // insert, and both commit — the operation executes twice. + // With the unique index in place, the second inserter fails + // E11000 and the write path converts that into a retryable + // error, giving the client the read-check path on retry. + coll.create_index( + IndexModel::builder() + .keys(doc! { "account_id": 1, "token": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("idempotency_tokens unique index: {e}")))?; + + // stream_shards: unique index on shard_id so a subsequent init/ + // recreate can never insert a duplicate shard document under the + // same shard_id. Combined with `table_id`-derived shard_ids + // (see stream_engine::build_shard_id) this rules out cross-tenant + // shard collisions structurally. + db.create_collection("stream_shards") + .await + .map_err(|e| OpError::Internal(format!("Failed to create stream_shards: {e}")))?; + db.collection::("stream_shards") + .create_index( + IndexModel::builder() + .keys(doc! { "shard_id": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("stream_shards shard_id index: {e}")))?; + + // stream_records: TTL index enforcing DDB's 24-hour retention. + // The `expireAfterSeconds` is the delta from the field value, + // not a hard cutoff, so a MongoDB background thread will + // delete records ~1 minute after created_at + 24h. The + // TTL cleanup worker in `ttl_worker.rs` provides a defense- + // in-depth deletion path but the primary enforcement is here. + db.create_collection("stream_records") + .await + .map_err(|e| OpError::Internal(format!("Failed to create stream_records: {e}")))?; + let stream_records = db.collection::("stream_records"); + stream_records + .create_index( + IndexModel::builder() + .keys(doc! { "created_at": 1 }) + .options( + IndexOptions::builder() + .expire_after(std::time::Duration::from_secs(24 * 3600)) + .build(), + ) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("stream_records TTL index: {e}")))?; + + // Query-side index on (shard_id, sequence_number). GetRecords + // filters by shard_id and paginates by sequence_number > cursor + // with an ascending sort — the only way this can be efficient + // is if the index prefix matches. Without it, GetRecords does a + // full collection scan every time consumer polls, and cost + // grows linearly in the retained record count. D-m6. + stream_records + .create_index( + IndexModel::builder() + .keys(doc! { "shard_id": 1, "sequence_number": 1 }) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("stream_records shard index: {e}")))?; + + Ok(()) + } + + async fn run_catalog_migrations(&self) -> OpResult<()> { + let db = self.catalog_db(); + + // Create all catalog collections + let collections = [ + "accounts", + "tables", + "indexes", + "tags", + "settings", + "admin_users", + "iam_users", + "access_keys", + "iam_groups", + "iam_roles", + "iam_sessions", + "iam_policies", + "iam_permissions_boundaries", + "metrics", + "login_attempts", + "backups", + "continuous_backups", + ]; + + for coll_name in collections { + // create_collection is idempotent in recent MongoDB versions + let _ = db.create_collection(coll_name).await; + } + + // Create indexes for catalog collections + + // accounts: unique index on account_name + let accounts = db.collection::("accounts"); + accounts + .create_index( + IndexModel::builder() + .keys(doc! { "account_name": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("accounts index: {e}")))?; + + // tables: unique index on table_id + let tables = db.collection::("tables"); + tables + .create_index( + IndexModel::builder() + .keys(doc! { "table_id": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("tables table_id index: {e}")))?; + + // iam_users: unique index on user_arn + let iam_users = db.collection::("iam_users"); + iam_users + .create_index( + IndexModel::builder() + .keys(doc! { "user_arn": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("iam_users user_arn index: {e}")))?; + + // access_keys: index on (account_id, user_name) + let access_keys = db.collection::("access_keys"); + access_keys + .create_index( + IndexModel::builder() + .keys(doc! { "account_id": 1, "user_name": 1 }) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("access_keys index: {e}")))?; + + // iam_groups: unique index on group_arn + let iam_groups = db.collection::("iam_groups"); + iam_groups + .create_index( + IndexModel::builder() + .keys(doc! { "group_arn": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("iam_groups index: {e}")))?; + + // iam_roles: unique index on role_arn + let iam_roles = db.collection::("iam_roles"); + iam_roles + .create_index( + IndexModel::builder() + .keys(doc! { "role_arn": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("iam_roles index: {e}")))?; + + // iam_sessions: unique index on access_key_id, TTL on expires_at + let iam_sessions = db.collection::("iam_sessions"); + iam_sessions + .create_index( + IndexModel::builder() + .keys(doc! { "access_key_id": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("iam_sessions access_key index: {e}")))?; + iam_sessions + .create_index( + IndexModel::builder() + .keys(doc! { "expires_at": 1 }) + .options( + IndexOptions::builder() + .expire_after(std::time::Duration::from_secs(0)) + .build(), + ) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("iam_sessions TTL index: {e}")))?; + + // metrics: index on bucket + let metrics = db.collection::("metrics"); + metrics + .create_index(IndexModel::builder().keys(doc! { "_id.bucket": 1 }).build()) + .await + .map_err(|e| OpError::Internal(format!("metrics bucket index: {e}")))?; + + // login_attempts: compound index + let login_attempts = db.collection::("login_attempts"); + login_attempts + .create_index( + IndexModel::builder() + .keys(doc! { "principal": 1, "attempted_at": 1 }) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("login_attempts index: {e}")))?; + + // backups: index on (account_id, table_name) + let backups = db.collection::("backups"); + backups + .create_index( + IndexModel::builder() + .keys(doc! { "account_id": 1, "table_name": 1 }) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("backups index: {e}")))?; + + // Seed catalog_version in settings + let settings = db.collection::("settings"); + let _ = settings + .update_one( + doc! { "_id": "catalog_version" }, + doc! { "$setOnInsert": { "value": "0.0.2" } }, + ) + .upsert(true) + .await; + + // Record migration + let schema_history = db.collection::("schema_history"); + let _ = schema_history + .insert_one(doc! { + "_id": "001_initial", + "applied_at": bson::DateTime::now(), + }) + .await; // Ignore E11000 (already applied) + + Ok(()) + } + + async fn run_data_migrations(&self) -> OpResult<()> { + // Data database schema is minimal for MongoDB (just idempotency_tokens) + // Table collections are created on-demand + Ok(()) + } + + async fn pending_data_migrations(&self) -> OpResult> { + // MongoDB has no versioned data-database migrations — table + // collections are created on-demand, and the idempotency_tokens + // collection is created in create_data_db. Nothing to apply. + Ok(Vec::new()) + } + + async fn record_data_connection(&self) -> OpResult<()> { + let db = self.catalog_db(); + let settings = db.collection::("settings"); + + let data_db_name = "extenddb_data".to_string(); + settings + .update_one( + doc! { "_id": "data_database_name" }, + doc! { "$set": { "value": &data_db_name } }, + ) + .upsert(true) + .await + .map_err(|e| OpError::Internal(format!("record_data_connection: {e}")))?; + + let redacted = redact_connection_string(&self.connection_string); + settings + .update_one( + doc! { "_id": "data_connection_string" }, + doc! { "$set": { "value": &redacted } }, + ) + .upsert(true) + .await + .map_err(|e| OpError::Internal(format!("record data_connection_string: {e}")))?; + + Ok(()) + } + + async fn bootstrap_encryption_key(&self) -> OpResult<()> { + use base64::Engine; + use rand::RngCore; + + let db = self.catalog_db(); + let settings = db.collection::("settings"); + + // Check if already exists + let existing = settings + .find_one(doc! { "_id": "encryption_key" }) + .await + .map_err(|e| OpError::Internal(format!("check encryption key: {e}")))?; + + if existing.is_some() { + return Ok(()); + } + + // Generate a 256-bit encryption key + let mut key_bytes = [0u8; 32]; + rand::rng().fill_bytes(&mut key_bytes); + let key_b64 = base64::engine::general_purpose::STANDARD.encode(key_bytes); + + // Ignore E11000 (race: someone else created it first) + let _ = settings + .insert_one(doc! { + "_id": "encryption_key", + "value": &key_b64, + }) + .await; + + Ok(()) + } + + async fn bootstrap_default_account(&self) -> OpResult<()> { + let db = self.catalog_db(); + let accounts = db.collection::("accounts"); + + // Check if any account exists + let count = accounts + .count_documents(doc! {}) + .await + .map_err(|e| OpError::Internal(format!("count accounts: {e}")))?; + + if count > 0 { + return Ok(()); + } + + let account_id = uuid::Uuid::new_v4().to_string(); + accounts + .insert_one(doc! { + "_id": &account_id, + "account_name": "default", + "created_at": bson::DateTime::now(), + }) + .await + .map_err(|e| OpError::Internal(format!("create default account: {e}")))?; + + Ok(()) + } + + async fn bootstrap_admin_user( + &self, + env_user: Option<&str>, + env_password: Option<&str>, + ) -> OpResult { + let username = env_user.unwrap_or("admin").to_string(); + + let db = self.catalog_db(); + let admin_users = db.collection::("admin_users"); + + // Check if admin already exists + let existing = admin_users + .find_one(doc! { "_id": &username }) + .await + .map_err(|e| OpError::Internal(format!("check admin: {e}")))?; + + if existing.is_some() { + return Ok(AdminBootstrapResult { + username, + generated_password: None, + already_existed: true, + from_env: env_user.is_some(), + }); + } + + // Generate or use provided password + let (password, from_env) = if let Some(pw) = env_password { + (pw.to_string(), true) + } else { + use rand::Rng; + let pw: String = rand::rng() + .sample_iter(&rand::distr::Alphanumeric) + .take(24) + .map(char::from) + .collect(); + (pw, false) + }; + + let password_hash = bcrypt::hash(&password, bcrypt::DEFAULT_COST) + .map_err(|e| OpError::Internal(format!("bcrypt hash: {e}")))?; + + admin_users + .insert_one(doc! { + "_id": &username, + "password_hash": &password_hash, + "created_at": bson::DateTime::now(), + }) + .await + .map_err(|e| OpError::Internal(format!("create admin: {e}")))?; + + Ok(AdminBootstrapResult { + username, + generated_password: if from_env { None } else { Some(password) }, + already_existed: false, + from_env, + }) + } + + async fn is_catalog_initialized(&self) -> OpResult { + let db = self.catalog_db(); + let collections = db + .list_collection_names() + .await + .map_err(|e| OpError::Internal(format!("list collections: {e}")))?; + Ok(collections.contains(&"settings".to_string())) + } + + async fn list_table_names(&self) -> OpResult> { + use futures::TryStreamExt; + + let db = self.catalog_db(); + let tables = db.collection::("tables"); + + let cursor = tables + .find(doc! {}) + .projection(doc! { "_id.table_name": 1 }) + .await + .map_err(|e| OpError::Internal(format!("list tables: {e}")))?; + + let docs: Vec = cursor + .try_collect() + .await + .map_err(|e| OpError::Internal(format!("collect tables: {e}")))?; + + let names: Vec = docs + .iter() + .filter_map(|d| { + d.get_document("_id") + .ok() + .and_then(|id| id.get_str("table_name").ok()) + .map(std::string::ToString::to_string) + }) + .collect(); + + Ok(names) + } + + async fn get_data_db_name(&self) -> OpResult> { + let db = self.catalog_db(); + let settings = db.collection::("settings"); + let doc = settings + .find_one(doc! { "_id": "data_database_name" }) + .await + .map_err(|e| OpError::Internal(format!("get data_db_name: {e}")))?; + Ok(doc.and_then(|d| { + d.get_str("value") + .ok() + .map(std::string::ToString::to_string) + })) + } + + async fn drop_databases(&self, _data_db: &str) -> OpResult<()> { + self.data_db() + .drop() + .await + .map_err(|e| OpError::Internal(format!("drop data db: {e}")))?; + self.catalog_db() + .drop() + .await + .map_err(|e| OpError::Internal(format!("drop catalog db: {e}")))?; + Ok(()) + } + + async fn read_catalog_version(&self) -> OpResult> { + let db = self.catalog_db(); + let settings = db.collection::("settings"); + let doc = settings + .find_one(doc! { "_id": "catalog_version" }) + .await + .map_err(|e| OpError::Internal(format!("read catalog_version: {e}")))?; + Ok(doc.and_then(|d| { + d.get_str("value") + .ok() + .map(std::string::ToString::to_string) + })) + } + + fn expected_catalog_version(&self) -> String { + "0.0.2".to_string() + } + + fn catalog_database_name(&self) -> String { + "extenddb_catalog".to_string() + } + + fn endpoint_info(&self) -> String { + self.connection_string.clone() + } + + fn catalog_connection_url(&self) -> String { + format!("{}/extenddb_catalog", self.connection_string) + } + + fn generate_backend_config_section(&self) -> String { + format!( + r#"[storage.mongodb] +connection_string = "{}" +# max_connections = 50 # Max concurrent connections for data operations (default 50) +# max_catalog_connections = 20 # Max concurrent connections for catalog/management operations (default 20)"#, + self.connection_string + ) + } +} + +/// Replace `user:password` userinfo with `user:` in a MongoDB URI. +/// +/// Returns the original string unchanged if no `@` separator is present or if +/// the userinfo section contains no `:` (username-only, no password). +fn redact_connection_string(uri: &str) -> String { + let Some(scheme_end) = uri.find("://") else { + return uri.to_string(); + }; + let after_scheme = scheme_end + 3; + let Some(at_offset) = uri[after_scheme..].find('@') else { + return uri.to_string(); + }; + let at_idx = after_scheme + at_offset; + let userinfo = &uri[after_scheme..at_idx]; + let Some(colon_offset) = userinfo.find(':') else { + return uri.to_string(); + }; + let user = &userinfo[..colon_offset]; + format!( + "{}{}:{}", + &uri[..after_scheme], + user, + &uri[at_idx..] + ) +} + +#[cfg(test)] +mod tests { + use super::redact_connection_string; + + #[test] + fn redact_userinfo_with_password() { + assert_eq!( + redact_connection_string("mongodb://alice:secret@host:27017/?replicaSet=rs0"), + "mongodb://alice:@host:27017/?replicaSet=rs0" + ); + } + + #[test] + fn redact_srv_scheme() { + assert_eq!( + redact_connection_string("mongodb+srv://alice:p%40ss@cluster.example.com/?tls=true"), + "mongodb+srv://alice:@cluster.example.com/?tls=true" + ); + } + + #[test] + fn no_userinfo_untouched() { + assert_eq!( + redact_connection_string("mongodb://localhost:27017/?replicaSet=rs0"), + "mongodb://localhost:27017/?replicaSet=rs0" + ); + } + + #[test] + fn user_only_untouched() { + assert_eq!( + redact_connection_string("mongodb://alice@host:27017/"), + "mongodb://alice@host:27017/" + ); + } + + #[test] + fn at_in_query_is_not_userinfo() { + assert_eq!( + redact_connection_string("mongodb://host:27017/?authSource=admin"), + "mongodb://host:27017/?authSource=admin" + ); + } + + #[test] + fn no_scheme_untouched() { + assert_eq!(redact_connection_string("not a uri"), "not a uri"); + } +} diff --git a/crates/storage-mongodb/src/catalog_store.rs b/crates/storage-mongodb/src/catalog_store.rs new file mode 100644 index 00000000..e5f8158c --- /dev/null +++ b/crates/storage-mongodb/src/catalog_store.rs @@ -0,0 +1,105 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Catalog store implementation for `MongoDB`. + +use futures::future::BoxFuture; +use mongodb::bson::doc; + +/// `MongoDB` catalog store. +pub struct MongoCatalogStore { + client: mongodb::Client, + catalog_db: mongodb::Database, + pub(crate) encryption_key: Option, +} + +impl MongoCatalogStore { + #[must_use] + pub fn new(client: mongodb::Client) -> Self { + let catalog_db = client.database("extenddb_catalog"); + Self { + client, + catalog_db, + encryption_key: None, + } + } + + #[must_use] + pub fn with_encryption_key(client: mongodb::Client, encryption_key: String) -> Self { + let catalog_db = client.database("extenddb_catalog"); + Self { + client, + catalog_db, + encryption_key: Some(encryption_key), + } + } + + /// Get a reference to the catalog database. + pub(crate) fn catalog_db(&self) -> &mongodb::Database { + &self.catalog_db + } +} + +// Implement CatalogStore supertrait +impl extenddb_storage::CatalogStore for MongoCatalogStore { + fn cached_encryption_key(&self) -> Option { + self.encryption_key.clone() + } +} + +// Implement DiagnosticsStore +impl extenddb_storage::diagnostics::DiagnosticsStore for MongoCatalogStore { + fn count_tables(&self) -> BoxFuture<'_, extenddb_storage::diagnostics::DiagResult> { + Box::pin(async { + let coll = self + .catalog_db + .collection::("tables"); + let count = coll.count_documents(doc! {}).await.map_err(|e| { + extenddb_storage::diagnostics::DiagError::QueryFailed(e.to_string()) + })?; + Ok(count as i64) + }) + } + + fn count_indexes(&self) -> BoxFuture<'_, extenddb_storage::diagnostics::DiagResult> { + Box::pin(async { + use futures::TryStreamExt; + + // Count tables that have GSIs or LSIs defined + let coll = self + .catalog_db + .collection::("tables"); + let mut cursor = coll.find(doc! {}).await.map_err(|e| { + extenddb_storage::diagnostics::DiagError::QueryFailed(e.to_string()) + })?; + + let mut index_count: i64 = 0; + while let Some(table_doc) = cursor + .try_next() + .await + .map_err(|e| extenddb_storage::diagnostics::DiagError::QueryFailed(e.to_string()))? + { + if let Ok(gsis) = table_doc.get_array("global_secondary_indexes") { + index_count += gsis.len() as i64; + } + if let Ok(lsis) = table_doc.get_array("local_secondary_indexes") { + index_count += lsis.len() as i64; + } + } + Ok(index_count) + }) + } + + fn test_data_database_connection( + &self, + ) -> BoxFuture<'_, extenddb_storage::diagnostics::DiagResult> { + Box::pin(async { + // Ping the data database to verify connectivity + let data_db = self.client.database("extenddb_data"); + data_db.run_command(doc! { "ping": 1 }).await.map_err(|e| { + extenddb_storage::diagnostics::DiagError::ConnectionFailed(e.to_string()) + })?; + Ok("extenddb_data".to_string()) + }) + } +} diff --git a/crates/storage-mongodb/src/condition.rs b/crates/storage-mongodb/src/condition.rs new file mode 100644 index 00000000..c2301b94 --- /dev/null +++ b/crates/storage-mongodb/src/condition.rs @@ -0,0 +1,759 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Condition expression to `MongoDB` filter compiler. +//! +//! Translates `extenddb_core::expression::Expr` AST into a `bson::Document` +//! query filter for use with `MongoDB`'s filter pushdown (findOneAndReplace, etc.). + +use bson::{Bson, Document, doc}; + +use extenddb_core::expression::{CompareOp, Expr, ExpressionMaps, PathElement}; +use extenddb_core::types::AttributeValue; +use extenddb_storage::error::StorageError; + +/// Compile a condition expression to a `MongoDB` filter document. +/// +/// The resulting filter operates on the `item_data` field of the `MongoDB` document. +/// Returns an empty document if no condition is provided. +pub fn condition_to_filter(expr: &Expr, maps: &ExpressionMaps) -> Result { + compile_expr(expr, maps) +} + +/// Resolve a path to the `MongoDB` field path within `item_data`. +/// +/// `DynamoDB` paths like `address.city` or `tags[0]` become +/// `item_data.address.city` or `item_data.tags.L.0` in `MongoDB`. +fn resolve_path_to_field( + elements: &[PathElement], + maps: &ExpressionMaps, +) -> Result { + let mut parts = vec!["item_data".to_string()]; + for elem in elements { + match elem { + PathElement::Attribute(name) => { + let resolved = if let Some(stripped) = name.strip_prefix('#') { + maps.resolve_name(stripped) + .map_err(|e| StorageError::Validation(e.to_string()))? + .to_string() + } else { + name.clone() + }; + parts.push(resolved); + } + PathElement::Index(idx) => { + // For list index access: item_data.attr.L. + parts.push("L".to_string()); + parts.push(idx.to_string()); + } + } + } + Ok(parts.join(".")) +} + +/// Convert an `AttributeValue` to a BSON value for filter comparisons. +/// +/// The value must match how `data/mod.rs::item_to_document` stores the +/// value inside `item_data`, otherwise the compiled filter will not +/// match real stored items. Storage serializes each attribute value via +/// the `AttributeValue` `Serialize` impl (JSON-shape) and then converts +/// to BSON — so: +/// +/// - `S(s)` → JSON string → BSON string +/// - `N(n)` → JSON string (numbers are wire-encoded as strings) → BSON string +/// - `B(b)` → JSON string (base64) → BSON string +/// - `Bool(b)` → BSON boolean +/// - `Null` → JSON true (the `{"NULL": true}` tag) → BSON boolean true +/// +/// Emitting raw BSON `Binary` for `.B` here (as an earlier version did) +/// produced a filter that never matched real stored items because +/// storage writes the base64 string form. +fn av_to_bson(av: &AttributeValue) -> Bson { + use base64::Engine; + match av { + AttributeValue::S(s) => Bson::String(s.clone()), + AttributeValue::N(n) => Bson::String(n.clone()), + AttributeValue::B(b) => Bson::String(base64::engine::general_purpose::STANDARD.encode(b)), + AttributeValue::Bool(b) => Bson::Boolean(*b), + AttributeValue::Null => Bson::Boolean(true), // NULL type stores {"NULL": true} + _ => Bson::Null, // Sets and complex types handled differently + } +} + +/// Get the type suffix for a `DynamoDB` `AttributeValue` (S, N, B, BOOL, NULL, L, M, SS, NS, BS). +fn av_type_suffix(av: &AttributeValue) -> &'static str { + match av { + AttributeValue::S(_) => "S", + AttributeValue::N(_) => "N", + AttributeValue::B(_) => "B", + AttributeValue::Bool(_) => "BOOL", + AttributeValue::Null => "NULL", + AttributeValue::L(_) => "L", + AttributeValue::M(_) => "M", + AttributeValue::SS(_) => "SS", + AttributeValue::NS(_) => "NS", + AttributeValue::BS(_) => "BS", + } +} + +/// Resolve a value expression (Path or Placeholder) to the field path and BSON value. +/// +/// Returns (`field_path_for_filter`, `bson_value`) or just the `bson_value` for placeholders. +enum ResolvedValue { + /// A field path in the document (e.g., "`item_data.age.N`") + Field(String), + /// A literal BSON value with its type suffix + Literal(Bson, &'static str), +} + +fn resolve_value(expr: &Expr, maps: &ExpressionMaps) -> Result { + match expr { + Expr::Path(elements) => { + let field = resolve_path_to_field(elements, maps)?; + Ok(ResolvedValue::Field(field)) + } + Expr::Placeholder(name) => { + let av = maps + .resolve_value(name) + .map_err(|e| StorageError::Validation(e.to_string()))?; + let suffix = av_type_suffix(av); + let bson_val = av_to_bson(av); + Ok(ResolvedValue::Literal(bson_val, suffix)) + } + _ => Err(StorageError::Validation( + "Unexpected expression type in condition value position".to_string(), + )), + } +} + +/// Build a comparison filter between two expressions. +fn build_comparison( + left: &Expr, + op: CompareOp, + right: &Expr, + maps: &ExpressionMaps, +) -> Result { + let left_resolved = resolve_value(left, maps)?; + let right_resolved = resolve_value(right, maps)?; + + // Determine the field path and value for the comparison + let (field_path, value) = match (left_resolved, right_resolved) { + (ResolvedValue::Field(path), ResolvedValue::Literal(val, suffix)) => { + // field op :value -> item_data.field.TYPE op val + let typed_path = format!("{path}.{suffix}"); + (typed_path, val) + } + (ResolvedValue::Literal(val, suffix), ResolvedValue::Field(path)) => { + // :value op field -> reverse the comparison + let typed_path = format!("{path}.{suffix}"); + let reversed_op = reverse_op(op); + return build_field_comparison(&typed_path, reversed_op, val); + } + (ResolvedValue::Field(left_path), ResolvedValue::Field(right_path)) => { + // field op field -> use $expr + return build_field_vs_field_comparison(&left_path, op, &right_path); + } + (ResolvedValue::Literal(_, _), ResolvedValue::Literal(_, _)) => { + // literal op literal -> evaluate statically (unusual case) + // For simplicity, just return an empty filter (always true) + return Ok(doc! {}); + } + }; + + build_field_comparison(&field_path, op, value) +} + +fn build_field_comparison( + field: &str, + op: CompareOp, + value: Bson, +) -> Result { + let filter = match op { + CompareOp::Eq => doc! { field: value }, + CompareOp::Ne => doc! { field: { "$ne": value } }, + CompareOp::Lt => doc! { field: { "$lt": value } }, + CompareOp::Le => doc! { field: { "$lte": value } }, + CompareOp::Gt => doc! { field: { "$gt": value } }, + CompareOp::Ge => doc! { field: { "$gte": value } }, + }; + Ok(filter) +} + +fn build_field_vs_field_comparison( + left_field: &str, + op: CompareOp, + right_field: &str, +) -> Result { + let mongo_op = match op { + CompareOp::Eq => "$eq", + CompareOp::Ne => "$ne", + CompareOp::Lt => "$lt", + CompareOp::Le => "$lte", + CompareOp::Gt => "$gt", + CompareOp::Ge => "$gte", + }; + Ok(doc! { + "$expr": { + mongo_op: [format!("${left_field}"), format!("${right_field}")] + } + }) +} + +fn reverse_op(op: CompareOp) -> CompareOp { + match op { + CompareOp::Eq => CompareOp::Eq, + CompareOp::Ne => CompareOp::Ne, + CompareOp::Lt => CompareOp::Gt, + CompareOp::Le => CompareOp::Ge, + CompareOp::Gt => CompareOp::Lt, + CompareOp::Ge => CompareOp::Le, + } +} + +/// Compile an expression AST node to a `MongoDB` filter document. +fn compile_expr(expr: &Expr, maps: &ExpressionMaps) -> Result { + match expr { + Expr::Compare { left, op, right } => build_comparison(left, *op, right, maps), + + Expr::And(left, right) => { + let left_filter = compile_expr(left, maps)?; + let right_filter = compile_expr(right, maps)?; + Ok(doc! { "$and": [left_filter, right_filter] }) + } + + Expr::Or(left, right) => { + let left_filter = compile_expr(left, maps)?; + let right_filter = compile_expr(right, maps)?; + Ok(doc! { "$or": [left_filter, right_filter] }) + } + + Expr::Not(inner) => { + let inner_filter = compile_expr(inner, maps)?; + Ok(doc! { "$nor": [inner_filter] }) + } + + Expr::Function { name, args } => compile_function(name, args, maps), + + Expr::Between { operand, low, high } => compile_between(operand, low, high, maps), + + Expr::In { operand, list } => compile_in(operand, list, maps), + + _ => Err(StorageError::Validation( + "Unsupported expression type in condition filter".to_string(), + )), + } +} + +fn compile_function( + name: &str, + args: &[Expr], + maps: &ExpressionMaps, +) -> Result { + match name.to_lowercase().as_str() { + "attribute_exists" => { + if args.len() != 1 { + return Err(StorageError::Validation( + "attribute_exists requires exactly one argument".to_string(), + )); + } + let field = resolve_path_from_expr(&args[0], maps)?; + Ok(doc! { &field: { "$exists": true } }) + } + + "attribute_not_exists" => { + if args.len() != 1 { + return Err(StorageError::Validation( + "attribute_not_exists requires exactly one argument".to_string(), + )); + } + let field = resolve_path_from_expr(&args[0], maps)?; + Ok(doc! { &field: { "$exists": false } }) + } + + "begins_with" => { + if args.len() != 2 { + return Err(StorageError::Validation( + "begins_with requires exactly two arguments".to_string(), + )); + } + let field = resolve_path_from_expr(&args[0], maps)?; + let prefix_val = resolve_literal(&args[1], maps)?; + match prefix_val { + AttributeValue::S(prefix) => { + let escaped = regex_escape(&prefix); + let typed_field = format!("{field}.S"); + Ok(doc! { &typed_field: { "$regex": format!("^{escaped}") } }) + } + _ => Err(StorageError::Validation( + "begins_with requires a string prefix".to_string(), + )), + } + } + + "contains" => { + if args.len() != 2 { + return Err(StorageError::Validation( + "contains requires exactly two arguments".to_string(), + )); + } + let field = resolve_path_from_expr(&args[0], maps)?; + let val = resolve_literal(&args[1], maps)?; + match &val { + AttributeValue::S(substr) => { + // String contains: check substring in string field OR membership in SS/L + let escaped = regex_escape(substr); + let string_field = format!("{field}.S"); + let ss_field = format!("{field}.SS"); + let list_field = format!("{field}.L"); + let bson_val = av_to_bson(&val); + let list_elem = doc! { "S": substr.as_str() }; + Ok(doc! { "$or": [ + { &string_field: { "$regex": &escaped } }, + { &ss_field: &bson_val }, + { &list_field: &list_elem }, + ] }) + } + AttributeValue::N(n) => { + // Number membership in NS or L + let ns_field = format!("{field}.NS"); + let list_field = format!("{field}.L"); + let bson_val = av_to_bson(&val); + let list_elem = doc! { "N": n.as_str() }; + Ok(doc! { "$or": [ + { &ns_field: &bson_val }, + { &list_field: &list_elem }, + ] }) + } + AttributeValue::B(_) => { + // Binary membership in BS or L. Storage serializes B + // as base64 strings inside item_data (matches wire + // format via the JSON serializer), so both the field + // predicate and the list-element predicate use the + // base64 form. + let bs_field = format!("{field}.BS"); + let list_field = format!("{field}.L"); + let bson_val = av_to_bson(&val); + let list_elem = doc! { "B": bson_val.clone() }; + Ok(doc! { "$or": [ + { &bs_field: &bson_val }, + { &list_field: &list_elem }, + ] }) + } + _ => { + // For other types, check membership in L (list) + let list_field = format!("{field}.L"); + let suffix = av_type_suffix(&val); + let list_elem = doc! { suffix: av_to_bson(&val) }; + Ok(doc! { &list_field: &list_elem }) + } + } + } + + "attribute_type" => { + if args.len() != 2 { + return Err(StorageError::Validation( + "attribute_type requires exactly two arguments".to_string(), + )); + } + let field = resolve_path_from_expr(&args[0], maps)?; + let type_val = resolve_literal(&args[1], maps)?; + match type_val { + AttributeValue::S(type_name) => { + let typed_field = format!("{field}.{type_name}"); + Ok(doc! { &typed_field: { "$exists": true } }) + } + _ => Err(StorageError::Validation( + "attribute_type requires a string type argument".to_string(), + )), + } + } + + "size" => { + // size() is used in comparisons, not standalone. + // This case handles it if it appears as a standalone function call, + // which shouldn't happen in well-formed expressions. + Err(StorageError::Validation( + "size() cannot be used as a standalone condition".to_string(), + )) + } + + _ => Err(StorageError::Validation(format!( + "Unsupported function in condition: {name}" + ))), + } +} + +fn compile_between( + operand: &Expr, + low: &Expr, + high: &Expr, + maps: &ExpressionMaps, +) -> Result { + let operand_resolved = resolve_value(operand, maps)?; + let low_resolved = resolve_value(low, maps)?; + let high_resolved = resolve_value(high, maps)?; + + if let ( + ResolvedValue::Field(path), + ResolvedValue::Literal(low_val, suffix), + ResolvedValue::Literal(high_val, _), + ) = (operand_resolved, low_resolved, high_resolved) + { + let typed_path = format!("{path}.{suffix}"); + Ok(doc! { &typed_path: { "$gte": low_val, "$lte": high_val } }) + } else { + // Fallback: compile as AND of two comparisons + let gte = build_comparison(operand, CompareOp::Ge, low, maps)?; + let lte = build_comparison(operand, CompareOp::Le, high, maps)?; + Ok(doc! { "$and": [gte, lte] }) + } +} + +fn compile_in( + operand: &Expr, + list: &[Expr], + maps: &ExpressionMaps, +) -> Result { + let operand_resolved = resolve_value(operand, maps)?; + + match operand_resolved { + ResolvedValue::Field(path) => { + // Collect all values, assuming they are the same type + if list.is_empty() { + // Empty IN list never matches — use $and with contradictory conditions + return Ok( + doc! { "$and": [ { "_id": { "$exists": true } }, { "_id": { "$type": "null" } } ] }, + ); + } + + let first_literal = resolve_literal(&list[0], maps)?; + let suffix = av_type_suffix(&first_literal); + let typed_path = format!("{path}.{suffix}"); + + let values: Vec = list + .iter() + .map(|expr| { + let av = resolve_literal(expr, maps)?; + Ok(av_to_bson(&av)) + }) + .collect::, StorageError>>()?; + + Ok(doc! { &typed_path: { "$in": values } }) + } + ResolvedValue::Literal(_, _) => { + // Literal IN list of fields — unusual, compile as OR + let mut or_clauses = Vec::new(); + for item in list { + let eq = build_comparison(operand, CompareOp::Eq, item, maps)?; + or_clauses.push(Bson::Document(eq)); + } + Ok(doc! { "$or": or_clauses }) + } + } +} + +fn resolve_path_from_expr(expr: &Expr, maps: &ExpressionMaps) -> Result { + match expr { + Expr::Path(elements) => resolve_path_to_field(elements, maps), + _ => Err(StorageError::Validation( + "Expected a path expression".to_string(), + )), + } +} + +fn resolve_literal(expr: &Expr, maps: &ExpressionMaps) -> Result { + match expr { + Expr::Placeholder(name) => maps + .resolve_value(name) + .cloned() + .map_err(|e| StorageError::Validation(e.to_string())), + _ => Err(StorageError::Validation( + "Expected a value placeholder".to_string(), + )), + } +} + +/// Escape special regex characters in a string. +fn regex_escape(s: &str) -> String { + let special = [ + '.', '^', '$', '*', '+', '?', '(', ')', '[', ']', '{', '}', '|', '\\', + ]; + let mut result = String::with_capacity(s.len()); + for c in s.chars() { + if special.contains(&c) { + result.push('\\'); + } + result.push(c); + } + result +} + +// ============================================================================ +// Unit Tests +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + use extenddb_core::expression::ExpressionMaps; + use extenddb_core::types::AttributeValue; + use std::collections::HashMap; + + fn make_maps(names: Vec<(&str, &str)>, values: Vec<(&str, AttributeValue)>) -> ExpressionMaps { + let names_map: HashMap = names + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + let values_map: HashMap = values + .into_iter() + .map(|(k, v)| (k.to_string(), v)) + .collect(); + ExpressionMaps::new(names_map, values_map) + } + + #[test] + fn test_attribute_exists() { + let maps = make_maps(vec![], vec![]); + let expr = Expr::Function { + name: "attribute_exists".to_string(), + args: vec![Expr::Path(vec![PathElement::Attribute("foo".to_string())])], + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!(filter, doc! { "item_data.foo": { "$exists": true } }); + } + + #[test] + fn test_attribute_not_exists() { + let maps = make_maps(vec![], vec![]); + let expr = Expr::Function { + name: "attribute_not_exists".to_string(), + args: vec![Expr::Path(vec![PathElement::Attribute("bar".to_string())])], + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!(filter, doc! { "item_data.bar": { "$exists": false } }); + } + + #[test] + fn test_equality_comparison_string() { + let maps = make_maps( + vec![], + vec![(":val", AttributeValue::S("hello".to_string()))], + ); + let expr = Expr::Compare { + left: Box::new(Expr::Path(vec![PathElement::Attribute("name".to_string())])), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":val".to_string())), + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!(filter, doc! { "item_data.name.S": "hello" }); + } + + #[test] + fn test_less_than_comparison_number() { + let maps = make_maps(vec![], vec![(":min", AttributeValue::N("100".to_string()))]); + let expr = Expr::Compare { + left: Box::new(Expr::Path(vec![PathElement::Attribute( + "price".to_string(), + )])), + op: CompareOp::Lt, + right: Box::new(Expr::Placeholder(":min".to_string())), + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!(filter, doc! { "item_data.price.N": { "$lt": "100" } }); + } + + #[test] + fn test_and_condition() { + let maps = make_maps( + vec![], + vec![ + (":v1", AttributeValue::S("active".to_string())), + (":v2", AttributeValue::N("5".to_string())), + ], + ); + let expr = Expr::And( + Box::new(Expr::Compare { + left: Box::new(Expr::Path(vec![PathElement::Attribute( + "status".to_string(), + )])), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":v1".to_string())), + }), + Box::new(Expr::Compare { + left: Box::new(Expr::Path(vec![PathElement::Attribute( + "count".to_string(), + )])), + op: CompareOp::Gt, + right: Box::new(Expr::Placeholder(":v2".to_string())), + }), + ); + let filter = condition_to_filter(&expr, &maps).unwrap(); + let expected = doc! { + "$and": [ + { "item_data.status.S": "active" }, + { "item_data.count.N": { "$gt": "5" } } + ] + }; + assert_eq!(filter, expected); + } + + #[test] + fn test_or_condition() { + let maps = make_maps( + vec![], + vec![ + (":a", AttributeValue::S("x".to_string())), + (":b", AttributeValue::S("y".to_string())), + ], + ); + let expr = Expr::Or( + Box::new(Expr::Compare { + left: Box::new(Expr::Path(vec![PathElement::Attribute("f".to_string())])), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":a".to_string())), + }), + Box::new(Expr::Compare { + left: Box::new(Expr::Path(vec![PathElement::Attribute("f".to_string())])), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":b".to_string())), + }), + ); + let filter = condition_to_filter(&expr, &maps).unwrap(); + let expected = doc! { + "$or": [ + { "item_data.f.S": "x" }, + { "item_data.f.S": "y" } + ] + }; + assert_eq!(filter, expected); + } + + #[test] + fn test_not_condition() { + let maps = make_maps(vec![], vec![]); + let expr = Expr::Not(Box::new(Expr::Function { + name: "attribute_exists".to_string(), + args: vec![Expr::Path(vec![PathElement::Attribute( + "deleted".to_string(), + )])], + })); + let filter = condition_to_filter(&expr, &maps).unwrap(); + let expected = doc! { + "$nor": [{ "item_data.deleted": { "$exists": true } }] + }; + assert_eq!(filter, expected); + } + + #[test] + fn test_begins_with() { + let maps = make_maps( + vec![], + vec![(":prefix", AttributeValue::S("user#".to_string()))], + ); + let expr = Expr::Function { + name: "begins_with".to_string(), + args: vec![ + Expr::Path(vec![PathElement::Attribute("sk".to_string())]), + Expr::Placeholder(":prefix".to_string()), + ], + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!(filter, doc! { "item_data.sk.S": { "$regex": "^user#" } }); + } + + #[test] + fn test_between() { + let maps = make_maps( + vec![], + vec![ + (":lo", AttributeValue::N("10".to_string())), + (":hi", AttributeValue::N("20".to_string())), + ], + ); + let expr = Expr::Between { + operand: Box::new(Expr::Path(vec![PathElement::Attribute("age".to_string())])), + low: Box::new(Expr::Placeholder(":lo".to_string())), + high: Box::new(Expr::Placeholder(":hi".to_string())), + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!( + filter, + doc! { "item_data.age.N": { "$gte": "10", "$lte": "20" } } + ); + } + + #[test] + fn test_in_condition() { + let maps = make_maps( + vec![], + vec![ + (":v1", AttributeValue::S("a".to_string())), + (":v2", AttributeValue::S("b".to_string())), + (":v3", AttributeValue::S("c".to_string())), + ], + ); + let expr = Expr::In { + operand: Box::new(Expr::Path(vec![PathElement::Attribute("x".to_string())])), + list: vec![ + Expr::Placeholder(":v1".to_string()), + Expr::Placeholder(":v2".to_string()), + Expr::Placeholder(":v3".to_string()), + ], + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!(filter, doc! { "item_data.x.S": { "$in": ["a", "b", "c"] } }); + } + + #[test] + fn test_name_ref_resolution() { + let maps = make_maps( + vec![("n", "status")], + vec![(":v", AttributeValue::S("active".to_string()))], + ); + let expr = Expr::Compare { + left: Box::new(Expr::Path(vec![PathElement::Attribute("#n".to_string())])), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":v".to_string())), + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!(filter, doc! { "item_data.status.S": "active" }); + } + + #[test] + fn test_ne_comparison() { + let maps = make_maps( + vec![], + vec![(":v", AttributeValue::S("deleted".to_string()))], + ); + let expr = Expr::Compare { + left: Box::new(Expr::Path(vec![PathElement::Attribute( + "status".to_string(), + )])), + op: CompareOp::Ne, + right: Box::new(Expr::Placeholder(":v".to_string())), + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!(filter, doc! { "item_data.status.S": { "$ne": "deleted" } }); + } + + #[test] + fn test_regex_escape() { + assert_eq!(regex_escape("user.name"), "user\\.name"); + assert_eq!(regex_escape("a+b"), "a\\+b"); + assert_eq!(regex_escape("normal"), "normal"); + } + + #[test] + fn test_attribute_type() { + let maps = make_maps(vec![], vec![(":t", AttributeValue::S("S".to_string()))]); + let expr = Expr::Function { + name: "attribute_type".to_string(), + args: vec![ + Expr::Path(vec![PathElement::Attribute("field".to_string())]), + Expr::Placeholder(":t".to_string()), + ], + }; + let filter = condition_to_filter(&expr, &maps).unwrap(); + assert_eq!(filter, doc! { "item_data.field.S": { "$exists": true } }); + } +} diff --git a/crates/storage-mongodb/src/config.rs b/crates/storage-mongodb/src/config.rs new file mode 100644 index 00000000..9e5a4e26 --- /dev/null +++ b/crates/storage-mongodb/src/config.rs @@ -0,0 +1,58 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Configuration for `MongoDB` storage backend. + +use serde::{Deserialize, Serialize}; + +/// `MongoDB` storage backend configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MongoStorageConfig { + /// `MongoDB` connection string (mongodb://...) + pub connection_string: String, + /// Maximum concurrent connections for data operations + #[serde(default = "default_max_connections")] + pub max_connections: u32, + /// Maximum concurrent connections for catalog/management operations + #[serde(default = "default_max_catalog_connections")] + pub max_catalog_connections: u32, +} + +fn default_max_connections() -> u32 { + 50 +} + +fn default_max_catalog_connections() -> u32 { + 20 +} + +impl extenddb_storage::config::StorageConfig for MongoStorageConfig { + fn connection_config(&self) -> &str { + &self.connection_string + } + + fn max_connections(&self) -> u32 { + self.max_connections + } + + fn max_catalog_connections(&self) -> u32 { + self.max_catalog_connections + } + + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +impl TryFrom for MongoStorageConfig { + type Error = toml::de::Error; + + fn try_from(table: toml::Table) -> Result { + let value = toml::Value::Table(table); + value.try_into() + } +} diff --git a/crates/storage-mongodb/src/credential_store.rs b/crates/storage-mongodb/src/credential_store.rs new file mode 100644 index 00000000..05655eec --- /dev/null +++ b/crates/storage-mongodb/src/credential_store.rs @@ -0,0 +1,224 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Credential store implementation for `MongoDB`. + +use mongodb::bson::{Document, doc}; +use zeroize::{Zeroize, ZeroizeOnDrop}; + +use extenddb_auth::{CredentialStore, StoredCredential}; +use extenddb_core::error::DynamoDbError; + +/// `MongoDB` credential store for authentication. +/// +/// The `encryption_key` is zeroed from memory on drop. +#[derive(Zeroize, ZeroizeOnDrop)] +pub struct MongoCredentialStore { + #[zeroize(skip)] + client: mongodb::Client, + encryption_key: String, +} + +impl MongoCredentialStore { + #[must_use] + pub fn new(client: mongodb::Client, encryption_key: String) -> Self { + Self { + client, + encryption_key, + } + } + + fn catalog_db(&self) -> mongodb::Database { + self.client.database("extenddb_catalog") + } + + async fn lookup_user_credential( + &self, + access_key_id: &str, + ) -> Result, DynamoDbError> { + let coll = self.catalog_db().collection::("access_keys"); + let doc = coll + .find_one(doc! { "access_key_id": access_key_id }) + .await + .map_err(|e| { + tracing::error!("Credential lookup failed for access key {access_key_id}: {e}"); + DynamoDbError::InternalServerError( + "Internal error during authentication".to_owned(), + ) + })?; + + let Some(key_doc) = doc else { + return Ok(None); + }; + + let encrypted = match key_doc.get_binary_generic("secret_key_encrypted") { + Ok(bytes) => bytes.clone(), + Err(_) => return Ok(None), + }; + let account_id = key_doc.get_str("account_id").unwrap_or_default().to_owned(); + let user_name = key_doc.get_str("user_name").unwrap_or_default().to_owned(); + // Fail closed: treat a missing or malformed is_active as inactive so + // a corrupted or partially written access-key record cannot silently + // authenticate. + let is_active = key_doc.get_bool("is_active").unwrap_or(false); + + let secret_key = + decrypt_secret(&encrypted, &self.encryption_key, access_key_id).map_err(|e| { + tracing::error!("Secret key decryption failed for access key {access_key_id}: {e}"); + DynamoDbError::InternalServerError( + "Internal error during authentication".to_owned(), + ) + })?; + + Ok(Some(StoredCredential { + secret_key, + account_id, + principal_name: user_name, + session_name: None, + is_session: false, + session_token: None, + is_active, + expires_at: None, + })) + } + + async fn lookup_session_credential( + &self, + access_key_id: &str, + ) -> Result, DynamoDbError> { + let coll = self.catalog_db().collection::("iam_sessions"); + let doc = coll + .find_one(doc! { "access_key_id": access_key_id }) + .await + .map_err(|e| { + tracing::error!( + "Session credential lookup failed for access key {access_key_id}: {e}" + ); + DynamoDbError::InternalServerError( + "Internal error during authentication".to_owned(), + ) + })?; + + let Some(session_doc) = doc else { + return Ok(None); + }; + + let encrypted = match session_doc.get_binary_generic("secret_key_encrypted") { + Ok(bytes) => bytes.clone(), + Err(_) => return Ok(None), + }; + let account_id = session_doc + .get_str("account_id") + .unwrap_or_default() + .to_owned(); + let role_name = session_doc + .get_str("role_name") + .unwrap_or_default() + .to_owned(); + let session_name = session_doc + .get_str("session_name") + .unwrap_or_default() + .to_owned(); + let session_token = session_doc + .get_str("session_token") + .unwrap_or_default() + .to_owned(); + + let expires_at = session_doc.get_datetime("expires_at").map_err(|_| { + DynamoDbError::InternalServerError("Internal error during authentication".to_owned()) + })?; + + let expires_ts = time::OffsetDateTime::from_unix_timestamp_nanos( + i128::from(expires_at.timestamp_millis()) * 1_000_000, + ) + .unwrap_or(time::OffsetDateTime::UNIX_EPOCH); + + if expires_ts < time::OffsetDateTime::now_utc() { + return Err(DynamoDbError::ExpiredTokenException( + "The security token included in the request is expired".to_owned(), + )); + } + + let secret_key = + decrypt_secret(&encrypted, &self.encryption_key, access_key_id).map_err(|e| { + tracing::error!( + "Session secret key decryption failed for access key {access_key_id}: {e}" + ); + DynamoDbError::InternalServerError( + "Internal error during authentication".to_owned(), + ) + })?; + + Ok(Some(StoredCredential { + secret_key, + account_id, + principal_name: role_name, + session_name: Some(session_name), + is_session: true, + session_token: Some(session_token), + is_active: true, + expires_at: Some(expires_ts), + })) + } +} + +#[async_trait::async_trait] +impl CredentialStore for MongoCredentialStore { + async fn lookup_credential( + &self, + access_key_id: &str, + ) -> Result, DynamoDbError> { + if access_key_id.starts_with("AKIA") { + return self.lookup_user_credential(access_key_id).await; + } + + if access_key_id.starts_with("ASIA") { + return self.lookup_session_credential(access_key_id).await; + } + + Ok(None) + } +} + +// ── Crypto helpers ────────────────────────────────────────────────────── + +fn decrypt_secret(encrypted: &[u8], key_b64: &str, aad: &str) -> Result { + use aes_gcm::Aes256Gcm; + use aes_gcm::KeyInit; + use aes_gcm::aead::Aead; + use aes_gcm::aead::Payload; + use base64::Engine; + + if encrypted.len() < 28 { + return Err( + "ciphertext too short (need at least 12-byte nonce + 16-byte auth tag)".to_owned(), + ); + } + + let key_bytes = base64::engine::general_purpose::STANDARD + .decode(key_b64) + .map_err(|e| format!("decode encryption key: {e}"))?; + + let key = aes_gcm::Key::::from_slice(&key_bytes); + let cipher = Aes256Gcm::new(key); + let nonce = aes_gcm::Nonce::from_slice(&encrypted[..12]); + + // Try with AAD first (CB-11 format). + let payload_with_aad = Payload { + msg: &encrypted[12..], + aad: aad.as_bytes(), + }; + if let Ok(plaintext_bytes) = cipher.decrypt(nonce, payload_with_aad) { + return String::from_utf8(plaintext_bytes) + .map_err(|e| format!("decrypted secret is not valid UTF-8: {e}")); + } + + // Fall back to without AAD (pre-CB-11 format). + tracing::debug!("Decrypting secret without AAD (pre-CB-11 format) for {aad}"); + let plaintext_bytes = cipher + .decrypt(nonce, &encrypted[12..]) + .map_err(|e| format!("decrypt: {e}"))?; + + String::from_utf8(plaintext_bytes) + .map_err(|e| format!("decrypted secret is not valid UTF-8: {e}")) +} diff --git a/crates/storage-mongodb/src/data/mod.rs b/crates/storage-mongodb/src/data/mod.rs new file mode 100644 index 00000000..45465ba6 --- /dev/null +++ b/crates/storage-mongodb/src/data/mod.rs @@ -0,0 +1,634 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Data engine helpers for the `MongoDB` backend. +//! +//! Contains document conversion, collection naming, and key extraction utilities. + +use bson::{Document, doc}; + +#[cfg(test)] +use extenddb_core::types::KeyType; +use extenddb_core::types::{ + AttributeDefinition, AttributeValue, Item, KeySchemaElement, ScalarAttributeType, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{composite_pk_to_text, encode_netstring_composite, sk_info}; + +/// Returns the `MongoDB` collection name for a `DynamoDB` table. +pub fn data_collection_name(table_id: &str) -> String { + format!("_ddb_{table_id}") +} + +/// Build the mongo document `_id` for a composite (partition + sort) key. +/// +/// Uses netstring encoding — `:,:,` — so the boundary +/// between `pk` and `sk` is unambiguous regardless of the contents of +/// either. A naive `"{pk}#{sk}"` scheme collides when `pk` or `sk` contains +/// the delimiter (e.g., `pk="a#b", sk="c"` and `pk="a", sk="b#c"` both +/// produce `"a#b#c"`). +#[must_use] +pub fn composite_id(pk_text: &str, sk_text: &str) -> String { + encode_netstring_composite(&[pk_text.to_owned(), sk_text.to_owned()]) +} + +/// Convert a `DynamoDB` Item to a `MongoDB` BSON document for storage. +/// +/// Document structure: `{ _id, pk, sk_s/sk_n/sk_b, item_data }` +pub fn item_to_document( + item: &Item, + key_schema: &[KeySchemaElement], + attribute_definitions: &[AttributeDefinition], +) -> Result { + let pk_text = composite_pk_to_text(item, key_schema)?; + + // Serialize the full item as item_data + let item_json = + serde_json::to_value(item).map_err(|e| StorageError::Internal(e.to_string()))?; + let item_bson = bson::to_bson(&item_json).map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut doc = Document::new(); + + // Build the _id field + if let Some((sk_name, sk_type)) = sk_info(key_schema, attribute_definitions) { + let sk_value = item + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk_text = sk_to_text(sk_value)?; + // Netstring-encoded composite _id — see composite_id() for why the + // naive "{pk}#{sk}" form is collision-prone. + doc.insert("_id", composite_id(&pk_text, &sk_text)); + doc.insert("pk", pk_text); + let sk_field = format!("sk_{}", sk_suffix(sk_type)); + insert_typed_sk(&mut doc, &sk_field, sk_type, sk_value)?; + } else { + // PK-only table + doc.insert("_id", pk_text.clone()); + doc.insert("pk", pk_text); + } + + doc.insert("item_data", item_bson); + Ok(doc) +} + +/// Insert a typed sort-key value into a document under the given field name. +/// +/// Shared between base tables (`sk_s`/`sk_n`/`sk_b`) and index documents +/// carrying the base table's sort key (`base_sk_s`/`base_sk_n`/`base_sk_b`). +fn insert_typed_sk( + doc: &mut Document, + field: &str, + sk_type: ScalarAttributeType, + sk_value: &AttributeValue, +) -> Result<(), StorageError> { + match (sk_type, sk_value) { + (ScalarAttributeType::S, AttributeValue::S(s)) => { + doc.insert(field, s.clone()); + } + (ScalarAttributeType::N, AttributeValue::N(n)) => { + // Store as Decimal128 for correct numeric ordering. Values that + // exceed Decimal128's 34 significant digits are rejected rather + // than downcasting to f64, which would silently lose precision. + let d = n.parse::().map_err(|_| { + StorageError::Validation(format!( + "Numeric sort key value '{n}' exceeds supported precision (Decimal128, 34 significant digits)" + )) + })?; + doc.insert(field, d); + } + (ScalarAttributeType::B, AttributeValue::B(b)) => { + // Store as hex string, not BSON Binary. See `binary_sk_to_hex` + // for the rationale — MongoDB's Binary sort order diverges + // from DDB's unsigned-lex byte order for unequal-length + // values (D-M5 / RFC-0003 §1.4). + doc.insert(field, binary_sk_to_hex(b)); + } + _ => { + // Mismatched types are silently skipped — matches the existing + // behavior of item_to_document. Callers can rely on + // validate_index_keys / validate_item_keys upstream to reject + // these before write. + } + } + Ok(()) +} + +/// Build a `MongoDB` index-collection document. +/// +/// Index documents differ from base-table documents in two ways: +/// +/// 1. The `_id` incorporates the base-table primary key in addition to the +/// index primary key. GSI keys are non-unique — multiple base items can +/// share identical `(index_pk, index_sk)` values. Encoding the base key +/// into `_id` gives each index entry a unique identity keyed to the base +/// item it describes. +/// +/// 2. The document carries the base-table key attributes as first-class +/// fields — `base_pk` (text) and `base_sk_s`/`base_sk_n`/`base_sk_b` +/// (typed). This lets index pagination form a compound cursor +/// `(index_sk, base_pk, base_sk)` without traversing the JSON +/// `item_data` payload for base-key values. +/// +/// The `item_data` payload is unchanged — it is the full projected item +/// as serialized by `AttributeValue`. +/// +/// `projected` is the item projected into the index (see `project_item` in +/// `data_engine.rs`). It must contain both the index-key attributes and +/// the base-table key attributes. +pub fn index_document( + projected: &Item, + idx_key_schema: &[KeySchemaElement], + base_key_schema: &[KeySchemaElement], + attribute_definitions: &[AttributeDefinition], +) -> Result { + let idx_pk_text = composite_pk_to_text(projected, idx_key_schema)?; + let base_pk_text = composite_pk_to_text(projected, base_key_schema)?; + + let idx_sk = sk_info(idx_key_schema, attribute_definitions); + let base_sk = sk_info(base_key_schema, attribute_definitions); + + // Build the netstring composite _id. Order: + // [index_pk, index_sk_or_"", base_pk, base_sk_or_""] + // Netstring parts are self-delimiting, so absent sk components encode as + // "0:," and the boundary is preserved. + let idx_sk_text = match idx_sk { + Some((sk_name, _)) => projected + .get(sk_name) + .map(sk_to_text) + .transpose()? + .unwrap_or_default(), + None => String::new(), + }; + let base_sk_text = match base_sk { + Some((sk_name, _)) => projected + .get(sk_name) + .map(sk_to_text) + .transpose()? + .unwrap_or_default(), + None => String::new(), + }; + let id = encode_netstring_composite(&[ + idx_pk_text.clone(), + idx_sk_text, + base_pk_text.clone(), + base_sk_text, + ]); + + let mut doc = Document::new(); + doc.insert("_id", id); + doc.insert("pk", &idx_pk_text); + doc.insert("base_pk", &base_pk_text); + + if let Some((sk_name, sk_type)) = idx_sk + && let Some(sk_value) = projected.get(sk_name) + { + let field = format!("sk_{}", sk_suffix(sk_type)); + insert_typed_sk(&mut doc, &field, sk_type, sk_value)?; + } + if let Some((sk_name, sk_type)) = base_sk + && let Some(sk_value) = projected.get(sk_name) + { + let field = format!("base_sk_{}", sk_suffix(sk_type)); + insert_typed_sk(&mut doc, &field, sk_type, sk_value)?; + } + + let item_json = + serde_json::to_value(projected).map_err(|e| StorageError::Internal(e.to_string()))?; + let item_bson = bson::to_bson(&item_json).map_err(|e| StorageError::Internal(e.to_string()))?; + doc.insert("item_data", item_bson); + + Ok(doc) +} + +/// Build a delete filter for a specific index entry. +/// +/// The filter must match the exact base item's index entry, so it needs +/// both the index-key and base-key components — a filter on index keys +/// alone would delete every base item's entry that shares those index +/// keys (silent data loss on GSIs with duplicate keys). Returns a filter +/// on `(pk, sk?, base_pk, base_sk?)` — the same tuple that composes +/// the `_id`, but we filter on the individual fields so mongo can use +/// per-field indexes if present. +pub fn index_entry_filter( + projected: &Item, + idx_key_schema: &[KeySchemaElement], + base_key_schema: &[KeySchemaElement], + attribute_definitions: &[AttributeDefinition], +) -> Result { + let idx_pk_text = composite_pk_to_text(projected, idx_key_schema)?; + let base_pk_text = composite_pk_to_text(projected, base_key_schema)?; + let mut filter = doc! { + "pk": idx_pk_text, + "base_pk": base_pk_text, + }; + + if let Some((sk_name, sk_type)) = sk_info(idx_key_schema, attribute_definitions) + && let Some(sk_value) = projected.get(sk_name) + { + let field = format!("sk_{}", sk_suffix(sk_type)); + insert_typed_sk(&mut filter, &field, sk_type, sk_value)?; + } + if let Some((sk_name, sk_type)) = sk_info(base_key_schema, attribute_definitions) + && let Some(sk_value) = projected.get(sk_name) + { + let field = format!("base_sk_{}", sk_suffix(sk_type)); + insert_typed_sk(&mut filter, &field, sk_type, sk_value)?; + } + Ok(filter) +} + +/// Sort-key column suffix for a scalar attribute type. Shared by index and +/// base-key field naming. +#[must_use] +pub fn sk_suffix(sk_type: ScalarAttributeType) -> &'static str { + match sk_type { + ScalarAttributeType::S => "s", + ScalarAttributeType::N => "n", + ScalarAttributeType::B => "b", + } +} + +/// Convert a `MongoDB` document back to a `DynamoDB` Item. +pub fn document_to_item(doc: &Document) -> Result { + let item_data = doc + .get("item_data") + .ok_or_else(|| StorageError::Internal("Document missing item_data field".to_string()))?; + + let json_value: serde_json::Value = bson::from_bson(item_data.clone()) + .map_err(|e| StorageError::Internal(format!("BSON to JSON conversion error: {e}")))?; + + let item: Item = serde_json::from_value(json_value) + .map_err(|e| StorageError::Internal(format!("JSON to Item conversion error: {e}")))?; + + Ok(item) +} + +/// Convert a sort key value to text for use in the _id field. +fn sk_to_text(value: &AttributeValue) -> Result { + match value { + AttributeValue::S(s) => Ok(s.clone()), + AttributeValue::N(n) => Ok(n.clone()), + AttributeValue::B(b) => { + use base64::Engine; + Ok(base64::engine::general_purpose::STANDARD.encode(b)) + } + _ => Err(StorageError::Internal( + "sort key must be S, N, or B".to_owned(), + )), + } +} + +/// Encode a byte slice as a lowercase hex string. +/// +/// Used to store binary sort keys as strings in the typed +/// `sk_b`/`base_sk_b` fields. Lexicographic comparison of hex-encoded +/// strings preserves DynamoDB's unsigned-lex byte order — MongoDB's +/// native BSON Binary comparison is length-first-then-content, which +/// diverges from DDB for values of different lengths (e.g., DDB says +/// `[0x01,0xFF] < [0x02]`; BSON Binary reverses that). Hex strings +/// also make `begins_with` implementable as a plain string range +/// filter instead of a full-partition post-fetch scan. RFC-0003 §1.4. +#[must_use] +pub fn binary_sk_to_hex(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for &b in bytes { + out.push(HEX[(b >> 4) as usize] as char); + out.push(HEX[(b & 0x0f) as usize] as char); + } + out +} + +/// Build a primary key filter for `MongoDB` queries. +pub fn pk_filter( + key: &Item, + key_schema: &[KeySchemaElement], + attribute_definitions: &[AttributeDefinition], +) -> Result { + let pk_text = composite_pk_to_text(key, key_schema)?; + let mut filter = doc! { "pk": &pk_text }; + + if let Some((sk_name, sk_type)) = sk_info(key_schema, attribute_definitions) { + let sk_value = key + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key in key".to_owned()))?; + match sk_type { + ScalarAttributeType::S => { + if let AttributeValue::S(s) = sk_value { + filter.insert("sk_s", s.clone()); + } + } + ScalarAttributeType::N => { + if let AttributeValue::N(n) = sk_value { + let d = n.parse::().map_err(|_| { + StorageError::Validation(format!( + "Numeric key value '{n}' exceeds supported precision (Decimal128, 34 significant digits)" + )) + })?; + filter.insert("sk_n", d); + } + } + ScalarAttributeType::B => { + if let AttributeValue::B(b) = sk_value { + // Hex-encoded string, matching how insert_typed_sk + // writes sk_b — see D-M5. + filter.insert("sk_b", binary_sk_to_hex(b)); + } + } + } + } + + Ok(filter) +} + +/// Get the sort key column name for a table. +pub fn sk_field_name( + key_schema: &[KeySchemaElement], + attribute_definitions: &[AttributeDefinition], +) -> Option<&'static str> { + sk_info(key_schema, attribute_definitions).map(|(_, sk_type)| match sk_type { + ScalarAttributeType::S => "sk_s", + ScalarAttributeType::N => "sk_n", + ScalarAttributeType::B => "sk_b", + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn schema_pk_str_sk_num() -> (Vec, Vec) { + ( + vec![ + KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "sk".to_owned(), + key_type: KeyType::Range, + }, + ], + vec![ + AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "sk".to_owned(), + attribute_type: ScalarAttributeType::N, + }, + ], + ) + } + + #[test] + fn item_to_document_rejects_numeric_sort_key_exceeding_decimal128() { + let (schema, attrs) = schema_pk_str_sk_num(); + // 35 significant digits — exceeds Decimal128's 34-digit precision. + let over_precision = "1".to_owned() + &"2".repeat(34); + assert_eq!( + over_precision + .chars() + .filter(|c| c.is_ascii_digit()) + .count(), + 35 + ); + + let mut item = Item::new(); + item.insert("pk".to_owned(), AttributeValue::S("x".to_owned())); + item.insert("sk".to_owned(), AttributeValue::N(over_precision.clone())); + + let err = item_to_document(&item, &schema, &attrs).unwrap_err(); + match err { + StorageError::Validation(msg) => { + assert!(msg.contains(&over_precision)); + assert!(msg.contains("Decimal128")); + } + other => panic!("expected Validation error, got {other:?}"), + } + } + + #[test] + fn item_to_document_accepts_numeric_sort_key_at_decimal128_boundary() { + let (schema, attrs) = schema_pk_str_sk_num(); + // 34 significant digits — at the Decimal128 boundary. + let at_boundary = "1".repeat(34); + + let mut item = Item::new(); + item.insert("pk".to_owned(), AttributeValue::S("x".to_owned())); + item.insert("sk".to_owned(), AttributeValue::N(at_boundary)); + + assert!(item_to_document(&item, &schema, &attrs).is_ok()); + } + + #[test] + fn pk_filter_rejects_numeric_sort_key_exceeding_decimal128() { + let (schema, attrs) = schema_pk_str_sk_num(); + let over_precision = "1".to_owned() + &"2".repeat(34); + let mut key = Item::new(); + key.insert("pk".to_owned(), AttributeValue::S("x".to_owned())); + key.insert("sk".to_owned(), AttributeValue::N(over_precision)); + + let err = pk_filter(&key, &schema, &attrs).unwrap_err(); + assert!(matches!(err, StorageError::Validation(_))); + } + + #[test] + fn composite_id_disambiguates_delimiter_in_pk_or_sk() { + // Two items whose naive "{pk}#{sk}" strings would collide must + // produce distinct netstring-encoded _ids. + let a = composite_id("a#b", "c"); + let b = composite_id("a", "b#c"); + assert_ne!( + a, b, + "composite _id must not collide on delimiter-containing keys" + ); + } + + #[test] + fn composite_id_stable_on_normal_inputs() { + // Reasonable inputs still round-trip through netstring cleanly. + assert_eq!( + composite_id("user1", "2024-01-01"), + "5:user1,10:2024-01-01," + ); + assert_eq!(composite_id("", "sk"), "0:,2:sk,"); + assert_eq!(composite_id("pk", ""), "2:pk,0:,"); + } + + #[test] + fn composite_id_is_written_by_item_to_document() { + let (schema, attrs) = schema_pk_str_sk_num(); + let mut item = Item::new(); + item.insert("pk".to_owned(), AttributeValue::S("user1".to_owned())); + item.insert("sk".to_owned(), AttributeValue::N("42".to_owned())); + + let doc = item_to_document(&item, &schema, &attrs).unwrap(); + let id = doc.get_str("_id").unwrap(); + assert!( + id.starts_with("5:user1,"), + "expected netstring-encoded _id, got {id:?}" + ); + } + + // ── index_document / index_entry_filter ───────────────────────── + + fn base_schema_composite() -> (Vec, Vec) { + ( + vec![ + KeySchemaElement { + attribute_name: "customer_id".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "order_id".to_owned(), + key_type: KeyType::Range, + }, + ], + vec![ + AttributeDefinition { + attribute_name: "customer_id".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "order_id".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "status".to_owned(), + attribute_type: ScalarAttributeType::S, + }, + AttributeDefinition { + attribute_name: "priority".to_owned(), + attribute_type: ScalarAttributeType::N, + }, + ], + ) + } + + fn gsi_schema_hash_range() -> Vec { + vec![ + KeySchemaElement { + attribute_name: "status".to_owned(), + key_type: KeyType::Hash, + }, + KeySchemaElement { + attribute_name: "priority".to_owned(), + key_type: KeyType::Range, + }, + ] + } + + fn item_with(customer_id: &str, order_id: &str, status: &str, priority: &str) -> Item { + let mut item = Item::new(); + item.insert( + "customer_id".to_owned(), + AttributeValue::S(customer_id.to_owned()), + ); + item.insert( + "order_id".to_owned(), + AttributeValue::S(order_id.to_owned()), + ); + item.insert("status".to_owned(), AttributeValue::S(status.to_owned())); + item.insert( + "priority".to_owned(), + AttributeValue::N(priority.to_owned()), + ); + item + } + + #[test] + fn index_document_encodes_base_keys() { + let (base_schema, attrs) = base_schema_composite(); + let idx_schema = gsi_schema_hash_range(); + let item = item_with("cust1", "order1", "pending", "5"); + + let doc = index_document(&item, &idx_schema, &base_schema, &attrs).unwrap(); + assert_eq!(doc.get_str("pk").unwrap(), "pending"); + assert_eq!(doc.get_str("base_pk").unwrap(), "cust1"); + // Index sk (N) is Decimal128, base sk (S) is a plain string. + assert!(doc.get("sk_n").is_some(), "expected sk_n field"); + assert_eq!(doc.get_str("base_sk_s").unwrap(), "order1"); + } + + #[test] + fn index_document_id_disambiguates_duplicate_index_keys() { + // Two base items sharing (status, priority) but different base keys + // must produce distinct index _ids. Without base keys in _id both + // upserts would write to the same document — the D-C1 data-loss bug. + let (base_schema, attrs) = base_schema_composite(); + let idx_schema = gsi_schema_hash_range(); + + let a = item_with("custA", "orderA", "pending", "5"); + let b = item_with("custB", "orderB", "pending", "5"); + + let da = index_document(&a, &idx_schema, &base_schema, &attrs).unwrap(); + let db = index_document(&b, &idx_schema, &base_schema, &attrs).unwrap(); + assert_ne!(da.get_str("_id").unwrap(), db.get_str("_id").unwrap()); + } + + #[test] + fn index_entry_filter_matches_own_document() { + // The filter built for a projected item must select exactly that + // item's index document — same _id, base_pk, and base_sk fields. + let (base_schema, attrs) = base_schema_composite(); + let idx_schema = gsi_schema_hash_range(); + let item = item_with("cust1", "order1", "pending", "5"); + + let doc = index_document(&item, &idx_schema, &base_schema, &attrs).unwrap(); + let filter = index_entry_filter(&item, &idx_schema, &base_schema, &attrs).unwrap(); + // Every filter field must appear in the doc with the same value. + for (k, v) in filter.iter() { + let actual = doc.get(k).expect("filter field missing on doc"); + assert_eq!(v, actual, "filter field {k} mismatch"); + } + } + + #[test] + fn index_document_supports_hash_only_gsi_on_composite_base() { + // R-2 shape: hash-only GSI on a composite base table. The doc must + // still carry base_pk and base_sk so pagination can tie-break. + let (base_schema, attrs) = base_schema_composite(); + let idx_schema = vec![KeySchemaElement { + attribute_name: "status".to_owned(), + key_type: KeyType::Hash, + }]; + let item = item_with("cust1", "order1", "pending", "5"); + + let doc = index_document(&item, &idx_schema, &base_schema, &attrs).unwrap(); + assert_eq!(doc.get_str("pk").unwrap(), "pending"); + assert!( + doc.get("sk_s").is_none() && doc.get("sk_n").is_none() && doc.get("sk_b").is_none() + ); + assert_eq!(doc.get_str("base_pk").unwrap(), "cust1"); + assert_eq!(doc.get_str("base_sk_s").unwrap(), "order1"); + } + + #[test] + fn binary_sk_to_hex_preserves_ddb_byte_order() { + // DynamoDB compares binary sort keys as unsigned lex bytes. + // The stored hex-string form must preserve that ordering under + // MongoDB's default lexicographic string comparison — verify + // both same-length and cross-length cases. + let a = binary_sk_to_hex(&[0x01, 0xff]); + let b = binary_sk_to_hex(&[0x02]); + // DDB: [0x01, 0xff] < [0x02]. Hex: "01ff" < "02". + assert!(a < b, "{a} < {b}"); + + // Shorter-prefix rule: [0x01] < [0x01, 0x00] in DDB. + let a = binary_sk_to_hex(&[0x01]); + let b = binary_sk_to_hex(&[0x01, 0x00]); + assert!(a < b); + + // Same first byte, longer runner in DDB. + let a = binary_sk_to_hex(&[0x01, 0x00, 0x00]); + let b = binary_sk_to_hex(&[0x02]); + assert!(a < b); + + // Empty is the smallest. + let empty = binary_sk_to_hex(&[]); + let single = binary_sk_to_hex(&[0x00]); + assert!(empty < single); + } +} diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs new file mode 100644 index 00000000..074dd7cd --- /dev/null +++ b/crates/storage-mongodb/src/data_engine.rs @@ -0,0 +1,3565 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `DataEngine` trait implementation for `MongoEngine`. + +use bson::{Document, doc}; +use futures::future::BoxFuture; +use mongodb::options::{FindOneAndReplaceOptions, ReturnDocument}; + +use extenddb_core::expression::{ + self, Expr, ExpressionMaps, KeyCondition, PathElement, SortKeyCondition, UpdateAction, + resolve_name_ref, +}; +use extenddb_core::types::{ + AttributeValue, Item, KeySchemaElement, ReturnValuesOnConditionCheckFailure, + ScalarAttributeType, StreamEventName, StreamRecord, StreamRecordData, TableKeyInfo, + extract_key, item_size_bytes, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{ + composite_pk_to_text, encode_netstring_composite, pk_to_text, sk_info, +}; +use extenddb_storage::{ + DataEngine, IdempotencyKey, ItemPairResult, QueryResult, StreamCapture, TransactGetOp, + TransactWriteOp, +}; + +use crate::MongoEngine; +use crate::condition::condition_to_filter; +use crate::data::{ + binary_sk_to_hex, composite_id, data_collection_name, document_to_item, index_document, + index_entry_filter, item_to_document, pk_filter, sk_field_name, sk_suffix, +}; +use crate::pushdown::{Pushable, is_pushable}; + +use extenddb_core::types::{Projection, ProjectionType}; + +impl DataEngine for MongoEngine { + fn put_item( + &self, + key_info: &TableKeyInfo, + item: Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, Result, StorageError>> { + let key_info = key_info.clone(); + let item = item.clone(); + let condition = condition.cloned(); + let maps = maps.clone(); + let stream = stream.cloned(); + Box::pin(async move { + self.put_item_impl( + &key_info, + item, + return_old, + condition.as_ref(), + &maps, + stream.as_ref(), + ) + .await + }) + } + + fn get_item( + &self, + key_info: &TableKeyInfo, + key: &Item, + ) -> BoxFuture<'_, Result, StorageError>> { + let key_info = key_info.clone(); + let key = key.clone(); + Box::pin(async move { self.get_item_impl(&key_info, &key).await }) + } + + fn delete_item( + &self, + key_info: &TableKeyInfo, + key: &Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, Result, StorageError>> { + let key_info = key_info.clone(); + let key = key.clone(); + let condition = condition.cloned(); + let maps = maps.clone(); + let stream = stream.cloned(); + Box::pin(async move { + self.delete_item_impl( + &key_info, + &key, + return_old, + condition.as_ref(), + &maps, + stream.as_ref(), + ) + .await + }) + } + + fn update_item( + &self, + key_info: &TableKeyInfo, + key: &Item, + actions: &[UpdateAction], + return_old: bool, + return_new: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> BoxFuture<'_, ItemPairResult> { + let key_info = key_info.clone(); + let key = key.clone(); + let actions = actions.to_vec(); + let condition = condition.cloned(); + let maps = maps.clone(); + let stream = stream.cloned(); + Box::pin(async move { + self.update_item_impl( + &key_info, + &key, + &actions, + return_old, + return_new, + condition.as_ref(), + &maps, + stream.as_ref(), + ) + .await + }) + } + + fn query( + &self, + key_info: &TableKeyInfo, + key_condition: &KeyCondition, + maps: &ExpressionMaps, + forward: bool, + limit: Option, + exclusive_start_key: Option<&Item>, + index_name: Option<&str>, + ) -> BoxFuture<'_, QueryResult> { + let key_info = key_info.clone(); + let key_condition = key_condition.clone(); + let maps = maps.clone(); + let exclusive_start_key = exclusive_start_key.cloned(); + let index_name = index_name.map(std::string::ToString::to_string); + Box::pin(async move { + self.query_impl( + &key_info, + &key_condition, + &maps, + forward, + limit, + exclusive_start_key.as_ref(), + index_name.as_deref(), + ) + .await + }) + } + + fn scan( + &self, + key_info: &TableKeyInfo, + limit: Option, + exclusive_start_key: Option<&Item>, + segment: Option, + total_segments: Option, + index_name: Option<&str>, + ) -> BoxFuture<'_, QueryResult> { + let key_info = key_info.clone(); + let exclusive_start_key = exclusive_start_key.cloned(); + let index_name = index_name.map(std::string::ToString::to_string); + Box::pin(async move { + self.scan_impl( + &key_info, + limit, + exclusive_start_key.as_ref(), + segment, + total_segments, + index_name.as_deref(), + ) + .await + }) + } + + fn transact_get_items( + &self, + ops: &[TransactGetOp<'_>], + ) -> BoxFuture<'_, Result>, StorageError>> { + let ops_data: Vec<_> = ops + .iter() + .map(|op| (op.key_info.clone(), op.key.clone())) + .collect(); + Box::pin(async move { self.transact_get_items_impl(&ops_data).await }) + } + + fn transact_write_items( + &self, + ops: &[TransactWriteOp<'_>], + idempotency: Option>, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let ops_owned: Vec<_> = ops.iter().map(clone_transact_write_op).collect(); + let idempotency_owned = idempotency.map(|k| { + ( + k.account_id.to_owned(), + k.token.to_owned(), + k.fingerprint.to_owned(), + ) + }); + Box::pin(async move { + let idem_ref = idempotency_owned.as_ref().map(|(a, t, f)| IdempotencyKey { + account_id: a.as_str(), + token: t.as_str(), + fingerprint: f.as_str(), + }); + self.transact_write_items_impl(&ops_owned, idem_ref).await + }) + } + + fn cleanup_expired_idempotency_tokens( + &self, + max_age_seconds: i64, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { + let coll = self.data_db.collection::("idempotency_tokens"); + let cutoff = time::OffsetDateTime::now_utc() + - std::time::Duration::from_secs(max_age_seconds as u64); + let cutoff_bson = mongodb::bson::DateTime::from_millis(cutoff.unix_timestamp() * 1000); + let result = coll + .delete_many(doc! { "created_at": { "$lt": cutoff_bson } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(result.deleted_count) + }) + } +} + +impl MongoEngine { + async fn put_item_impl( + &self, + key_info: &TableKeyInfo, + item: Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> Result, StorageError> { + // Up-front index-key validation — must run before any write + // work so the caller sees a top-level ValidationException on + // wrong-type or empty index-key attributes (D-M10, RFC-0003 + // §2.3). + self.validate_index_keys_for_item(key_info, &item).await?; + + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + + let key_filter = pk_filter(&item, &key_info.key_schema, &key_info.attribute_definitions)?; + + // Sessionless fast path for unconditional PutItem on a plain + // table (no cond, no stream, no GSI). DDB's contract is + // last-writer-wins with no client-visible conflict error + // (RFC-0003 §4.1). Wrapping this in a snapshot transaction + // would convert same-key contention into WriteConflict + // aborts that eventually surface as `Internal` — a wire- + // visible error DDB never emits. Rely on WiredTiger's + // single-document atomicity instead. Two concurrent writes + // serialize at the storage engine level; one wins the last- + // writer-wins race and the other's version is overwritten. + // No txn, no retry loop, no possible 500 from contention. + if condition.is_none() + && stream.is_none() + && self.gsi_cache_get_fresh(&key_info.table_id) == Some(false) + { + let new_doc = + item_to_document(&item, &key_info.key_schema, &key_info.attribute_definitions)?; + let opts = FindOneAndReplaceOptions::builder() + .upsert(true) + .return_document(ReturnDocument::Before) + .build(); + let old_doc = coll + .find_one_and_replace(key_filter, new_doc) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let old_item = old_doc.as_ref().map(document_to_item).transpose()?; + return Ok(if return_old { old_item } else { None }); + } + + let mut session = self + .client + .start_session() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let tx_options = mongodb::options::TransactionOptions::builder() + .read_concern(mongodb::options::ReadConcern::snapshot()) + .write_concern( + mongodb::options::WriteConcern::builder() + .w(mongodb::options::Acknowledgment::Majority) + .build(), + ) + .build(); + + for attempt in 0..TRANSIENT_RETRY_ATTEMPTS { + let new_doc = + item_to_document(&item, &key_info.key_schema, &key_info.attribute_definitions)?; + session + .start_transaction() + .with_options(tx_options.clone()) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let attempt_res: Result, TxErr> = async { + let old_item: Option; + + if let Some(cond) = condition { + let existing_doc = coll + .find_one(key_filter.clone()) + .session(&mut session) + .await + .map_err(TxErr::from)?; + + if let Some(ref existing) = existing_doc { + let existing_item = document_to_item(existing)?; + let passed = expression::evaluate_condition(cond, &existing_item, maps) + .map_err(|e| TxErr::Fatal(StorageError::Validation(e.to_string())))?; + if !passed { + return Err(TxErr::Fatal(StorageError::ConditionFailed(Some( + existing_item, + )))); + } + let opts = FindOneAndReplaceOptions::builder() + .return_document(ReturnDocument::Before) + .build(); + let old_doc = coll + .find_one_and_replace(key_filter.clone(), new_doc) + .with_options(opts) + .session(&mut session) + .await + .map_err(TxErr::from)?; + old_item = old_doc.as_ref().map(document_to_item).transpose()?; + } else { + let empty = std::collections::BTreeMap::new(); + let passed = expression::evaluate_condition(cond, &empty, maps) + .map_err(|e| TxErr::Fatal(StorageError::Validation(e.to_string())))?; + if !passed { + return Err(TxErr::Fatal(StorageError::ConditionFailed(None))); + } + // Conditional insert: a concurrent inserter + // manifests either as E11000 (unique-index + // race) or as WriteConflict (snapshot-isolation + // race). Both are the runtime signature of a + // failed condition. Map dup-key to CCF with + // the winner's image; let WriteConflict fall + // through TxErr::Transient and retry — the + // retry will re-read and see the winner. + if let Err(e) = coll.insert_one(new_doc).session(&mut session).await { + if is_duplicate_key(&e) { + let _ = session.abort_transaction().await; + let winner = coll + .find_one(key_filter.clone()) + .await + .map_err(|e2| { + TxErr::Fatal(StorageError::Internal(e2.to_string())) + })? + .map(|d| document_to_item(&d)) + .transpose()?; + return Err(TxErr::Fatal(StorageError::ConditionFailed(winner))); + } + return Err(TxErr::from(e)); + } + old_item = None; + } + } else { + let opts = FindOneAndReplaceOptions::builder() + .upsert(true) + .return_document(ReturnDocument::Before) + .build(); + let old_doc = coll + .find_one_and_replace(key_filter.clone(), new_doc) + .with_options(opts) + .session(&mut session) + .await + .map_err(TxErr::from)?; + old_item = old_doc.as_ref().map(document_to_item).transpose()?; + } + + self.sync_indexes_in_session( + key_info, + old_item.as_ref(), + Some(&item), + &mut session, + ) + .await?; + + if let Some(capture) = stream { + self.write_stream_inline_in_session( + key_info, + capture, + old_item.as_ref(), + Some(&item), + &mut session, + ) + .await?; + } + + Ok(if return_old { old_item } else { None }) + } + .await; + + match attempt_res { + Ok(return_val) => match session.commit_transaction().await { + Ok(()) => return Ok(return_val), + Err(e) if is_transient_write_conflict(&e) => { + backoff_sleep(attempt).await; + continue; + } + Err(e) => return Err(StorageError::Internal(e.to_string())), + }, + Err(TxErr::Transient) => { + let _ = session.abort_transaction().await; + backoff_sleep(attempt).await; + continue; + } + Err(TxErr::Fatal(e)) => { + let _ = session.abort_transaction().await; + return Err(e); + } + } + } + + // Retry ceiling exhausted. RFC-0003 §4.3 requires + // `TransactionConflictException` when a single-item write can't + // serialize against concurrent activity — never a bare 500. + Err(StorageError::TransactionConflict( + "PutItem: too many concurrent write conflicts, giving up".to_owned(), + )) + } + + async fn get_item_impl( + &self, + key_info: &TableKeyInfo, + key: &Item, + ) -> Result, StorageError> { + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + + let filter = pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions)?; + let doc = coll + .find_one(filter) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + doc.as_ref().map(document_to_item).transpose() + } + + async fn delete_item_impl( + &self, + key_info: &TableKeyInfo, + key: &Item, + return_old: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> Result, StorageError> { + // Pushdown fast path: conditional delete on a no-stream / no-GSI + // table with a pushable condition. Collapses read-then-check-then- + // write inside a session to a single `find_one_and_delete` with + // the merged filter. See `crates/storage-mongodb/src/pushdown.rs`. + if let Some(cond) = condition + && stream.is_none() + && self.gsi_cache_get_fresh(&key_info.table_id) == Some(false) + && matches!(is_pushable(cond, maps), Pushable::Yes) + { + return self + .delete_item_pushdown(key_info, key, return_old, cond, maps) + .await; + } + + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + + let key_filter = pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions)?; + + // Sessionless fast path for unconditional DeleteItem on a plain + // table (no cond, no stream, no GSI). Same rationale as + // put_item_impl — DDB never surfaces contention on unconditional + // single-item deletes. RFC-0003 §4.1. + if condition.is_none() + && stream.is_none() + && self.gsi_cache_get_fresh(&key_info.table_id) == Some(false) + { + let old_doc = coll + .find_one_and_delete(key_filter) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let deleted_item = old_doc.as_ref().map(document_to_item).transpose()?; + return Ok(if return_old { deleted_item } else { None }); + } + + let mut session = self + .client + .start_session() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let tx_options = mongodb::options::TransactionOptions::builder() + .read_concern(mongodb::options::ReadConcern::snapshot()) + .write_concern( + mongodb::options::WriteConcern::builder() + .w(mongodb::options::Acknowledgment::Majority) + .build(), + ) + .build(); + + for attempt in 0..TRANSIENT_RETRY_ATTEMPTS { + session + .start_transaction() + .with_options(tx_options.clone()) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let attempt_res: Result, TxErr> = async { + let deleted_item: Option; + + if let Some(cond) = condition { + let existing_doc = coll + .find_one(key_filter.clone()) + .session(&mut session) + .await + .map_err(TxErr::from)?; + + if let Some(ref existing) = existing_doc { + let existing_item = document_to_item(existing)?; + let passed = expression::evaluate_condition(cond, &existing_item, maps) + .map_err(|e| TxErr::Fatal(StorageError::Validation(e.to_string())))?; + if !passed { + return Err(TxErr::Fatal(StorageError::ConditionFailed(Some( + existing_item, + )))); + } + coll.delete_one(key_filter.clone()) + .session(&mut session) + .await + .map_err(TxErr::from)?; + deleted_item = Some(existing_item); + } else { + let empty = std::collections::BTreeMap::new(); + let passed = expression::evaluate_condition(cond, &empty, maps) + .map_err(|e| TxErr::Fatal(StorageError::Validation(e.to_string())))?; + if !passed { + return Err(TxErr::Fatal(StorageError::ConditionFailed(None))); + } + deleted_item = None; + } + } else { + let old_doc = coll + .find_one_and_delete(key_filter.clone()) + .session(&mut session) + .await + .map_err(TxErr::from)?; + deleted_item = old_doc.as_ref().map(document_to_item).transpose()?; + } + + if deleted_item.is_some() { + self.sync_indexes_in_session( + key_info, + deleted_item.as_ref(), + None, + &mut session, + ) + .await?; + } + + if let Some(capture) = stream { + self.write_stream_inline_in_session( + key_info, + capture, + deleted_item.as_ref(), + None, + &mut session, + ) + .await?; + } + + Ok(if return_old { deleted_item } else { None }) + } + .await; + + match attempt_res { + Ok(return_val) => match session.commit_transaction().await { + Ok(()) => return Ok(return_val), + Err(e) if is_transient_write_conflict(&e) => { + backoff_sleep(attempt).await; + continue; + } + Err(e) => return Err(StorageError::Internal(e.to_string())), + }, + Err(TxErr::Transient) => { + let _ = session.abort_transaction().await; + backoff_sleep(attempt).await; + continue; + } + Err(TxErr::Fatal(e)) => { + let _ = session.abort_transaction().await; + return Err(e); + } + } + } + + Err(StorageError::TransactionConflict( + "DeleteItem: too many concurrent write conflicts, giving up".to_owned(), + )) + } + + #[allow(clippy::too_many_arguments)] + async fn update_item_impl( + &self, + key_info: &TableKeyInfo, + key: &Item, + actions: &[UpdateAction], + return_old: bool, + return_new: bool, + condition: Option<&Expr>, + maps: &ExpressionMaps, + stream: Option<&StreamCapture>, + ) -> Result<(Option, Option), StorageError> { + // Pushdown fast path (A5): conditional update on a no-stream / + // no-GSI table with a pushable condition. Skips the session/ + // transaction overhead. See `crates/storage-mongodb/src/pushdown.rs`. + if let Some(cond) = condition + && stream.is_none() + && self.gsi_cache_get_fresh(&key_info.table_id) == Some(false) + && matches!(is_pushable(cond, maps), Pushable::Yes) + { + match self + .update_item_pushdown(key_info, key, actions, return_old, return_new, cond, maps) + .await + { + Ok(pair) => return Ok(pair), + Err(StorageError::Internal(msg)) if msg.contains("raced by concurrent writer") => { + // Fall through to session-scoped path which + // has a proper retry loop. + } + Err(other) => return Err(other), + } + } + + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + + let key_filter = pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions)?; + + // Fast path: use native MongoDB atomic operators when possible. + // This avoids transactions and retries for simple unconditional + // updates. Gated on the table having no GSIs — the fast path + // does not read the pre-image, so it has no way to compute the + // GSI-key delta and would leave stale index entries when an + // indexed attribute is $set to a new value or $unset (RFC-0003 + // §2.2). The GSI cache lets us avoid a catalog query in the + // common (no-GSI) case; when the cache is stale-or-unknown + // we fall through to the slow path which is authoritative. + if condition.is_none() + && !return_old + && stream.is_none() + && self.gsi_cache_get_fresh(&key_info.table_id) == Some(false) + && let Some(mongo_update) = self.try_build_native_update(actions, maps) + { + let (fast_filter, took_fast) = match mongo_update { + NativeUpdate::Doc(d) => { + let opts = mongodb::options::FindOneAndUpdateOptions::builder() + .upsert(true) + .return_document(ReturnDocument::After) + .build(); + let result = coll + .find_one_and_update(key_filter.clone(), d) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + (Some(result), true) + } + NativeUpdate::Pipeline { + type_guard, + pipeline, + } => { + // Compose the key filter with the type guard. If + // the doc exists but fails the guard, findAndModify + // returns None and we fall through to the slow + // path (which raises ValidationException). + // Upsert is disabled here because a missing-doc + // "no match" is indistinguishable from a + // type-mismatch "no match"; the slow path handles + // both correctly. + let combined_filter = if let Some(guard) = type_guard { + doc! { "$and": [key_filter.clone(), guard] } + } else { + key_filter.clone() + }; + let opts = mongodb::options::FindOneAndUpdateOptions::builder() + .upsert(false) + .return_document(ReturnDocument::After) + .build(); + let result = coll + .find_one_and_update(combined_filter, pipeline) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if result.is_none() { + // Either the doc doesn't exist yet (need to + // upsert with proper type handling) or the + // guard rejected it. Fall through. + (None, false) + } else { + (Some(result), true) + } + } + }; + + if took_fast { + let result_doc = fast_filter.and_then(|d| d); + let new_item = if return_new { + result_doc.as_ref().map(document_to_item).transpose()? + } else { + None + }; + return Ok((None, new_item)); + } + } + + let mut session = self + .client + .start_session() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let tx_options = mongodb::options::TransactionOptions::builder() + .read_concern(mongodb::options::ReadConcern::snapshot()) + .write_concern( + mongodb::options::WriteConcern::builder() + .w(mongodb::options::Acknowledgment::Majority) + .build(), + ) + .build(); + + for attempt in 0..TRANSIENT_RETRY_ATTEMPTS { + session + .start_transaction() + .with_options(tx_options.clone()) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Sentinel returned by the attempt body to signal "the + // OCC version guard didn't match; retry from a fresh + // read." Distinct from TxErr::Transient because it isn't + // a mongo-side conflict — the whole snapshot succeeded, + // we just lost the CAS race. + struct StaleVersion; + #[allow(clippy::large_enum_variant)] + enum AttemptOk { + Committed(Option, Option), + Stale, + } + + let attempt_res: Result = async { + let existing_doc = coll + .find_one(key_filter.clone()) + .session(&mut session) + .await + .map_err(TxErr::from)?; + + let current_version = existing_doc + .as_ref() + .and_then(|d| d.get_i64("_v").ok()) + .unwrap_or(0); + + let existing_item = if let Some(doc) = existing_doc.as_ref() { + document_to_item(doc)? + } else { + key.clone() + }; + + if let Some(cond) = condition { + let eval_item = if existing_doc.is_some() { + &existing_item + } else { + &std::collections::BTreeMap::new() + }; + let passed = expression::evaluate_condition(cond, eval_item, maps) + .map_err(|e| TxErr::Fatal(StorageError::Validation(e.to_string())))?; + if !passed { + return Err(TxErr::Fatal(StorageError::ConditionFailed( + if existing_doc.is_some() { + Some(existing_item.clone()) + } else { + None + }, + ))); + } + } + + let pre_image = existing_doc.as_ref().map(|_| existing_item.clone()); + let old_item_for_stream = if return_old || stream.is_some() { + pre_image.clone() + } else { + None + }; + + let mut new_item = existing_item; + expression::apply_update(actions, &mut new_item, maps) + .map_err(|e| TxErr::Fatal(StorageError::Validation(e.to_string())))?; + + // Reject wrong-type or empty index-key attributes on + // the resulting item — D-M10, RFC-0003 §2.3. Same + // shape as put_item's up-front check, but the + // post-update item is what actually gets written. + self.validate_index_keys_for_item(key_info, &new_item) + .await + .map_err(TxErr::Fatal)?; + + let mut new_doc = item_to_document( + &new_item, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; + let new_version = current_version + 1; + new_doc.insert("_v", new_version); + + if existing_doc.is_some() { + let mut versioned_filter = key_filter.clone(); + if current_version == 0 { + versioned_filter.insert("_v", doc! { "$not": { "$gt": 0_i64 } }); + } else { + versioned_filter.insert("_v", current_version); + } + let result = coll + .replace_one(versioned_filter, new_doc) + .session(&mut session) + .await + .map_err(TxErr::from)?; + + if result.matched_count == 0 { + // OCC CAS lost: someone else bumped _v after + // our find_one. Not a mongo conflict — the + // snapshot txn succeeded, we just have a + // stale read. Signal retry. + let _ = StaleVersion; + return Ok(AttemptOk::Stale); + } + } else { + let opts = mongodb::options::ReplaceOptions::builder() + .upsert(true) + .build(); + coll.replace_one(key_filter.clone(), new_doc) + .with_options(opts) + .session(&mut session) + .await + .map_err(TxErr::from)?; + } + + self.sync_indexes_in_session( + key_info, + pre_image.as_ref(), + Some(&new_item), + &mut session, + ) + .await?; + + if let Some(capture) = stream { + self.write_stream_inline_in_session( + key_info, + capture, + old_item_for_stream.as_ref(), + Some(&new_item), + &mut session, + ) + .await?; + } + + let old_item_result = if return_old { + old_item_for_stream + } else { + None + }; + let new_item_result = if return_new { Some(new_item) } else { None }; + Ok(AttemptOk::Committed(old_item_result, new_item_result)) + } + .await; + + match attempt_res { + Ok(AttemptOk::Committed(old, new)) => match session.commit_transaction().await { + Ok(()) => return Ok((old, new)), + Err(e) if is_transient_write_conflict(&e) => { + backoff_sleep(attempt).await; + continue; + } + Err(e) => return Err(StorageError::Internal(e.to_string())), + }, + Ok(AttemptOk::Stale) => { + let _ = session.abort_transaction().await; + backoff_sleep(attempt).await; + continue; + } + Err(TxErr::Transient) => { + let _ = session.abort_transaction().await; + backoff_sleep(attempt).await; + continue; + } + Err(TxErr::Fatal(e)) => { + let _ = session.abort_transaction().await; + return Err(e); + } + } + } + + Err(StorageError::TransactionConflict( + "UpdateItem: too many concurrent write conflicts, giving up".to_owned(), + )) + } + + #[allow(clippy::too_many_arguments)] + async fn query_impl( + &self, + key_info: &TableKeyInfo, + key_condition: &KeyCondition, + maps: &ExpressionMaps, + forward: bool, + limit: Option, + exclusive_start_key: Option<&Item>, + index_name: Option<&str>, + ) -> Result<(Vec, Option), StorageError> { + use futures::TryStreamExt; + + // Determine collection and effective key schema for the query target + let (coll_name, effective_key_schema) = if let Some(idx_name) = index_name { + let idx_info = self + .index_info_by_table_id_impl(&key_info.table_id, idx_name) + .await?; + ( + data_collection_name(&idx_info.index_id), + idx_info.key_schema.clone(), + ) + } else { + ( + data_collection_name(&key_info.table_id), + key_info.key_schema.clone(), + ) + }; + let coll = self.data_db.collection::(&coll_name); + + // Build the query filter — handle multi-part HASH keys + let pk_text = if key_condition.extra_pk_conditions.is_empty() { + let pk_value = resolve_key_expr(&key_condition.pk_value, maps)?; + pk_to_text(&pk_value) + .map_err(|e| StorageError::Internal(e.to_string()))? + .into_owned() + } else { + let mut parts = Vec::with_capacity(1 + key_condition.extra_pk_conditions.len()); + let first_val = resolve_key_expr(&key_condition.pk_value, maps)?; + parts.push( + pk_to_text(&first_val) + .map_err(|e| StorageError::Internal(e.to_string()))? + .into_owned(), + ); + for (_, value) in &key_condition.extra_pk_conditions { + let val = resolve_key_expr(value, maps)?; + parts.push( + pk_to_text(&val) + .map_err(|e| StorageError::Internal(e.to_string()))? + .into_owned(), + ); + } + encode_netstring_composite(&parts) + }; + + let mut filter = doc! { "pk": &pk_text }; + + // Determine sort key field using effective key schema + let sk_field = sk_field_name(&effective_key_schema, &key_info.attribute_definitions); + + // Apply sort key condition + if let Some(ref sk_cond) = key_condition.sk_condition + && let Some(sk_f) = sk_field + { + let sk_filter = build_sk_filter(sk_cond, sk_f, maps)?; + for (k, v) in sk_filter { + filter.insert(k, v); + } + } + + // Apply exclusive_start_key pagination. + // + // For a base-table query the cursor is a single sort-key comparison: + // base-table items are uniquely keyed by (pk, sk), so `sk > cursor` + // is unambiguous. + // + // For an index query the cursor is a compound tuple over + // (index_sk?, base_pk, base_sk?). Index-key values are non-unique — + // duplicates fall through to the base-key tie-breaker. Express as + // a lexicographic `$or` of the shape + // (a > A) OR (a == A AND b > B) OR (a == A AND b == B AND c > C) + // (with `<` when reverse). RFC-0003 §2.6. + let is_index = index_name.is_some(); + if let Some(start_key) = exclusive_start_key { + let cmp_gt = if forward { "$gt" } else { "$lt" }; + if is_index { + let idx_sk_pair = + match sk_info(&effective_key_schema, &key_info.attribute_definitions) { + Some((sk_name, sk_type)) => start_key + .get(sk_name) + .map(|v| sk_to_bson(v, sk_type)) + .transpose()? + .map(|b| (sk_field.expect("sk_field present when sk_info is Some"), b)), + None => None, + }; + let base_pk_bson: Option = { + // Build the base_pk text the same way the write path + // does — composite_pk_to_text on the base key schema. + // If the start_key is malformed we skip pagination + // (result is a query that may return duplicates). + let text = composite_pk_to_text(start_key, &key_info.base_key_schema).ok(); + text.map(bson::Bson::String) + }; + let base_sk_pair = + match sk_info(&key_info.base_key_schema, &key_info.attribute_definitions) { + Some((sk_name, sk_type)) => start_key + .get(sk_name) + .map(|v| sk_to_bson(v, sk_type)) + .transpose()? + .map(|b| (format!("base_sk_{}", sk_suffix(sk_type)), b)), + None => None, + }; + + let mut or_clauses: Vec = Vec::new(); + if let Some((sk_f, sk_bson)) = idx_sk_pair.clone() { + or_clauses.push(doc! { sk_f: { cmp_gt: sk_bson } }); + } + if let Some(bp) = base_pk_bson.clone() { + let mut clause = Document::new(); + if let Some((sk_f, sk_bson)) = idx_sk_pair.clone() { + clause.insert(sk_f, sk_bson); + } + clause.insert("base_pk", doc! { cmp_gt: bp }); + or_clauses.push(clause); + } + if let (Some(bp), Some((base_sk_f, base_sk_bson))) = + (base_pk_bson, base_sk_pair.clone()) + { + let mut clause = Document::new(); + if let Some((sk_f, sk_bson)) = idx_sk_pair { + clause.insert(sk_f, sk_bson); + } + clause.insert("base_pk", bp); + clause.insert(base_sk_f, doc! { cmp_gt: base_sk_bson }); + or_clauses.push(clause); + } + + if !or_clauses.is_empty() { + // Merge with any existing $or (unlikely — sk_condition + // uses ranged operators, not $or) by wrapping in $and. + if filter.contains_key("$or") { + let existing = filter.remove("$or").unwrap(); + filter.insert( + "$and", + bson::bson!([{ "$or": existing }, { "$or": or_clauses }]), + ); + } else { + filter.insert("$or", or_clauses); + } + } + } else if let (Some(sk_f), Some((sk_name, sk_type))) = ( + sk_field, + sk_info(&effective_key_schema, &key_info.attribute_definitions), + ) && let Some(sk_val) = start_key.get(sk_name) + { + let sk_bson = sk_to_bson(sk_val, sk_type)?; + // Merge the resume bound into any existing sort-key + // predicate rather than replacing it. Naive + // `filter.insert(sk_f, {$gt: cursor})` drops the + // caller's original range/prefix/eq bound and returns + // items outside it on page 2+ (RFC-0003 §7.2). + let cursor_bound = doc! { cmp_gt: sk_bson }; + match filter.remove(sk_f) { + None => { + filter.insert(sk_f, cursor_bound); + } + Some(bson::Bson::Document(mut existing)) => { + // Existing predicate already uses operators + // ($gte/$lte/$lt/...); merge ours into the + // same operator map. If the caller and the + // cursor share an operator (both $gt on a + // forward page whose caller filtered $gt), + // overwriting with the cursor is correct — + // the cursor's bound is always strictly + // beyond the caller's for that direction. + for (k, v) in cursor_bound { + existing.insert(k, v); + } + filter.insert(sk_f, existing); + } + Some(scalar) => { + // Caller's predicate was an equality + // (`sk = X`). Combine with the resume bound + // under $and — a scalar sk_f binding can't + // hold a $gt sibling. + let clauses = bson::bson!([ + { sk_f: scalar }, + { sk_f: cursor_bound }, + ]); + if let Some(existing_and) = filter.remove("$and") { + let mut combined = match existing_and { + bson::Bson::Array(a) => a, + other => vec![other], + }; + if let bson::Bson::Array(new) = clauses { + combined.extend(new); + } + filter.insert("$and", combined); + } else { + filter.insert("$and", clauses); + } + } + } + } + } + + // Build sort direction. For indexes the sort tuple is + // (index_sk?, base_pk, base_sk?) so pagination lands deterministic + // within a group of items sharing index keys. + let sort_direction = if forward { 1 } else { -1 }; + let sort_doc = if is_index { + let mut sd = Document::new(); + if let Some(sk_f) = sk_field { + sd.insert(sk_f, sort_direction); + } + sd.insert("base_pk", sort_direction); + if let Some((_, sk_type)) = + sk_info(&key_info.base_key_schema, &key_info.attribute_definitions) + { + sd.insert(format!("base_sk_{}", sk_suffix(sk_type)), sort_direction); + } + sd + } else if let Some(sk_f) = sk_field { + doc! { sk_f: sort_direction } + } else { + doc! { "pk": sort_direction } + }; + + // Apply limit (fetch one extra for pagination) + let fetch_limit = limit.map(|l| l + 1); + + let collation_opt = if sk_field == Some("sk_s") { + Some( + mongodb::options::Collation::builder() + .locale("simple".to_string()) + .build(), + ) + } else { + None + }; + + let opts = mongodb::options::FindOptions::builder() + .sort(sort_doc) + .limit(fetch_limit) + .collation(collation_opt) + .build(); + + let cursor = coll + .find(filter) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let docs: Vec = cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut items: Vec = docs + .iter() + .map(document_to_item) + .collect::, _>>()?; + + // Binary begins_with used to require a post-fetch pass because + // BSON Binary comparison is length-first and $gte/$lt-on-Binary + // dropped matches whenever the prefix was shorter than the stored + // value. Since D-M5 stores binary sort keys as hex strings, the + // $gte/$lt filter emitted by build_sk_filter is now authoritative + // and no post-fetch filtering is needed. RFC-0003 §1.4. + + // Handle pagination. For an index query, LEK carries both the + // index-key components and the base-key components so the next + // page's ExclusiveStartKey can resolve the compound cursor. + // RFC-0003 §7.2. + let last_evaluated_key = if let Some(l) = limit { + #[allow(clippy::cast_sign_loss)] + let l_usize = l as usize; + if items.len() > l_usize { + items.truncate(l_usize); + items.last().map(|item| { + if is_index { + let mut key = extract_key(item, &effective_key_schema); + let base_key = extract_key(item, &key_info.base_key_schema); + for (k, v) in base_key { + key.entry(k).or_insert(v); + } + key + } else { + extract_key(item, &key_info.key_schema) + } + }) + } else { + None + } + } else { + None + }; + + Ok((items, last_evaluated_key)) + } + + async fn scan_impl( + &self, + key_info: &TableKeyInfo, + limit: Option, + exclusive_start_key: Option<&Item>, + segment: Option, + total_segments: Option, + index_name: Option<&str>, + ) -> Result<(Vec, Option), StorageError> { + use futures::TryStreamExt; + + // The effective key schema for the collection under scan: index + // schema for index scans (where the collection's _id encodes index + // keys), base schema for base-table scans. + let (coll_name, effective_key_schema) = if let Some(idx_name) = index_name { + let idx_info = self + .index_info_by_table_id_impl(&key_info.table_id, idx_name) + .await?; + ( + data_collection_name(&idx_info.index_id), + idx_info.key_schema.clone(), + ) + } else { + ( + data_collection_name(&key_info.table_id), + key_info.key_schema.clone(), + ) + }; + let coll = self.data_db.collection::(&coll_name); + + let is_index = index_name.is_some(); + let mut filter = Document::new(); + + // Apply exclusive_start_key for pagination. Base tables use _id + // ordering (netstring-encoded); index scans use a compound cursor + // over (pk, sk?, base_pk, base_sk?) so items with duplicate index + // keys don't confuse pagination. RFC-0003 §7.2, §2.6. + if let Some(start_key) = exclusive_start_key { + if is_index { + let idx_pk_bson: Option = + composite_pk_to_text(start_key, &effective_key_schema) + .ok() + .map(bson::Bson::String); + let idx_sk_pair = + match sk_info(&effective_key_schema, &key_info.attribute_definitions) { + Some((sk_name, sk_type)) => start_key + .get(sk_name) + .map(|v| sk_to_bson(v, sk_type)) + .transpose()? + .map(|b| (format!("sk_{}", sk_suffix(sk_type)), b)), + None => None, + }; + let base_pk_bson: Option = + composite_pk_to_text(start_key, &key_info.base_key_schema) + .ok() + .map(bson::Bson::String); + let base_sk_pair = + match sk_info(&key_info.base_key_schema, &key_info.attribute_definitions) { + Some((sk_name, sk_type)) => start_key + .get(sk_name) + .map(|v| sk_to_bson(v, sk_type)) + .transpose()? + .map(|b| (format!("base_sk_{}", sk_suffix(sk_type)), b)), + None => None, + }; + + let mut or_clauses: Vec = Vec::new(); + if let Some(ip) = idx_pk_bson.clone() { + or_clauses.push(doc! { "pk": { "$gt": ip } }); + } + if let (Some(ip), Some((sk_f, sk_bson))) = + (idx_pk_bson.clone(), idx_sk_pair.clone()) + { + or_clauses.push(doc! { + "pk": ip, + sk_f: { "$gt": sk_bson }, + }); + } + if let (Some(ip), Some(bp)) = (idx_pk_bson.clone(), base_pk_bson.clone()) { + let mut clause = doc! { "pk": ip }; + if let Some((sk_f, sk_bson)) = idx_sk_pair.clone() { + clause.insert(sk_f, sk_bson); + } + clause.insert("base_pk", doc! { "$gt": bp }); + or_clauses.push(clause); + } + if let (Some(ip), Some(bp), Some((base_sk_f, base_sk_bson))) = + (idx_pk_bson, base_pk_bson, base_sk_pair) + { + let mut clause = doc! { "pk": ip }; + if let Some((sk_f, sk_bson)) = idx_sk_pair { + clause.insert(sk_f, sk_bson); + } + clause.insert("base_pk", bp); + clause.insert(base_sk_f, doc! { "$gt": base_sk_bson }); + or_clauses.push(clause); + } + + if !or_clauses.is_empty() { + filter.insert("$or", or_clauses); + } + } else { + // Base-table scan: unique (pk, sk) means _id > cursor. + let start_pk = composite_pk_to_text(start_key, &effective_key_schema)?; + if let Some((sk_name, _)) = + sk_info(&effective_key_schema, &key_info.attribute_definitions) + { + if let Some(sk_val) = start_key.get(sk_name) { + let sk_text = match sk_val { + AttributeValue::S(s) => s.clone(), + AttributeValue::N(n) => n.clone(), + AttributeValue::B(b) => { + use base64::Engine; + base64::engine::general_purpose::STANDARD.encode(b) + } + _ => return Err(StorageError::Internal("invalid sk type".to_string())), + }; + let start_id = composite_id(&start_pk, &sk_text); + filter.insert("_id", doc! { "$gt": start_id }); + } + } else { + filter.insert("_id", doc! { "$gt": &start_pk }); + } + } + } + + // Sort key. Index scans sort by (pk, sk?, base_pk, base_sk?) so + // pagination is well-defined across items sharing index keys. + // Base-table scans sort by _id (unique). + let sort_doc = if is_index { + let mut sd = doc! { "pk": 1 }; + if let Some((_, sk_type)) = + sk_info(&effective_key_schema, &key_info.attribute_definitions) + { + sd.insert(format!("sk_{}", sk_suffix(sk_type)), 1); + } + sd.insert("base_pk", 1); + if let Some((_, sk_type)) = + sk_info(&key_info.base_key_schema, &key_info.attribute_definitions) + { + sd.insert(format!("base_sk_{}", sk_suffix(sk_type)), 1); + } + sd + } else { + doc! { "_id": 1 } + }; + + // Lazy cursor iteration. The segment filter (CRC32 hash of pk + // mod total_segments) is applied per-item after fetching, so + // any hard server-side limit interacts badly with skew: with + // a modest hot-key concentration, a whole `(limit+1) * + // total_segments` window can land in one segment and leave + // the others empty, terminating the scan early with the + // remaining items silently dropped. RFC-0003 §7.3. + // + // Instead, stream the cursor and stop when either + // (a) we have `limit + 1` in-segment items (so we know we + // need a LEK for the next page), or + // (b) the cursor is exhausted. + // mongo batches under the hood (~101 docs per network trip), + // so this is efficient without a hard limit — we consume at + // most one extra network batch beyond what we return. + let opts = mongodb::options::FindOptions::builder() + .sort(sort_doc) + .build(); + + let mut cursor = coll + .find(filter) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut items: Vec = Vec::new(); + let target = limit.map(|l| { + #[allow(clippy::cast_sign_loss)] + let l = l as usize; + l + 1 + }); + + while let Some(doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let item = document_to_item(&doc)?; + + if let (Some(seg), Some(total)) = (segment, total_segments) { + let pk_text = composite_pk_to_text(&item, &key_info.key_schema)?; + let hash = crc32fast::hash(pk_text.as_bytes()); + #[allow(clippy::cast_sign_loss)] + let total_u = total as u32; + #[allow(clippy::cast_sign_loss)] + let seg_u = seg as u32; + if hash % total_u != seg_u { + continue; + } + } + + items.push(item); + + if let Some(t) = target + && items.len() >= t + { + break; + } + } + + // Handle pagination. For index scans, LEK includes both the + // index-key and base-key components. RFC-0003 §7.2. + let last_evaluated_key = if let Some(l) = limit { + #[allow(clippy::cast_sign_loss)] + let l_usize = l as usize; + if items.len() > l_usize { + items.truncate(l_usize); + items.last().map(|item| { + if is_index { + let mut key = extract_key(item, &effective_key_schema); + let base_key = extract_key(item, &key_info.base_key_schema); + for (k, v) in base_key { + key.entry(k).or_insert(v); + } + key + } else { + extract_key(item, &key_info.key_schema) + } + }) + } else { + None + } + } else { + None + }; + + Ok((items, last_evaluated_key)) + } + + // ── Native MongoDB Update (fast path) ───────────────────────────── + + /// Try to express the update as a native MongoDB atomic update. + /// + /// Returns: + /// - `Some(NativeUpdate::Doc(...))` for a plain operator update + /// (`$set`/`$unset`/`$inc`), served by + /// `find_one_and_update(filter, doc)`. + /// - `Some(NativeUpdate::Pipeline(...))` for numeric `ADD`, which + /// requires an aggregation-pipeline update (`$set` with computed + /// expressions) to convert the string-stored `.N` value to a + /// `Decimal128`, add the delta, and convert back — all + /// server-side. + /// - `None` on anything else — set-typed `ADD`, `DELETE`, + /// `list_append`, `if_not_exists`, arithmetic, multi-component + /// paths. Those fall through to the session-scoped + /// read-modify-write path. + /// + /// The pipeline form is what makes RFC-0003 §4.4 achievable + /// without a numeric shadow field: 50+ concurrent + /// `UpdateItem ADD counter :one` calls all apply cumulatively + /// because mongo serializes doc-scoped write locks around the + /// pipeline's read + compute + write, no OCC retry needed. + fn try_build_native_update( + &self, + actions: &[UpdateAction], + maps: &ExpressionMaps, + ) -> Option { + let mut set_doc = Document::new(); + let mut unset_doc = Document::new(); + let mut num_adds: Vec<(String, String)> = Vec::new(); + + for action in actions { + match action { + UpdateAction::Add { path, value } => { + if path.len() != 1 { + return None; + } + let raw_name = match &path[0] { + PathElement::Attribute(name) => name, + _ => return None, + }; + let attr_name = resolve_name_ref(raw_name, maps).ok()?.into_owned(); + let val = match value { + Expr::Placeholder(name) => maps.resolve_value(name).ok()?, + _ => return None, + }; + match val { + AttributeValue::N(n) => { + // Validate the delta parses as Decimal128 + // up-front so a bad number fails fast + // rather than mid-pipeline on mongo. + if n.parse::().is_err() { + return None; + } + num_adds.push((attr_name, n.clone())); + } + AttributeValue::SS(_) | AttributeValue::NS(_) | AttributeValue::BS(_) => { + // Set ADD — $addToSet would work in theory + // but our .SS/.NS/.BS storage keeps the + // values inside item_data..SS as an + // array. Not urgent enough to expand yet. + return None; + } + _ => return None, + } + } + UpdateAction::Delete { .. } => return None, + UpdateAction::Set { path, value } => { + if path.len() != 1 { + return None; + } + let raw_name = match &path[0] { + PathElement::Attribute(name) => name, + _ => return None, + }; + let attr_name = resolve_name_ref(raw_name, maps).ok()?; + let val = match value { + Expr::Placeholder(name) => maps.resolve_value(name).ok()?, + _ => return None, // complex expressions (if_not_exists, list_append, arithmetic) + }; + let field = format!("item_data.{attr_name}"); + let val_json = serde_json::to_value(val).ok()?; + let val_bson = bson::to_bson(&val_json).ok()?; + set_doc.insert(field, val_bson); + } + UpdateAction::Remove { path } => { + if path.len() != 1 { + return None; + } + let raw_name = match &path[0] { + PathElement::Attribute(name) => name, + _ => return None, + }; + let attr_name = resolve_name_ref(raw_name, maps).ok()?; + let field = format!("item_data.{attr_name}"); + unset_doc.insert(field, 1); + } + } + } + + if set_doc.is_empty() && unset_doc.is_empty() && num_adds.is_empty() { + return None; + } + + if !num_adds.is_empty() { + // Aggregation-pipeline stage. `$set` accepts computed + // expressions here (unlike an operator update's `$set`). + // Each numeric ADD is ` = toString(toDecimal(field + // or 0) + delta)`; `$unset` is expressed as ` = + // "$$REMOVE"`; SET actions are literal assignments. `_v` + // is bumped in the same stage. + let mut stage: Document = Document::new(); + for (k, v) in &set_doc { + stage.insert(k, v.clone()); + } + for k in unset_doc.keys() { + stage.insert(k, "$$REMOVE"); + } + // Guard: the fast path never reads the pre-image, so we + // can't detect an existing non-numeric attribute (e.g. + // ADD to a string). Require every ADD target to be + // absent or already hold `.N` — else return no match and + // let the caller fall back to the slow path, which reads + // the pre-image and returns a proper ValidationException. + let mut guard_clauses: Vec = Vec::with_capacity(num_adds.len()); + for (attr, delta_s) in &num_adds { + let field = format!("item_data.{attr}.N"); + let field_ref = format!("${field}"); + let attr_path = format!("item_data.{attr}"); + let delta_dec = delta_s + .parse::() + .expect("validated above"); + stage.insert( + &field, + doc! { + "$toString": { + "$add": [ + { "$toDecimal": { "$ifNull": [ &field_ref, "0" ] } }, + { "$toDecimal": bson::Bson::Decimal128(delta_dec) }, + ] + } + }, + ); + guard_clauses.push(doc! { + "$or": [ + { &attr_path: { "$exists": false } }, + { &field: { "$exists": true } }, + ] + }); + } + stage.insert( + "_v", + doc! { "$add": [ { "$ifNull": [ "$_v", 0_i64 ] }, 1_i64 ] }, + ); + let type_guard = if guard_clauses.is_empty() { + None + } else if guard_clauses.len() == 1 { + Some(guard_clauses.into_iter().next().unwrap()) + } else { + Some(doc! { "$and": guard_clauses }) + }; + return Some(NativeUpdate::Pipeline { + type_guard, + pipeline: vec![doc! { "$set": stage }], + }); + } + + let mut update = Document::new(); + if !set_doc.is_empty() { + update.insert("$set", set_doc); + } + if !unset_doc.is_empty() { + update.insert("$unset", unset_doc); + } + + // Bump `_v` on every native fast-path write. Without this a + // fast-path commit leaves `_v` at its previous value, and a + // slow-path update running concurrently against that same + // stale value can pass its versioned-filter guard and + // overwrite the fast-path write (lost update, RFC-0003 §4.4). + let mut inc_doc = Document::new(); + inc_doc.insert("_v", 1_i64); + update.insert("$inc", inc_doc); + + Some(NativeUpdate::Doc(update)) + } + + // ── GSI Sync ────────────────────────────────────────────────────── + + /// Fetch (index_name, key_schema) for every index on the table. + /// Used by up-front input validation so PutItem / UpdateItem + /// rejects wrong-type or empty index-key attributes with a + /// top-level ValidationException before doing any write work + /// (D-M10, matches postgres put_item.rs). + async fn fetch_index_key_schemas( + &self, + table_id: &str, + ) -> Result)>, StorageError> { + use futures::TryStreamExt; + + if let Some(false) = self.gsi_cache_get_fresh(table_id) { + return Ok(Vec::new()); + } + + let indexes_coll = self.catalog_db.collection::("indexes"); + let mut cursor = indexes_coll + .find(doc! { "_id.table_id": table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut out = Vec::new(); + while let Some(idx_doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let index_name = match idx_doc + .get_document("_id") + .and_then(|d| d.get_str("index_name")) + { + Ok(n) => n.to_string(), + Err(_) => continue, + }; + let key_schema: Vec = match idx_doc.get("key_schema") { + Some(ks) => bson::from_bson(ks.clone()).unwrap_or_default(), + None => continue, + }; + out.push((index_name, key_schema)); + } + Ok(out) + } + + /// Reject an item whose secondary-index key attributes have the + /// wrong scalar type or are empty. Called by put/update before + /// the transaction is opened — matches postgres semantics of + /// surfacing this as a top-level ValidationException rather than + /// letting sync_indexes silently drop the malformed index doc + /// (`data/mod.rs::index_document` skips typed sk fields on type + /// mismatch, leaving the row un-locatable for subsequent + /// deletes). RFC-0003 §2.3. + async fn validate_index_keys_for_item( + &self, + key_info: &TableKeyInfo, + item: &Item, + ) -> Result<(), StorageError> { + let idx_pairs = self.fetch_index_key_schemas(&key_info.table_id).await?; + if idx_pairs.is_empty() { + return Ok(()); + } + let refs: Vec> = idx_pairs + .iter() + .map(|(name, ks)| extenddb_core::validation::IndexKeyRef { + index_name: name.as_str(), + key_schema: ks.as_slice(), + }) + .collect(); + extenddb_core::validation::validate_index_keys(item, &refs, &key_info.attribute_definitions) + .map_err(|e| StorageError::Validation(e.to_string())) + } + + async fn sync_indexes_in_session( + &self, + key_info: &TableKeyInfo, + old_item: Option<&Item>, + new_item: Option<&Item>, + session: &mut mongodb::ClientSession, + ) -> Result<(), StorageError> { + if let Some(false) = self.gsi_cache_get_fresh(&key_info.table_id) { + return Ok(()); + } + + let indexes_coll = self.catalog_db.collection::("indexes"); + let mut cursor = indexes_coll + .find(doc! { "_id.table_id": &key_info.table_id }) + .session(&mut *session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut found_any = false; + while let Some(idx_doc) = cursor + .next(session) + .await + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))? + { + found_any = true; + let index_id = match idx_doc.get_str("index_id") { + Ok(id) => id.to_string(), + Err(_) => continue, + }; + let idx_key_schema: Vec = match idx_doc.get("key_schema") { + Some(ks) => bson::from_bson(ks.clone()).unwrap_or_default(), + None => continue, + }; + let projection: Projection = match idx_doc.get("projection") { + Some(p) => bson::from_bson(p.clone()).unwrap_or(Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }), + None => Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }, + }; + + let idx_coll_name = data_collection_name(&index_id); + let idx_coll = self.data_db.collection::(&idx_coll_name); + + if let Some(old) = old_item + && item_has_index_keys(old, &idx_key_schema) + { + let projected_old = + project_item(old, &idx_key_schema, &key_info.key_schema, &projection); + let old_filter = index_entry_filter( + &projected_old, + &idx_key_schema, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; + // Propagate the error rather than swallowing it — RFC-0003 + // §2.2 requires deleting the old entry when a write changes + // or removes a GSI key attribute, and RFC-0003 §9.1 forbids + // silent side-effect drops. A transient error here would + // leave the stale index row live under the old GSI-key + // value even though the base item no longer has it, and + // subsequent queries would return the stale projection + // forever. + idx_coll + .delete_one(old_filter) + .session(&mut *session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + if let Some(new) = new_item + && item_has_index_keys(new, &idx_key_schema) + { + let projected = + project_item(new, &idx_key_schema, &key_info.key_schema, &projection); + let idx_doc = index_document( + &projected, + &idx_key_schema, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; + let filter = index_entry_filter( + &projected, + &idx_key_schema, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; + let opts = mongodb::options::ReplaceOptions::builder() + .upsert(true) + .build(); + idx_coll + .replace_one(filter, idx_doc) + .with_options(opts) + .session(&mut *session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + } + + self.gsi_cache_set(&key_info.table_id, found_any); + Ok(()) + } + + async fn write_stream_inline_in_session( + &self, + key_info: &TableKeyInfo, + capture: &StreamCapture, + old_item: Option<&Item>, + new_item: Option<&Item>, + session: &mut mongodb::ClientSession, + ) -> Result<(), StorageError> { + use extenddb_core::types::StreamViewType; + + let source_item = new_item.or(old_item); + let Some(source) = source_item else { + return Ok(()); + }; + + let event = match (old_item, new_item) { + (None, Some(_)) => StreamEventName::Insert, + (Some(_), Some(_)) => StreamEventName::Modify, + (Some(_), None) => StreamEventName::Remove, + (None, None) => return Ok(()), + }; + + let keys: std::collections::BTreeMap = key_info + .key_schema + .iter() + .filter_map(|ks| { + source + .get(&ks.attribute_name) + .map(|v| (ks.attribute_name.clone(), v.clone())) + }) + .collect(); + + let new_image = match capture.view_type { + StreamViewType::NewImage | StreamViewType::NewAndOldImages => new_item.cloned(), + _ => None, + }; + let old_image = match capture.view_type { + StreamViewType::OldImage | StreamViewType::NewAndOldImages => old_item.cloned(), + _ => None, + }; + + let size = source_item.map_or(0, |i| i64::try_from(item_size_bytes(i)).unwrap_or(i64::MAX)); + + let pk_name = &key_info.key_schema[0].attribute_name; + let pk_str = source + .get(pk_name) + .map(|v| match v { + AttributeValue::S(s) => s.clone(), + AttributeValue::N(n) => n.clone(), + AttributeValue::B(b) => { + base64::Engine::encode(&base64::engine::general_purpose::STANDARD, b) + } + _ => String::new(), + }) + .unwrap_or_default(); + + // Both shard resolution and sequence-number assignment run inside + // the same session as the data write. This is what makes stream + // ordering safe under contention — see + // stream_engine::next_sequence_number_in_session for the full + // rationale. + let shard_id = self + .assign_shard_in_session( + &key_info.account_id, + &key_info.table_name, + &pk_str, + &mut *session, + ) + .await?; + let seq = self + .next_sequence_number_in_session(&shard_id, &mut *session) + .await?; + + let record = StreamRecord { + event_id: uuid::Uuid::new_v4().to_string(), + event_name: event, + event_version: "1.1".to_owned(), + event_source: "aws:dynamodb".to_owned(), + aws_region: capture.region.to_string(), + dynamodb: StreamRecordData { + approximate_creation_date_time: i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + ) + .unwrap_or(i64::MAX), + keys, + new_image, + old_image, + sequence_number: seq, + size_bytes: size, + stream_view_type: capture.view_type, + }, + user_identity: capture.user_identity.clone(), + }; + + let record_json = + serde_json::to_value(&record).map_err(|e| StorageError::Internal(e.to_string()))?; + let record_bson = + bson::to_bson(&record_json).map_err(|e| StorageError::Internal(e.to_string()))?; + + // key_info already carries table_id — no need to re-read the catalog + // just to resolve it, and re-reading inside the session against the + // tables collection would join to the counter/records write set + // needlessly. + let table_id = &key_info.table_id; + + let records_coll = self.data_db.collection::("stream_records"); + records_coll + .insert_one(doc! { + "sequence_number": &record.dynamodb.sequence_number, + "shard_id": &shard_id, + "table_id": table_id, + "event_name": crate::stream_engine::event_name_ddb_str(record.event_name), + "record_data": record_bson, + "created_at": mongodb::bson::DateTime::now(), + }) + .session(&mut *session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(()) + } + + // ── Transactions ────────────────────────────────────────────────── + + async fn transact_get_items_impl( + &self, + ops: &[(TableKeyInfo, Item)], + ) -> Result>, StorageError> { + use extenddb_core::types::CancellationReason; + use extenddb_core::validation; + + // Validate key types before starting transaction + let mut reasons: Vec = Vec::with_capacity(ops.len()); + let mut any_failed = false; + for (key_info, key) in ops { + match validation::validate_key_only( + key, + &key_info.key_schema, + &key_info.attribute_definitions, + ) { + Ok(()) => reasons.push(CancellationReason::none()), + Err(e) => { + any_failed = true; + reasons.push(CancellationReason::validation_error(e.to_string())); + } + } + } + if any_failed { + return Err(StorageError::TransactionCanceled(reasons)); + } + + // Use a MongoDB session with snapshot read concern for consistent reads + let mut session = self + .client + .start_session() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let tx_options = mongodb::options::TransactionOptions::builder() + .read_concern(mongodb::options::ReadConcern::snapshot()) + .build(); + + session + .start_transaction() + .with_options(tx_options) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut results = Vec::with_capacity(ops.len()); + for (key_info, key) in ops { + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + let filter = pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions)?; + let doc = coll + .find_one(filter) + .session(&mut session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let item = doc.as_ref().map(document_to_item).transpose()?; + results.push(item); + } + + session + .commit_transaction() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(results) + } + + async fn transact_write_items_impl( + &self, + ops: &[OwnedTransactWriteOp], + idempotency: Option>, + ) -> Result<(), StorageError> { + use extenddb_core::types::CancellationReason; + + // Start a MongoDB multi-document transaction + let mut session = self + .client + .start_session() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let tx_options = mongodb::options::TransactionOptions::builder() + .read_concern(mongodb::options::ReadConcern::snapshot()) + .write_concern( + mongodb::options::WriteConcern::builder() + .w(mongodb::options::Acknowledgment::Majority) + .build(), + ) + .build(); + + // Outcome of one attempt at running the whole idempotency check + // + op fan-out + commit. `Retry` means MongoDB aborted the txn + // as a transient conflict; the caller should re-run from the top. + enum AttemptOutcome { + Committed, + CanceledReasons(Vec), + Retry, + } + + // Rehydrate the `IdempotencyKey` per attempt from owned strings. + // The input struct holds `&str`s, so it cannot be moved across + // loop iterations. This keeps the retry loop lifetime-clean + // without asking upstream to change the trait signature. + let idem_owned = idempotency.map(|k| { + ( + k.account_id.to_owned(), + k.token.to_owned(), + k.fingerprint.to_owned(), + ) + }); + + for attempt in 0..TRANSIENT_RETRY_ATTEMPTS { + let idempotency = idem_owned.as_ref().map(|(a, t, f)| IdempotencyKey { + account_id: a.as_str(), + token: t.as_str(), + fingerprint: f.as_str(), + }); + session + .start_transaction() + .with_options(tx_options.clone()) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let outcome: Result = async { + // Check idempotency token, scoped to the caller's account + // so that identical tokens from different accounts never + // collide. + if let Some(key) = idempotency { + let idem_coll = self.data_db.collection::("idempotency_tokens"); + let existing = match idem_coll + .find_one(doc! { "account_id": key.account_id, "token": key.token }) + .session(&mut session) + .await + { + Ok(v) => v, + Err(e) if is_transient_write_conflict(&e) => { + return Ok(AttemptOutcome::Retry); + } + Err(e) => return Err(StorageError::Internal(e.to_string())), + }; + + // Filter out rows older than the DDB spec's 10-minute + // dedup window. Even with the TTL index set to 540s, + // MongoDB's TTL monitor runs on a ~60s cadence so a + // just-expired row can linger briefly. Treating a stale + // row as "not present" makes the read strictly correct + // regardless of monitor timing. See D-m4. + let existing = existing.filter(|doc| { + doc.get_datetime("created_at").is_ok_and(|dt| { + let age_ms = mongodb::bson::DateTime::now() + .timestamp_millis() + .saturating_sub(dt.timestamp_millis()); + age_ms < 600_000 + }) + }); + + if let Some(existing_doc) = existing { + let stored_fp = existing_doc.get_str("fingerprint").unwrap_or_default(); + return Err(if stored_fp == key.fingerprint { + StorageError::IdempotentReplay + } else { + StorageError::IdempotentMismatch + }); + } + + // Store the token. A unique index on (account_id, token) + // catches the case where a concurrent request under + // snapshot isolation didn't see our pre-check but raced + // us to the insert. On E11000, resolve the winner by + // re-reading outside the session — same replay/mismatch + // logic as the pre-check path. + let insert_res = idem_coll + .insert_one(doc! { + "account_id": key.account_id, + "token": key.token, + "fingerprint": key.fingerprint, + "created_at": mongodb::bson::DateTime::now(), + }) + .session(&mut session) + .await; + if let Err(e) = insert_res { + if is_duplicate_key(&e) { + let winner = idem_coll + .find_one(doc! { + "account_id": key.account_id, + "token": key.token, + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + // Same 10-min age filter as the pre-check — + // don't let a barely-expired row masquerade + // as a live token. D-m4. + let winner = winner.filter(|d| { + d.get_datetime("created_at").is_ok_and(|dt| { + let age_ms = mongodb::bson::DateTime::now() + .timestamp_millis() + .saturating_sub(dt.timestamp_millis()); + age_ms < 600_000 + }) + }); + // If the winner aged out between our insert + // and this follow-up read, the row will be + // TTL'd shortly and the request is not a + // real dup — retry so the next attempt + // inserts fresh. + if winner.is_none() { + return Ok(AttemptOutcome::Retry); + } + return Err( + match winner.as_ref().and_then(|d| d.get_str("fingerprint").ok()) { + Some(fp) if fp == key.fingerprint => { + StorageError::IdempotentReplay + } + _ => StorageError::IdempotentMismatch, + }, + ); + } + if is_transient_write_conflict(&e) { + return Ok(AttemptOutcome::Retry); + } + return Err(StorageError::Internal(e.to_string())); + } + } + + let mut reasons: Vec = Vec::with_capacity(ops.len()); + let mut any_failed = false; + + for op in ops { + match self + .execute_transact_write_op_in_session(op, &mut session) + .await + { + Ok(()) => reasons.push(CancellationReason::none()), + Err(TransactOpError::Cancel(r)) => { + any_failed = true; + reasons.push(r); + } + Err(TransactOpError::Transient) => { + return Ok(AttemptOutcome::Retry); + } + Err(TransactOpError::Storage(e)) => return Err(e), + } + } + + if any_failed { + return Ok(AttemptOutcome::CanceledReasons(reasons)); + } + Ok(AttemptOutcome::Committed) + } + .await; + + match outcome { + Ok(AttemptOutcome::Committed) => match session.commit_transaction().await { + Ok(()) => return Ok(()), + Err(e) if is_transient_write_conflict(&e) => { + backoff_sleep(attempt).await; + continue; + } + Err(e) => return Err(StorageError::Internal(e.to_string())), + }, + Ok(AttemptOutcome::CanceledReasons(reasons)) => { + let _ = session.abort_transaction().await; + return Err(StorageError::TransactionCanceled(reasons)); + } + Ok(AttemptOutcome::Retry) => { + let _ = session.abort_transaction().await; + backoff_sleep(attempt).await; + continue; + } + Err(e) => { + let _ = session.abort_transaction().await; + return Err(e); + } + } + } + + // Exhausted retries under sustained contention. Surface as a + // canceled transaction with a synthetic per-op TransactionConflict + // reason so wire consumers see the DDB-canonical error string + // instead of a bare HTTP 500. The engine maps StorageError:: + // TransactionCanceled to TransactionCanceledException; the + // reason codes are echoed back in the message. + let reasons = ops + .iter() + .map(|_| CancellationReason { + code: "TransactionConflict".to_owned(), + message: Some("Transaction is ongoing for the item".to_owned()), + item: None, + }) + .collect(); + Err(StorageError::TransactionCanceled(reasons)) + } + + async fn execute_transact_write_op_in_session( + &self, + op: &OwnedTransactWriteOp, + session: &mut mongodb::ClientSession, + ) -> Result<(), TransactOpError> { + use extenddb_core::types::CancellationReason; + use extenddb_core::validation; + + match op { + OwnedTransactWriteOp::Put { + key_info, + item, + condition, + maps, + return_values_on_ccf, + stream, + } => { + validation::validate_item_keys( + item, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + + // Index-key type/empty faults inside a transaction + // surface as per-item cancellation reasons (matches + // postgres data/transactions.rs). D-M10. + let idx_pairs = self + .fetch_index_key_schemas(&key_info.table_id) + .await + .map_err(TransactOpError::Storage)?; + if !idx_pairs.is_empty() { + let idx_refs: Vec> = idx_pairs + .iter() + .map(|(n, ks)| extenddb_core::validation::IndexKeyRef { + index_name: n.as_str(), + key_schema: ks.as_slice(), + }) + .collect(); + extenddb_core::validation::validate_index_keys( + item, + &idx_refs, + &key_info.attribute_definitions, + ) + .map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + } + + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + let key_filter = + pk_filter(item, &key_info.key_schema, &key_info.attribute_definitions) + .map_err(TransactOpError::Storage)?; + + // Always fetch the pre-image. Needed to (a) evaluate any + // condition against it, (b) let sync_indexes_in_session delete + // stale index entries when this write changes or removes a + // GSI key attribute, and (c) supply OldImage to any attached + // stream capture. + let existing_doc = coll + .find_one(key_filter.clone()) + .session(&mut *session) + .await + .map_err(TransactOpError::from)?; + + let existing_item = if let Some(doc) = existing_doc.as_ref() { + Some(document_to_item(doc).map_err(TransactOpError::Storage)?) + } else { + None + }; + + if let Some(cond) = condition { + let for_eval = existing_item.clone().unwrap_or_default(); + let passed = + expression::evaluate_condition(cond, &for_eval, maps).map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error( + e.to_string(), + )) + })?; + if !passed { + return Err(TransactOpError::Cancel( + CancellationReason::condition_check_failed_with_item(ccf_return_item( + *return_values_on_ccf, + existing_item.as_ref(), + )), + )); + } + } + + let new_doc = + item_to_document(item, &key_info.key_schema, &key_info.attribute_definitions) + .map_err(TransactOpError::Storage)?; + + let opts = mongodb::options::ReplaceOptions::builder() + .upsert(true) + .build(); + coll.replace_one(key_filter, new_doc) + .with_options(opts) + .session(&mut *session) + .await + .map_err(TransactOpError::from)?; + + // Propagate to secondary indexes and the stream within the + // same transaction session — otherwise a transactional write + // to a streams-enabled or GSI-bearing table would commit + // the base row while silently dropping its dependent side + // effects. + self.sync_indexes_in_session( + key_info, + existing_item.as_ref(), + Some(item), + &mut *session, + ) + .await + .map_err(TransactOpError::Storage)?; + if let Some(capture) = stream { + self.write_stream_inline_in_session( + key_info, + capture, + existing_item.as_ref(), + Some(item), + &mut *session, + ) + .await + .map_err(TransactOpError::Storage)?; + } + + Ok(()) + } + OwnedTransactWriteOp::Delete { + key_info, + key, + condition, + maps, + return_values_on_ccf, + stream, + } => { + validation::validate_key_only( + key, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + let key_filter = + pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions) + .map_err(TransactOpError::Storage)?; + + // Always fetch the pre-image. Needed for condition evaluation, + // stale-index deletion in sync_indexes_in_session, and OldImage + // capture for any attached stream. + let existing_doc = coll + .find_one(key_filter.clone()) + .session(&mut *session) + .await + .map_err(TransactOpError::from)?; + + let existing_item = if let Some(doc) = existing_doc.as_ref() { + Some(document_to_item(doc).map_err(TransactOpError::Storage)?) + } else { + None + }; + + if let Some(cond) = condition { + let for_eval = existing_item.clone().unwrap_or_default(); + let passed = + expression::evaluate_condition(cond, &for_eval, maps).map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error( + e.to_string(), + )) + })?; + if !passed { + return Err(TransactOpError::Cancel( + CancellationReason::condition_check_failed_with_item(ccf_return_item( + *return_values_on_ccf, + existing_item.as_ref(), + )), + )); + } + } + + coll.delete_one(key_filter) + .session(&mut *session) + .await + .map_err(TransactOpError::from)?; + + // Propagate to secondary indexes and the stream within the + // same transaction session. + self.sync_indexes_in_session(key_info, existing_item.as_ref(), None, &mut *session) + .await + .map_err(TransactOpError::Storage)?; + if let Some(capture) = stream { + // DDB semantics: a delete on a non-existent key is a + // no-op, and no stream record is emitted. Guard on + // existing_item.is_some() to match. + if existing_item.is_some() { + self.write_stream_inline_in_session( + key_info, + capture, + existing_item.as_ref(), + None, + &mut *session, + ) + .await + .map_err(TransactOpError::Storage)?; + } + } + + Ok(()) + } + OwnedTransactWriteOp::Update { + key_info, + key, + actions, + condition, + maps, + return_values_on_ccf, + stream, + } => { + validation::validate_key_only( + key, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + let key_filter = + pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions) + .map_err(TransactOpError::Storage)?; + + let existing_doc = coll + .find_one(key_filter.clone()) + .session(&mut *session) + .await + .map_err(TransactOpError::from)?; + + let existing_item = if let Some(doc) = existing_doc.as_ref() { + Some(document_to_item(doc).map_err(TransactOpError::Storage)?) + } else { + None + }; + let is_creating = existing_item.is_none(); + + let mut item = existing_item.clone().unwrap_or_else(|| key.clone()); + + if let Some(cond) = condition { + let empty = std::collections::BTreeMap::new(); + let condition_item = if existing_item.is_some() { + &item + } else { + &empty + }; + let passed = expression::evaluate_condition(cond, condition_item, maps) + .map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error( + e.to_string(), + )) + })?; + if !passed { + return Err(TransactOpError::Cancel( + CancellationReason::condition_check_failed_with_item(ccf_return_item( + *return_values_on_ccf, + existing_item.as_ref(), + )), + )); + } + } + + expression::apply_update(actions, &mut item, maps).map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + + // Validate index-key types/emptiness on the post-update + // item; a violation here surfaces as a per-item + // cancellation reason. D-M10, RFC-0003 §2.3. + let idx_pairs = self + .fetch_index_key_schemas(&key_info.table_id) + .await + .map_err(TransactOpError::Storage)?; + if !idx_pairs.is_empty() { + let idx_refs: Vec> = idx_pairs + .iter() + .map(|(n, ks)| extenddb_core::validation::IndexKeyRef { + index_name: n.as_str(), + key_schema: ks.as_slice(), + }) + .collect(); + extenddb_core::validation::validate_index_keys( + &item, + &idx_refs, + &key_info.attribute_definitions, + ) + .map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + } + + let new_doc = + item_to_document(&item, &key_info.key_schema, &key_info.attribute_definitions) + .map_err(TransactOpError::Storage)?; + + let opts = mongodb::options::ReplaceOptions::builder() + .upsert(true) + .build(); + coll.replace_one(key_filter, new_doc) + .with_options(opts) + .session(&mut *session) + .await + .map_err(TransactOpError::from)?; + + // Propagate to secondary indexes and the stream within the + // same transaction session. When the update creates the + // item (previously did not exist), pass None as the old + // image so the stream layer produces an INSERT record, not + // a MODIFY with a synthesized key-only OldImage. + self.sync_indexes_in_session( + key_info, + existing_item.as_ref(), + Some(&item), + &mut *session, + ) + .await + .map_err(TransactOpError::Storage)?; + if let Some(capture) = stream { + let old_for_stream = if is_creating { + None + } else { + existing_item.as_ref() + }; + self.write_stream_inline_in_session( + key_info, + capture, + old_for_stream, + Some(&item), + &mut *session, + ) + .await + .map_err(TransactOpError::Storage)?; + } + + Ok(()) + } + OwnedTransactWriteOp::ConditionCheck { + key_info, + key, + condition, + maps, + return_values_on_ccf, + } => { + validation::validate_key_only( + key, + &key_info.key_schema, + &key_info.attribute_definitions, + ) + .map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + let key_filter = + pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions) + .map_err(TransactOpError::Storage)?; + + let existing_doc = coll + .find_one(key_filter) + .session(&mut *session) + .await + .map_err(TransactOpError::from)?; + + let existing_item = if let Some(doc) = existing_doc.as_ref() { + Some(document_to_item(doc).map_err(TransactOpError::Storage)?) + } else { + None + }; + + let for_eval = existing_item.clone().unwrap_or_default(); + let passed = + expression::evaluate_condition(condition, &for_eval, maps).map_err(|e| { + TransactOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; + if !passed { + return Err(TransactOpError::Cancel( + CancellationReason::condition_check_failed_with_item(ccf_return_item( + *return_values_on_ccf, + existing_item.as_ref(), + )), + )); + } + + Ok(()) + } + } + } + + // ── Pushdown fast path (A5) ────────────────────────────────────── + // + // Callers must pre-check the guard conditions: + // - condition is Some(cond) + // - stream.is_none() + // - gsi_cache_get_fresh(table_id) == Some(false) + // - is_pushable(cond, maps) == Pushable::Yes + // + // Under those guards, the write's atomicity is provided by MongoDB's + // single-document find_one_and_* operators — no session needed, no + // GSI sync, no stream record. The compiled filter merges with the + // key filter so the operator matches only when both apply. On null + // return, we follow up with a `find_one` against the key alone to + // distinguish "key doesn't exist" from "condition failed". + + async fn delete_item_pushdown( + &self, + key_info: &TableKeyInfo, + key: &Item, + return_old: bool, + condition: &Expr, + maps: &ExpressionMaps, + ) -> Result, StorageError> { + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + + let key_filter = pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions)?; + let cond_filter = condition_to_filter(condition, maps)?; + + // Merge key filter and condition filter under an $and so the + // delete only fires when both match. + let merged = doc! { "$and": [key_filter.clone(), cond_filter] }; + + let old_doc = coll + .find_one_and_delete(merged) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if let Some(doc) = old_doc { + let old_item = document_to_item(&doc)?; + return Ok(if return_old { Some(old_item) } else { None }); + } + + // Null return: either the key doesn't exist or the condition + // failed. Disambiguate with a follow-up find_one on the key. + let existing = coll + .find_one(key_filter) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + match existing { + Some(doc) => { + let existing_item = document_to_item(&doc)?; + Err(StorageError::ConditionFailed(Some(existing_item))) + } + None => { + // Key genuinely doesn't exist. Evaluate the condition + // against an empty item to match DDB semantics — some + // conditions (attribute_not_exists) evaluate to true + // even when the item is missing, in which case the + // delete is a no-op success rather than a condition + // failure. + let empty = std::collections::BTreeMap::new(); + let passed = expression::evaluate_condition(condition, &empty, maps) + .map_err(|e| StorageError::Validation(e.to_string()))?; + if passed { + Ok(None) + } else { + Err(StorageError::ConditionFailed(None)) + } + } + } + } + + #[allow(clippy::too_many_arguments)] + async fn update_item_pushdown( + &self, + key_info: &TableKeyInfo, + key: &Item, + actions: &[UpdateAction], + return_old: bool, + return_new: bool, + condition: &Expr, + maps: &ExpressionMaps, + ) -> Result<(Option, Option), StorageError> { + let coll_name = data_collection_name(&key_info.table_id); + let coll = self.data_db.collection::(&coll_name); + + let key_filter = pk_filter(key, &key_info.key_schema, &key_info.attribute_definitions)?; + let cond_filter = condition_to_filter(condition, maps)?; + let merged = doc! { "$and": [key_filter.clone(), cond_filter] }; + + // Load the item first so we can apply the update in Rust and + // then replace it. This is a two-round-trip pushdown rather than + // a single-RT one because DDB update expressions have richer + // semantics than MongoDB's atomic update operators can express + // in general (e.g. list_append, if_not_exists, arithmetic on + // decimal strings). The win over the session-scoped fallback is + // that we skip the session start/commit round trips. + let existing_doc = coll + .find_one(merged.clone()) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let Some(existing) = existing_doc else { + // No document matched the key+condition filter. Disambiguate. + let by_key = coll + .find_one(key_filter.clone()) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + return match by_key { + Some(doc) => { + let existing_item = document_to_item(&doc)?; + Err(StorageError::ConditionFailed(Some(existing_item))) + } + None => { + // Key didn't exist. Evaluate condition against + // empty item (for attribute_not_exists-style + // guards that permit upsert). + let empty = std::collections::BTreeMap::new(); + let passed = expression::evaluate_condition(condition, &empty, maps) + .map_err(|e| StorageError::Validation(e.to_string()))?; + if !passed { + return Err(StorageError::ConditionFailed(None)); + } + // Condition allows the upsert. Build the new item + // from `key` + apply update actions. + let mut new_item = key.clone(); + expression::apply_update(actions, &mut new_item, maps) + .map_err(|e| StorageError::Validation(e.to_string()))?; + let new_doc = item_to_document( + &new_item, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; + let opts = mongodb::options::ReplaceOptions::builder() + .upsert(true) + .build(); + coll.replace_one(key_filter, new_doc) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok((None, if return_new { Some(new_item) } else { None })) + } + }; + }; + + let existing_item = document_to_item(&existing)?; + let mut new_item = existing_item.clone(); + expression::apply_update(actions, &mut new_item, maps) + .map_err(|e| StorageError::Validation(e.to_string()))?; + + let new_doc = item_to_document( + &new_item, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; + + // Bump the OCC version. The session-scoped path uses a versioned + // filter to catch concurrent modifications; the pushdown path + // does the same by merging the current version into the replace + // filter. If a concurrent writer bumps _v between our find_one + // and our replace_one, the replace matches nothing and we fall + // back to a retry. + let current_version = existing.get_i64("_v").unwrap_or(0); + let mut new_doc_versioned = new_doc; + new_doc_versioned.insert("_v", current_version + 1); + + let mut versioned_filter = key_filter.clone(); + if current_version == 0 { + versioned_filter.insert("_v", doc! { "$not": { "$gt": 0_i64 } }); + } else { + versioned_filter.insert("_v", current_version); + } + + let result = coll + .replace_one(versioned_filter, new_doc_versioned) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if result.matched_count == 0 { + // Concurrent update raced us. Fall back to the session-scoped + // path which has a retry loop. This is rare — return an + // Internal error the trait implementer can catch and retry. + return Err(StorageError::Internal( + "pushdown update raced by concurrent writer; retry via session-scoped path" + .to_owned(), + )); + } + + let old_out = if return_old { + Some(existing_item) + } else { + None + }; + let new_out = if return_new { Some(new_item) } else { None }; + Ok((old_out, new_out)) + } + + // ── GSI Backfill ────────────────────────────────────────────── + // + // Called by the gsi_backfill_worker in ttl_worker.rs. Reads one + // batch of base-table items past the given cursor and upserts + // matching index rows. Returns the new cursor and whether more + // items remain to scan. The worker persists the cursor between + // batches so a mid-backfill server restart resumes from where it + // left off — see the CREATING → ACTIVE state machine in + // update_table_impl / spawn_workers. + + pub(crate) async fn backfill_gsi_batch( + &self, + key_info: &TableKeyInfo, + index_id: &str, + idx_key_schema: &[KeySchemaElement], + projection: &Projection, + cursor: Option<&bson::Bson>, + batch_size: i64, + ) -> Result { + use futures::TryStreamExt; + + let base_coll_name = data_collection_name(&key_info.table_id); + let base_coll = self.data_db.collection::(&base_coll_name); + let idx_coll_name = data_collection_name(index_id); + let idx_coll = self.data_db.collection::(&idx_coll_name); + + let mut filter = Document::new(); + if let Some(c) = cursor { + filter.insert("_id", doc! { "$gt": c.clone() }); + } + + let opts = mongodb::options::FindOptions::builder() + .sort(doc! { "_id": 1 }) + .limit(batch_size) + .build(); + + let base_cursor = base_coll + .find(filter) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let docs: Vec = base_cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let scanned = docs.len(); + let last_id = docs.last().and_then(|d| d.get("_id").cloned()); + + for doc in docs { + let item = document_to_item(&doc)?; + if !item_has_index_keys(&item, idx_key_schema) { + continue; + } + + let projected = project_item(&item, idx_key_schema, &key_info.key_schema, projection); + let idx_doc = index_document( + &projected, + idx_key_schema, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; + let filter = index_entry_filter( + &projected, + idx_key_schema, + &key_info.key_schema, + &key_info.attribute_definitions, + )?; + let opts = mongodb::options::ReplaceOptions::builder() + .upsert(true) + .build(); + idx_coll + .replace_one(filter, idx_doc) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + // A short-read (fewer docs than the batch size) means we've + // reached the end of the base collection. Upstream flips the + // index to ACTIVE when that happens. + Ok(GsiBackfillProgress { + scanned, + last_id, + done: (scanned as i64) < batch_size, + }) + } +} + +/// Progress from one `backfill_gsi_batch` invocation. +pub(crate) struct GsiBackfillProgress { + /// Number of base-collection documents read in this batch (before + /// filtering out those missing index-key attributes). + pub scanned: usize, + /// The `_id` of the last document scanned; the next batch resumes + /// with `_id > last_id`. `None` when the batch was empty. + pub last_id: Option, + /// Whether the base collection has been fully scanned. + pub done: bool, +} + +// ── Contention / retry helpers ────────────────────────────────────────── + +/// Maximum number of times to retry a write that MongoDB aborted as a +/// transient conflict. Small enough that we don't lock a partition on +/// sustained hot-key contention; large enough to absorb ordinary +/// snapshot-isolation aborts. Matches the OCC retry ceiling elsewhere +/// in this file. +const TRANSIENT_RETRY_ATTEMPTS: u32 = 50; + +/// Return type of `try_build_native_update`. Distinguishes an +/// operator-document update (`{$set, $unset, $inc}`) from an +/// aggregation-pipeline update (needed for numeric ADD, which +/// converts a string-stored `.N` value to Decimal128, applies the +/// delta, and converts back). +/// +/// `Pipeline` carries an optional `type_guard` filter — for numeric +/// ADD we require the target attribute to be absent or already an +/// `.N` so we don't clobber a string with a number. When the guard +/// rejects the match, `find_one_and_update` returns `None`, and the +/// caller falls back to the slow (session-scoped) path which reads +/// the pre-image and surfaces a proper `ValidationException`. +enum NativeUpdate { + Doc(Document), + Pipeline { + type_guard: Option, + pipeline: Vec, + }, +} + +/// Error signal used inside per-attempt transaction bodies. Lets the +/// body use `?` for control flow while distinguishing "retry this +/// whole transaction" from "return this error to the caller." +enum TxErr { + Transient, + Fatal(StorageError), +} + +impl From for TxErr { + fn from(e: mongodb::error::Error) -> Self { + if is_transient_write_conflict(&e) { + TxErr::Transient + } else { + TxErr::Fatal(StorageError::Internal(e.to_string())) + } + } +} + +impl From for TxErr { + fn from(e: StorageError) -> Self { + TxErr::Fatal(e) + } +} + +/// Detect the family of errors MongoDB uses to signal "your write lost +/// to another concurrent writer under snapshot isolation; retry." +/// +/// The transient-transaction label is set on any error that a +/// `withTransaction` client would automatically retry. In addition to +/// abstract labels the raw `WriteConflict` (code 112) still shows up +/// when a same-document collision surfaces on the write itself rather +/// than at commit — check that too. RFC-0003 §4.1 / §4.3. +fn is_transient_write_conflict(e: &mongodb::error::Error) -> bool { + if e.contains_label(mongodb::error::TRANSIENT_TRANSACTION_ERROR) + || e.contains_label(mongodb::error::UNKNOWN_TRANSACTION_COMMIT_RESULT) + { + return true; + } + matches!(*e.kind, mongodb::error::ErrorKind::Command(ref c) if c.code == 112) + || matches!( + *e.kind, + mongodb::error::ErrorKind::Write(mongodb::error::WriteFailure::WriteError( + mongodb::error::WriteError { code: 112, .. } + )) + ) +} + +/// Detect a duplicate-key error (E11000, code 11000). Used at +/// conditional-insert sites — a duplicate is the manifestation of a +/// conditional-put race, so it must be surfaced as +/// `ConditionalCheckFailedException` rather than a 500. +fn is_duplicate_key(e: &mongodb::error::Error) -> bool { + matches!( + *e.kind, + mongodb::error::ErrorKind::Write(mongodb::error::WriteFailure::WriteError( + mongodb::error::WriteError { code: 11000, .. } + )) + ) +} + +/// Exponential-backoff sleep with random jitter. Used inside the OCC +/// / WriteConflict retry loops so competing writers don't lock-step +/// re-retry into the same conflict window. +async fn backoff_sleep(attempt: u32) { + let base_us = 50u64.saturating_mul(1u64 << attempt.min(8)); + let jitter = rand::random_range(0..=base_us); + tokio::time::sleep(std::time::Duration::from_micros(jitter)).await; +} + +// ── Transaction helper types ────────────────────────────────────────── + +enum TransactOpError { + Cancel(extenddb_core::types::CancellationReason), + Storage(StorageError), + /// MongoDB aborted the transaction as a transient conflict — + /// the whole transact_write_items txn should be retried from + /// the top. + Transient, +} + +impl From for TransactOpError { + fn from(e: mongodb::error::Error) -> Self { + if is_transient_write_conflict(&e) { + TransactOpError::Transient + } else { + TransactOpError::Storage(StorageError::Internal(e.to_string())) + } + } +} + +impl From for TransactOpError { + fn from(e: StorageError) -> Self { + TransactOpError::Storage(e) + } +} + +/// Choose the `Item` value to include in a `CancellationReason` when a +/// condition check fails inside `TransactWriteItems`. +/// +/// Per DynamoDB's contract, the pre-existing item is returned only when the +/// caller requested `ReturnValuesOnConditionCheckFailure = ALL_OLD` AND the +/// item existed at the time of the check. In all other cases the field is +/// omitted (returned as `None`). +fn ccf_return_item( + rv: ReturnValuesOnConditionCheckFailure, + existing: Option<&Item>, +) -> Option { + match rv { + ReturnValuesOnConditionCheckFailure::AllOld => existing.cloned(), + ReturnValuesOnConditionCheckFailure::None => None, + } +} + +/// Owned version of `TransactWriteOp` to allow moving into async blocks. +enum OwnedTransactWriteOp { + Put { + key_info: TableKeyInfo, + item: Item, + condition: Option, + maps: ExpressionMaps, + return_values_on_ccf: ReturnValuesOnConditionCheckFailure, + stream: Option, + }, + Delete { + key_info: TableKeyInfo, + key: Item, + condition: Option, + maps: ExpressionMaps, + return_values_on_ccf: ReturnValuesOnConditionCheckFailure, + stream: Option, + }, + Update { + key_info: TableKeyInfo, + key: Item, + actions: Vec, + condition: Option, + maps: ExpressionMaps, + return_values_on_ccf: ReturnValuesOnConditionCheckFailure, + stream: Option, + }, + ConditionCheck { + key_info: TableKeyInfo, + key: Item, + condition: Expr, + maps: ExpressionMaps, + return_values_on_ccf: ReturnValuesOnConditionCheckFailure, + }, +} + +fn clone_transact_write_op(op: &TransactWriteOp<'_>) -> OwnedTransactWriteOp { + match op { + TransactWriteOp::Put { + key_info, + item, + condition, + maps, + return_values_on_ccf, + stream, + } => OwnedTransactWriteOp::Put { + key_info: (*key_info).clone(), + item: (*item).clone(), + condition: condition.cloned(), + maps: (*maps).clone(), + return_values_on_ccf: *return_values_on_ccf, + stream: stream.clone(), + }, + TransactWriteOp::Delete { + key_info, + key, + condition, + maps, + return_values_on_ccf, + stream, + } => OwnedTransactWriteOp::Delete { + key_info: (*key_info).clone(), + key: (*key).clone(), + condition: condition.cloned(), + maps: (*maps).clone(), + return_values_on_ccf: *return_values_on_ccf, + stream: stream.clone(), + }, + TransactWriteOp::Update { + key_info, + key, + actions, + condition, + maps, + return_values_on_ccf, + stream, + } => OwnedTransactWriteOp::Update { + key_info: (*key_info).clone(), + key: (*key).clone(), + actions: actions.to_vec(), + condition: condition.cloned(), + maps: (*maps).clone(), + return_values_on_ccf: *return_values_on_ccf, + stream: stream.clone(), + }, + TransactWriteOp::ConditionCheck { + key_info, + key, + condition, + maps, + return_values_on_ccf, + } => OwnedTransactWriteOp::ConditionCheck { + key_info: (*key_info).clone(), + key: (*key).clone(), + condition: (*condition).clone(), + maps: (*maps).clone(), + return_values_on_ccf: *return_values_on_ccf, + }, + } +} + +/// Resolve a key expression (Placeholder) to an `AttributeValue`. +fn resolve_key_expr(expr: &Expr, maps: &ExpressionMaps) -> Result { + match expr { + Expr::Placeholder(name) => maps + .resolve_value(name) + .cloned() + .map_err(|e| StorageError::Validation(e.to_string())), + _ => Err(StorageError::Internal( + "expected placeholder in key condition".to_owned(), + )), + } +} + +/// Build a `MongoDB` filter for a sort key condition. +fn build_sk_filter( + sk_cond: &SortKeyCondition, + sk_field: &str, + maps: &ExpressionMaps, +) -> Result { + match sk_cond { + SortKeyCondition::Compare { op, value, .. } => { + let av = resolve_key_expr(value, maps)?; + let sk_type = infer_sk_type_from_field(sk_field); + let bson_val = sk_to_bson(&av, sk_type)?; + + let filter = match op { + extenddb_core::expression::CompareOp::Eq => doc! { sk_field: bson_val }, + extenddb_core::expression::CompareOp::Lt => doc! { sk_field: { "$lt": bson_val } }, + extenddb_core::expression::CompareOp::Le => doc! { sk_field: { "$lte": bson_val } }, + extenddb_core::expression::CompareOp::Gt => doc! { sk_field: { "$gt": bson_val } }, + extenddb_core::expression::CompareOp::Ge => doc! { sk_field: { "$gte": bson_val } }, + extenddb_core::expression::CompareOp::Ne => doc! { sk_field: { "$ne": bson_val } }, + }; + Ok(filter) + } + SortKeyCondition::Between { low, high, .. } => { + let sk_type = infer_sk_type_from_field(sk_field); + let low_av = resolve_key_expr(low, maps)?; + let high_av = resolve_key_expr(high, maps)?; + if sk_between_low_gt_high(&low_av, &high_av) { + return Err(StorageError::Validation( + "Invalid KeyConditionExpression: The BETWEEN operator requires upper bound to be greater than or equal to lower bound".to_owned(), + )); + } + let low_bson = sk_to_bson(&low_av, sk_type)?; + let high_bson = sk_to_bson(&high_av, sk_type)?; + Ok(doc! { sk_field: { "$gte": low_bson, "$lte": high_bson } }) + } + SortKeyCondition::BeginsWith { prefix, .. } => { + let prefix_av = resolve_key_expr(prefix, maps)?; + match prefix_av { + AttributeValue::S(ref p) => { + // `sk BEGINS_WITH P` matches every X where P is a + // prefix of X. Emit that as `sk >= P AND sk < P'`, + // where P' is the least string strictly greater + // than any P-starting string. + // + // `next_string_prefix` finds P' by incrementing + // the last non-`char::MAX` code point. If P is + // entirely `char::MAX`, no such P' exists — return + // just the lower-bound filter and let mongo match + // every string ≥ P (which is what DDB does). + match next_string_prefix(p) { + Some(upper) => Ok(doc! { + sk_field: { "$gte": p.as_str(), "$lt": upper } + }), + None => Ok(doc! { sk_field: { "$gte": p.as_str() } }), + } + } + AttributeValue::B(ref b) => { + // Binary sort keys are stored as lowercase hex strings + // (D-M5), so `sk BEGINS_WITH B` is a string-prefix range + // over the hex encoding, exactly like the S case above: + // `sk_b >= hex(B) AND sk_b < next_string_prefix(hex(B))`. + // + // The exclusive upper bound must be the next prefix in + // hex-STRING space (increment the last hex character), not + // hex(increment_bytes(B)). Incrementing the raw bytes then + // re-encoding widens the range and admits unrelated keys — + // e.g. begins_with(0x2F,0xFF) -> ["2fff", hex(0x30,0x00) = + // "3000"), which wrongly matches the stored key 0x30 + // ("30"). next_string_prefix("2fff") = "2ffg" excludes it. + // When the prefix is empty, there is no upper bound and we + // match every key >= "" (all of them), matching DDB. + let lo = binary_sk_to_hex(b); + match next_string_prefix(&lo) { + Some(upper) => Ok(doc! { sk_field: { "$gte": lo, "$lt": upper } }), + None => Ok(doc! { sk_field: { "$gte": lo } }), + } + } + _ => Err(StorageError::Validation( + "begins_with requires string or binary sort key".to_string(), + )), + } + } + } +} + +/// Convert an `AttributeValue` sort key to the appropriate BSON type. +/// Return true when a sort-key BETWEEN's low bound is strictly greater than its high bound. +/// +/// DynamoDB rejects this at the wire layer with a ValidationException; the storage +/// backend must reject it too, since the engine layer only validates BETWEEN for +/// filter/condition expressions, not for KeyConditionExpression's sort-key path. +/// +/// The comparison is done in the source AttributeValue domain so it happens before +/// any Decimal128/f64 conversion that could mask ordering. Strings are compared +/// lexicographically (matching DynamoDB), numbers via f64 (adequate for ordering — +/// values exceeding Decimal128 range are rejected downstream in `sk_to_bson`), and +/// binary bytewise. +fn sk_between_low_gt_high(low: &AttributeValue, high: &AttributeValue) -> bool { + match (low, high) { + (AttributeValue::S(l), AttributeValue::S(h)) => l > h, + (AttributeValue::N(l), AttributeValue::N(h)) => { + match (l.parse::(), h.parse::()) { + (Ok(lf), Ok(hf)) => lf > hf, + _ => false, // downstream sk_to_bson will surface the parse error + } + } + (AttributeValue::B(l), AttributeValue::B(h)) => l > h, + _ => false, // type mismatch — downstream sk_to_bson will surface it + } +} + +fn sk_to_bson( + av: &AttributeValue, + sk_type: ScalarAttributeType, +) -> Result { + match (sk_type, av) { + (ScalarAttributeType::S, AttributeValue::S(s)) => Ok(bson::Bson::String(s.clone())), + (ScalarAttributeType::N, AttributeValue::N(n)) => n + .parse::() + .map(bson::Bson::Decimal128) + .map_err(|_| { + StorageError::Validation(format!( + "Numeric sort key value '{n}' exceeds supported precision (Decimal128, 34 significant digits)" + )) + }), + // Binary sort keys are stored as hex-encoded strings; see + // `binary_sk_to_hex` and the D-M5 rationale. Query/BETWEEN + // filters must project to the same encoding. + (ScalarAttributeType::B, AttributeValue::B(b)) => { + Ok(bson::Bson::String(binary_sk_to_hex(b))) + } + _ => Err(StorageError::Internal("sort key type mismatch".to_string())), + } +} + +/// Infer the `ScalarAttributeType` from the sort key field name. +fn infer_sk_type_from_field(field: &str) -> ScalarAttributeType { + if field.ends_with("_n") { + ScalarAttributeType::N + } else if field.ends_with("_b") { + ScalarAttributeType::B + } else { + ScalarAttributeType::S + } +} + +/// Compute the least string strictly greater than every string +/// beginning with `s`, used as the exclusive upper bound for +/// `sk BEGINS_WITH s`. +/// +/// Strategy: find the rightmost char in `s` that isn't `char::MAX`, +/// increment it, and truncate everything to its right. If every char +/// is `char::MAX` (an unlikely-but-real edge case), no upper bound +/// exists — return `None` so the caller can drop the `$lt` clause. +/// +/// The previous implementation appended `char::MAX` to `s` and used +/// `$lt`, which excluded any stored string equal to `s + char::MAX` +/// (or extending past it) — those still begin with `s` and DDB +/// matches them. D-m12. +fn next_string_prefix(s: &str) -> Option { + let chars: Vec = s.chars().collect(); + // Walk from the right, find the first char we can bump. + for i in (0..chars.len()).rev() { + if chars[i] < char::MAX { + let mut out = String::with_capacity(s.len()); + for c in &chars[..i] { + out.push(*c); + } + // char::from_u32 handles the surrogate gap by skipping + // to the next valid scalar. u32 → char via char::from_u32 + // returns None on the surrogate range D800..=DFFF, so + // walk past it. + let mut next = u32::from(chars[i]) + 1; + let bumped = loop { + if let Some(c) = char::from_u32(next) { + break c; + } + next += 1; + }; + out.push(bumped); + return Some(out); + } + } + None +} + +fn item_has_index_keys(item: &Item, idx_key_schema: &[KeySchemaElement]) -> bool { + idx_key_schema + .iter() + .all(|ks| item.contains_key(&ks.attribute_name)) +} + +fn project_item( + item: &Item, + idx_key_schema: &[KeySchemaElement], + base_key_schema: &[KeySchemaElement], + projection: &Projection, +) -> Item { + match projection.projection_type { + ProjectionType::All => item.clone(), + ProjectionType::KeysOnly => { + let mut projected = Item::new(); + for ks in idx_key_schema.iter().chain(base_key_schema.iter()) { + if let Some(v) = item.get(&ks.attribute_name) { + projected.insert(ks.attribute_name.clone(), v.clone()); + } + } + projected + } + ProjectionType::Include => { + let mut projected = Item::new(); + // Always include key attributes + for ks in idx_key_schema.iter().chain(base_key_schema.iter()) { + if let Some(v) = item.get(&ks.attribute_name) { + projected.insert(ks.attribute_name.clone(), v.clone()); + } + } + // Include non-key attributes from projection + if let Some(ref attrs) = projection.non_key_attributes { + for attr in attrs { + if let Some(v) = item.get(attr) { + projected.insert(attr.clone(), v.clone()); + } + } + } + projected + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn between_low_gt_high_string() { + assert!(sk_between_low_gt_high( + &AttributeValue::S("z".into()), + &AttributeValue::S("a".into()) + )); + assert!(!sk_between_low_gt_high( + &AttributeValue::S("a".into()), + &AttributeValue::S("z".into()) + )); + assert!(!sk_between_low_gt_high( + &AttributeValue::S("m".into()), + &AttributeValue::S("m".into()) + )); + } + + #[test] + fn between_low_gt_high_number() { + assert!(sk_between_low_gt_high( + &AttributeValue::N("100".into()), + &AttributeValue::N("50".into()) + )); + assert!(!sk_between_low_gt_high( + &AttributeValue::N("50".into()), + &AttributeValue::N("100".into()) + )); + assert!(!sk_between_low_gt_high( + &AttributeValue::N("42".into()), + &AttributeValue::N("42".into()) + )); + } + + #[test] + fn ccf_return_item_all_old_with_existing() { + let mut item = Item::new(); + item.insert("a".to_string(), AttributeValue::S("1".to_string())); + let returned = ccf_return_item(ReturnValuesOnConditionCheckFailure::AllOld, Some(&item)); + assert_eq!(returned, Some(item)); + } + + #[test] + fn ccf_return_item_all_old_without_existing() { + let returned = ccf_return_item(ReturnValuesOnConditionCheckFailure::AllOld, None); + assert_eq!(returned, None); + } + + #[test] + fn ccf_return_item_none_never_returns() { + let mut item = Item::new(); + item.insert("a".to_string(), AttributeValue::S("1".to_string())); + let returned = ccf_return_item(ReturnValuesOnConditionCheckFailure::None, Some(&item)); + assert_eq!(returned, None); + } + + #[test] + fn next_string_prefix_ascii() { + // Basic ASCII: "abc" -> "abd" as the exclusive upper bound. + assert_eq!(next_string_prefix("abc").as_deref(), Some("abd")); + + // Trailing char::MAX skips back to a bumpable char. + // E.g. "abZ\u{10FFFF}" -> "ab[" + let s: String = ['a', 'b', 'Z', char::MAX].iter().collect(); + let expected: String = ['a', 'b', '['].iter().collect(); + assert_eq!(next_string_prefix(&s).as_deref(), Some(expected.as_str())); + + // All-char::MAX -> None (no bound; caller drops $lt clause). + let s: String = std::iter::repeat_n(char::MAX, 3).collect(); + assert!(next_string_prefix(&s).is_none()); + + // Empty string is also unbounded (no chars to bump). + assert!(next_string_prefix("").is_none()); + } + + #[test] + fn between_low_gt_high_binary() { + assert!(sk_between_low_gt_high( + &AttributeValue::B(vec![0xff]), + &AttributeValue::B(vec![0x00]) + )); + assert!(!sk_between_low_gt_high( + &AttributeValue::B(vec![0x00]), + &AttributeValue::B(vec![0xff]) + )); + } + + fn binary_begins_with_bounds(prefix: Vec) -> (String, Option) { + let mut values = std::collections::HashMap::new(); + values.insert(":p".to_string(), AttributeValue::B(prefix)); + let maps = ExpressionMaps::new(std::collections::HashMap::new(), values); + let cond = SortKeyCondition::BeginsWith { + path: vec![PathElement::Attribute("sk".to_string())], + prefix: Expr::Placeholder(":p".to_string()), + }; + let doc = build_sk_filter(&cond, "sk_b", &maps).unwrap(); + let inner = doc.get_document("sk_b").unwrap(); + let lo = inner.get_str("$gte").unwrap().to_string(); + let hi = inner.get_str("$lt").ok().map(str::to_string); + (lo, hi) + } + + #[test] + fn binary_begins_with_uses_hex_space_prefix() { + // Upper bound is the next prefix in hex-STRING space, not + // hex(increment_bytes(prefix)). + + // begins_with(0x2F,0xFF): lo="2fff", hi must be "2ffg" (not "3000"). + // The old code produced "3000", which wrongly admitted stored key + // 0x30 ("30") since "2fff" <= "30" < "3000". With "2ffg", "30" is + // excluded because "30" > "2ffg". + let (lo, hi) = binary_begins_with_bounds(vec![0x2f, 0xff]); + assert_eq!(lo, "2fff"); + assert_eq!(hi.as_deref(), Some("2ffg")); + assert!("30" >= hi.as_deref().unwrap(), "0x30 must be excluded"); + + // begins_with(0xFF): lo="ff", hi must be "fg". The old code produced + // "00" (0xFF+1 wrapped then prepended 0x01 -> "01ff"? either way an + // empty/incorrect range), dropping every match. + let (lo, hi) = binary_begins_with_bounds(vec![0xff]); + assert_eq!(lo, "ff"); + assert_eq!(hi.as_deref(), Some("fg")); + assert!("ffab" < hi.as_deref().unwrap(), "0xFFAB must be included"); + } +} diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs new file mode 100644 index 00000000..cd859af1 --- /dev/null +++ b/crates/storage-mongodb/src/lib.rs @@ -0,0 +1,324 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `MongoDB` storage backend for extenddb. +//! +//! Implements the storage traits from `extenddb-storage` using `MongoDB` +//! as the backing store. + +mod admin_store; +mod authorization_store; +mod backup_engine; +mod bootstrapper; +mod catalog_store; +pub mod condition; +pub mod config; +mod credential_store; +mod data; +mod data_engine; +mod management_store; +mod metadata_engine; +mod operations; +pub mod pushdown; +mod stream_engine; +mod table_engine; +mod ttl_worker; +mod worker_store; + +pub use bootstrapper::MongoBootstrapper; +pub use catalog_store::MongoCatalogStore; +pub use config::MongoStorageConfig; +pub use credential_store::MongoCredentialStore; + +use std::sync::Arc; + +use extenddb_storage::error::StorageError; + +// ============================================================================ +// Backend registration +// ============================================================================ + +use extenddb_storage::hooks::{ServerRuntimeHooks, WorkerContext}; +use extenddb_storage::server_components::{BackendError, ServerComponents}; + +/// Backend-specific runtime hooks for `MongoDB`. +struct MongoRuntimeHooks { + engine: Arc, +} + +#[async_trait::async_trait] +impl ServerRuntimeHooks for MongoRuntimeHooks { + async fn spawn_workers(&self, ctx: &WorkerContext) -> Vec> { + let storage_for_ttl = self.engine.clone(); + let metrics = ctx.metrics.clone(); + let ttl = tokio::spawn(async move { + ttl_worker::ttl_cleanup_worker(storage_for_ttl, metrics).await; + }); + let storage_for_stream = self.engine.clone(); + let stream = tokio::spawn(async move { + ttl_worker::stream_record_cleanup_worker(storage_for_stream).await; + }); + let storage_for_backfill = self.engine.clone(); + let backfill = tokio::spawn(async move { + ttl_worker::gsi_backfill_worker(storage_for_backfill).await; + }); + let storage_for_control_plane = self.engine.clone(); + let control_plane = tokio::spawn(async move { + ttl_worker::control_plane_worker(storage_for_control_plane).await; + }); + tracing::info!( + "MongoDB backend: TTL, stream cleanup, GSI backfill, and control-plane workers spawned" + ); + vec![ttl, stream, backfill, control_plane] + } + + fn backend_info(&self) -> Option { + Some("mongodb".to_string()) + } +} + +/// Build the assembled server components for the mongo backend (`serve`). +fn server_components_factory( + config: &dyn extenddb_storage::config::StorageConfig, + region: &str, +) -> std::pin::Pin< + Box> + Send>, +> { + let connection_string = config.connection_config().to_string(); + let max_connections = config.max_connections(); + let region = region.to_string(); + Box::pin(async move { + // Create MongoEngine + let engine = MongoEngine::new(&connection_string, ®ion, max_connections) + .await + .map_err(|e| BackendError::ConnectionFailed { + backend: "mongodb".to_string(), + details: e.to_string(), + })?; + + let engine = Arc::new(engine); + + // Create catalog store + let catalog_client = mongodb::Client::with_uri_str(&connection_string) + .await + .map_err(|e| BackendError::ConnectionFailed { + backend: "mongodb".to_string(), + details: format!("Failed to create catalog client: {e}"), + })?; + + // Load encryption key from settings collection + let catalog_db = catalog_client.database("extenddb_catalog"); + let settings_coll = catalog_db.collection::("settings"); + let enc_key = settings_coll + .find_one(mongodb::bson::doc! { "_id": "encryption_key" }) + .await + .map_err(|e| BackendError::InitializationFailed(format!("Load encryption key: {e}")))? + .and_then(|d| d.get_str("value").ok().map(std::borrow::ToOwned::to_owned)) + .unwrap_or_default(); + + let catalog_store = Arc::new(MongoCatalogStore::with_encryption_key( + catalog_client, + enc_key.clone(), + )) as Arc; + + // Create credential store. The bin layer wraps this in + // CachedCredentialStore using the operator-configured TTL + // before constructing the auth provider. + let auth_client = mongodb::Client::with_uri_str(&connection_string) + .await + .map_err(|e| BackendError::InitializationFailed(format!("Auth client: {e}")))?; + let cred_store: Arc = + Arc::new(MongoCredentialStore::new(auth_client, enc_key)); + + // Create runtime hooks + let runtime_hooks = Box::new(MongoRuntimeHooks { + engine: engine.clone(), + }); + + Ok(ServerComponents { + engine, + catalog_store, + credential_store: cred_store, + runtime_hooks: Some(runtime_hooks), + }) + }) +} + +/// The MongoDB storage backend. A thin bin installs it via +/// `extenddb_storage::set_backend(extenddb_storage_mongodb::backend())`. +pub fn backend() -> extenddb_storage::Backend { + extenddb_storage::Backend { + name: "mongodb", + bootstrapper: |config_path, cli_args| { + Box::pin(async move { + let store = MongoBootstrapper::from_config(&config_path, &cli_args).await?; + Ok(Box::new(store) as Box) + }) + }, + storage_config: |table| { + let config: MongoStorageConfig = table + .clone() + .try_into() + .map_err(|e: toml::de::Error| format!("Failed to parse mongodb config: {e}"))?; + Ok(Box::new(config) as Box) + }, + operations: &operations::MongoOperationsEngine, + settings_store: |connection_string| { + let connection_string = connection_string.to_string(); + Box::pin(async move { + let client = mongodb::Client::with_uri_str(&connection_string) + .await + .map_err(|e| { + extenddb_storage::settings_store::SettingsStoreError::ConnectionFailed( + e.to_string(), + ) + })?; + Ok(Box::new(MongoCatalogStore::new(client)) + as Box< + dyn extenddb_storage::management_store::SettingsStore, + >) + }) + }, + diagnostics_store: |connection_string| { + let connection_string = connection_string.to_string(); + Box::pin(async move { + let client = mongodb::Client::with_uri_str(&connection_string) + .await + .map_err(|e| { + extenddb_storage::diagnostics_store::DiagnosticsStoreError::ConnectionFailed( + e.to_string(), + ) + })?; + Ok(Box::new(MongoCatalogStore::new(client)) + as Box) + }) + }, + server_components: server_components_factory, + } +} + +// ============================================================================ +// MongoEngine +// ============================================================================ + +/// TTL for entries in [`MongoEngine::gsi_cache`]. +/// +/// The GSI cache is per-process. When multiple ExtendDB instances share a +/// catalog, an admin creating or dropping a GSI on instance A does not +/// invalidate instance B's cache. Bounding cache entries by wall-clock age +/// gives eventual convergence at a small cost (one catalog `find` per table +/// per TTL window), which is far cheaper than the cost of silently skipping +/// index updates on tables where GSIs were added out-of-band. +const GSI_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(60); + +/// `MongoDB` storage backend. +pub struct MongoEngine { + client: mongodb::Client, + pub(crate) catalog_db: mongodb::Database, + data_db: mongodb::Database, + region: String, + /// Cache of `table_id` -> (`has_gsi`, insertion time). Avoids catalog + /// queries on every write for tables with no GSIs. Entries older than + /// [`GSI_CACHE_TTL`] are treated as misses and re-read from the catalog, + /// so GSI additions/removals on other ExtendDB instances converge within + /// the TTL window. + gsi_cache: dashmap::DashMap, +} + +impl MongoEngine { + pub async fn new( + connection_string: &str, + region: &str, + max_connections: u32, + ) -> Result { + let mut options = mongodb::options::ClientOptions::parse(connection_string) + .await + .map_err(|e| StorageError::Connection(e.to_string()))?; + options.max_pool_size = Some(max_connections); + + // Reject non-primary read preferences. DynamoDB's `ConsistentRead=true` + // requires linearizable reads; MongoDB's Primary read concern is the + // only mode that provides that. A connection string like + // `mongodb://.../?readPreference=secondaryPreferred` would silently + // route reads to a secondary and return stale data — a fidelity + // violation the caller has no way to detect. + if let Some(sel) = options.selection_criteria.as_ref() { + use mongodb::options::{ReadPreference, SelectionCriteria}; + let is_non_primary = match sel { + SelectionCriteria::ReadPreference(rp) => !matches!(rp, ReadPreference::Primary), + _ => false, + }; + if is_non_primary { + return Err(StorageError::Connection( + "MongoDB connection string must use readPreference=primary. \ + Non-primary read preferences (secondary, secondaryPreferred, \ + nearest, primaryPreferred) route reads to replicas and \ + silently break ConsistentRead=true." + .to_owned(), + )); + } + } + + if !matches!(options.tls, Some(mongodb::options::Tls::Enabled(_))) { + tracing::warn!( + "MongoDB connection is not using TLS; credentials and data will \ + traverse the network in cleartext. Enable TLS with `?tls=true` \ + in the connection string, or use a `mongodb+srv://` URI." + ); + } + + let client = mongodb::Client::with_options(options) + .map_err(|e| StorageError::Connection(e.to_string()))?; + + let catalog_db = client.database("extenddb_catalog"); + let data_db = client.database("extenddb_data"); + + Ok(Self { + client, + catalog_db, + data_db, + region: region.to_owned(), + gsi_cache: dashmap::DashMap::new(), + }) + } + + /// Look up a fresh GSI-cache entry for `table_id`. + /// + /// Returns `Some(has_gsi)` when a cache entry exists and is younger than + /// [`GSI_CACHE_TTL`], `None` otherwise (either no entry or expired). + /// Callers that get `None` must fall back to reading the catalog. + pub(crate) fn gsi_cache_get_fresh(&self, table_id: &str) -> Option { + let entry = self.gsi_cache.get(table_id)?; + let (has_gsi, inserted) = *entry; + if inserted.elapsed() <= GSI_CACHE_TTL { + Some(has_gsi) + } else { + None + } + } + + /// Record a fresh GSI-cache observation for `table_id`. + pub(crate) fn gsi_cache_set(&self, table_id: &str, has_gsi: bool) { + self.gsi_cache + .insert(table_id.to_owned(), (has_gsi, std::time::Instant::now())); + } + + /// Remove a GSI-cache entry (e.g., on GSI drop or table delete). + pub(crate) fn gsi_cache_invalidate(&self, table_id: &str) { + self.gsi_cache.remove(table_id); + } + + /// Validate `account_id` against injection attacks. + fn validate_account_id(account_id: &str) -> Result<(), StorageError> { + if account_id.contains('$') + || account_id.contains('.') + || account_id.contains('\0') + || !account_id.is_ascii() + { + return Err(StorageError::Validation(format!( + "Invalid account_id: {account_id}" + ))); + } + Ok(()) + } +} diff --git a/crates/storage-mongodb/src/management_store.rs b/crates/storage-mongodb/src/management_store.rs new file mode 100644 index 00000000..1e3052bc --- /dev/null +++ b/crates/storage-mongodb/src/management_store.rs @@ -0,0 +1,2400 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `ManagementStore` trait implementation for `MongoDB`. + +use futures::TryStreamExt; +use futures::future::BoxFuture; +use mongodb::bson::{self, Binary, DateTime as BsonDateTime, Document, doc}; +use mongodb::options::{FindOptions, UpdateOptions}; + +use extenddb_storage::management_store::{ + AccessKeyCreated, AccountDetail, AdminEntry, GroupDetail, GroupListEntry, ManagementStore, + MetricsRow, OpError, OpResult, RoleDetail, RoleListEntry, UserDetail, UserListEntry, +}; + +use crate::catalog_store::MongoCatalogStore; + +fn is_duplicate_key(e: &mongodb::error::Error) -> bool { + matches!( + *e.kind, + mongodb::error::ErrorKind::Write(mongodb::error::WriteFailure::WriteError( + mongodb::error::WriteError { code: 11000, .. } + )) + ) +} + +fn to_offset_dt(dt: BsonDateTime) -> time::OffsetDateTime { + time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(dt.timestamp_millis()) * 1_000_000) + .unwrap_or(time::OffsetDateTime::UNIX_EPOCH) +} + +fn now_bson() -> BsonDateTime { + BsonDateTime::now() +} + +// ── ManagementStore ───────────────────────────────────────────────────── + +impl ManagementStore for MongoCatalogStore { + fn create_account(&self, account_id: &str, account_name: &str) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let account_name = account_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("accounts"); + let result = coll + .insert_one(doc! { + "account_id": &account_id, + "account_name": &account_name, + "created_at": now_bson(), + }) + .await; + match result { + Ok(_) => Ok(()), + Err(e) if is_duplicate_key(&e) => { + Err(OpError::AlreadyExists("Account already exists".to_owned())) + } + Err(e) => { + tracing::error!("create_account failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + }) + } + + fn delete_account(&self, account_id: &str) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + Box::pin(async move { + let tables_coll = self.catalog_db().collection::("tables"); + let has_tables = tables_coll + .count_documents(doc! { "account_id": &account_id }) + .await + .map_err(|e| { + tracing::error!("delete_account check tables: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + if has_tables > 0 { + return Err(OpError::HasDependents( + "Cannot delete account with existing tables. Delete all tables first." + .to_owned(), + )); + } + + let coll = self.catalog_db().collection::("accounts"); + let result = coll + .delete_one(doc! { "account_id": &account_id }) + .await + .map_err(|e| { + tracing::error!("delete_account: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + if result.deleted_count == 0 { + return Err(OpError::NotFound("Account not found".to_owned())); + } + Ok(()) + }) + } + + fn list_all_accounts(&self) -> BoxFuture<'_, OpResult>> { + Box::pin(async { + let coll = self.catalog_db().collection::("accounts"); + let opts = FindOptions::builder() + .sort(doc! { "account_id": 1 }) + .build(); + let cursor = coll.find(doc! {}).with_options(opts).await.map_err(|e| { + tracing::error!("list_all_accounts: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_all_accounts cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("account_id").ok()?.to_owned(), + d.get_str("account_name").ok()?.to_owned(), + )) + }) + .collect()) + }) + } + + fn list_all_accounts_full( + &self, + ) -> BoxFuture<'_, OpResult>> { + Box::pin(async { + let coll = self.catalog_db().collection::("accounts"); + let opts = FindOptions::builder() + .sort(doc! { "account_id": 1 }) + .build(); + let cursor = coll.find(doc! {}).with_options(opts).await.map_err(|e| { + tracing::error!("list_all_accounts_full: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_all_accounts_full cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("account_id").ok()?.to_owned(), + d.get_str("account_name").ok()?.to_owned(), + to_offset_dt(d.get_datetime("created_at").ok()?.to_owned()), + )) + }) + .collect()) + }) + } + + fn list_accounts_for( + &self, + account_id: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("accounts"); + let cursor = coll + .find(doc! { "account_id": &account_id }) + .await + .map_err(|e| { + tracing::error!("list_accounts_for: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_accounts_for cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("account_id").ok()?.to_owned(), + d.get_str("account_name").ok()?.to_owned(), + )) + }) + .collect()) + }) + } + + fn get_account_detail( + &self, + account_id: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("accounts"); + let acct = coll + .find_one(doc! { "account_id": &account_id }) + .await + .map_err(|e| { + tracing::error!("get_account_detail: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let Some(acct_doc) = acct else { + return Ok(None); + }; + + let account_name = acct_doc + .get_str("account_name") + .unwrap_or_default() + .to_owned(); + + let users_coll = self.catalog_db().collection::("iam_users"); + let users_cursor = users_coll + .find(doc! { "account_id": &account_id }) + .with_options(FindOptions::builder().sort(doc! { "user_name": 1 }).build()) + .await + .map_err(|e| { + tracing::error!("get_account_detail users: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let users: Vec = users_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_account_detail users cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + d.get_str("user_name") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + .collect(); + + let groups_coll = self.catalog_db().collection::("iam_groups"); + let groups_cursor = groups_coll + .find(doc! { "account_id": &account_id }) + .with_options( + FindOptions::builder() + .sort(doc! { "group_name": 1 }) + .build(), + ) + .await + .map_err(|e| { + tracing::error!("get_account_detail groups: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let groups: Vec = groups_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_account_detail groups cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + d.get_str("group_name") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + .collect(); + + let roles_coll = self.catalog_db().collection::("iam_roles"); + let roles_cursor = roles_coll + .find(doc! { "account_id": &account_id }) + .with_options(FindOptions::builder().sort(doc! { "role_name": 1 }).build()) + .await + .map_err(|e| { + tracing::error!("get_account_detail roles: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let roles: Vec = roles_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_account_detail roles cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + d.get_str("role_name") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + .collect(); + + Ok(Some(AccountDetail { + account_name, + users, + groups, + roles, + })) + }) + } + + fn dashboard_counts(&self) -> BoxFuture<'_, OpResult<(i64, i64)>> { + Box::pin(async { + let accounts_coll = self.catalog_db().collection::("accounts"); + let account_count = accounts_coll.count_documents(doc! {}).await.map_err(|e| { + tracing::error!("dashboard_counts accounts: {e}"); + OpError::Internal("Database error".to_owned()) + })? as i64; + + let admins_coll = self.catalog_db().collection::("admin_users"); + let admin_count = admins_coll.count_documents(doc! {}).await.map_err(|e| { + tracing::error!("dashboard_counts admins: {e}"); + OpError::Internal("Database error".to_owned()) + })? as i64; + + Ok((account_count, admin_count)) + }) + } + + fn create_user( + &self, + account_id: &str, + user_name: &str, + password_hash: Option<&str>, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let password_hash = password_hash.map(std::borrow::ToOwned::to_owned); + Box::pin(async move { + let user_arn = format!("arn:aws:iam::{account_id}:user/{user_name}"); + + let mut user_doc = doc! { + "account_id": &account_id, + "user_name": &user_name, + "user_arn": &user_arn, + "created_at": now_bson(), + }; + if let Some(ref ph) = password_hash { + user_doc.insert("password_hash", ph.as_str()); + } + + let coll = self.catalog_db().collection::("iam_users"); + let result = coll.insert_one(user_doc).await; + match result { + Ok(_) => {} + Err(e) if is_duplicate_key(&e) => { + return Err(OpError::AlreadyExists("IAM user already exists".to_owned())); + } + Err(e) => { + tracing::error!("create_user failed: {e}"); + return Err(OpError::Internal("Database error".to_owned())); + } + } + + // Seed default self-service policy. + let self_service_policy = serde_json::json!({ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": [ + "iam:CreateAccessKey", + "iam:DeleteAccessKey", + "iam:ListAccessKeys", + "iam:ChangePassword" + ], + "Resource": format!("arn:aws:iam::{}:user/{}", account_id, user_name) + }] + }); + + let policies_coll = self.catalog_db().collection::("iam_policies"); + let policy_doc = doc! { + "account_id": &account_id, + "principal_type": "user", + "principal_name": &user_name, + "policy_name": "SelfServicePolicy", + "policy_document": bson::to_bson(&self_service_policy).unwrap_or_default(), + "created_at": now_bson(), + }; + // Use upsert to avoid errors on conflict + let filter = doc! { + "account_id": &account_id, + "principal_type": "user", + "principal_name": &user_name, + "policy_name": "SelfServicePolicy", + }; + let opts = UpdateOptions::builder().upsert(true).build(); + if let Err(e) = policies_coll + .update_one(filter, doc! { "$setOnInsert": policy_doc }) + .with_options(opts) + .await + { + tracing::error!("seed self-service policy failed: {e}"); + return Err(OpError::Internal("Database error".to_owned())); + } + + Ok(()) + }) + } + + fn delete_user(&self, account_id: &str, user_name: &str) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_users"); + let result = coll + .delete_one(doc! { "account_id": &account_id, "user_name": &user_name }) + .await + .map_err(|e| { + tracing::error!("delete_user failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.deleted_count == 0 { + return Err(OpError::NotFound("IAM user not found".to_owned())); + } + // Cascade: delete access keys, policies, group memberships + let keys_coll = self.catalog_db().collection::("access_keys"); + let _ = keys_coll + .delete_many(doc! { "account_id": &account_id, "user_name": &user_name }) + .await; + let policies_coll = self.catalog_db().collection::("iam_policies"); + let _ = policies_coll + .delete_many(doc! { "account_id": &account_id, "principal_type": "user", "principal_name": &user_name }) + .await; + let members_coll = self + .catalog_db() + .collection::("iam_group_members"); + let _ = members_coll + .delete_many(doc! { "account_id": &account_id, "user_name": &user_name }) + .await; + Ok(()) + }) + } + + fn list_users(&self, account_id: &str) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_users"); + let opts = FindOptions::builder().sort(doc! { "user_name": 1 }).build(); + let cursor = coll + .find(doc! { "account_id": &account_id }) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("list_users: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_users cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("account_id").ok()?.to_owned(), + d.get_str("user_name").ok()?.to_owned(), + d.get_str("user_arn").ok()?.to_owned(), + d.get_str("password_hash").is_ok(), + to_offset_dt(d.get_datetime("created_at").ok()?.to_owned()), + )) + }) + .collect()) + }) + } + + fn get_user_detail( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_users"); + let exists = coll + .find_one(doc! { "account_id": &account_id, "user_name": &user_name }) + .await + .map_err(|e| { + tracing::error!("get_user_detail exists: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if exists.is_none() { + return Ok(None); + } + + let keys_coll = self.catalog_db().collection::("access_keys"); + let keys_cursor = keys_coll + .find(doc! { "account_id": &account_id, "user_name": &user_name }) + .with_options( + FindOptions::builder() + .sort(doc! { "access_key_id": 1 }) + .build(), + ) + .await + .map_err(|e| { + tracing::error!("get_user_detail keys: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let keys: Vec<(String, bool)> = keys_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_user_detail keys cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("access_key_id").ok()?.to_owned(), + d.get_bool("is_active").unwrap_or(true), + )) + }) + .collect(); + + let policies_coll = self.catalog_db().collection::("iam_policies"); + let policies_cursor = policies_coll + .find(doc! { + "account_id": &account_id, + "principal_type": "user", + "principal_name": &user_name, + }) + .with_options( + FindOptions::builder() + .sort(doc! { "policy_name": 1 }) + .build(), + ) + .await + .map_err(|e| { + tracing::error!("get_user_detail policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let policies: Vec = policies_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_user_detail policies cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + d.get_str("policy_name") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + .collect(); + + let tags_coll = self.catalog_db().collection::("iam_user_tags"); + let tags_cursor = tags_coll + .find(doc! { "account_id": &account_id, "user_name": &user_name }) + .with_options(FindOptions::builder().sort(doc! { "tag_key": 1 }).build()) + .await + .map_err(|e| { + tracing::error!("get_user_detail tags: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let tags: Vec<(String, String)> = tags_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_user_detail tags cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect(); + + let members_coll = self + .catalog_db() + .collection::("iam_group_members"); + let groups_cursor = members_coll + .find(doc! { "account_id": &account_id, "user_name": &user_name }) + .with_options( + FindOptions::builder() + .sort(doc! { "group_name": 1 }) + .build(), + ) + .await + .map_err(|e| { + tracing::error!("get_user_detail groups: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let groups: Vec = groups_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_user_detail groups cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + d.get_str("group_name") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + .collect(); + + Ok(Some(UserDetail { + keys, + policies, + tags, + groups, + })) + }) + } + + fn verify_iam_user_password( + &self, + account_id: &str, + user_name: &str, + password: &str, + ) -> BoxFuture<'_, OpResult> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let password = password.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_users"); + let doc = coll + .find_one(doc! { "account_id": &account_id, "user_name": &user_name }) + .await + .map_err(|e| { + tracing::error!("verify_iam_user_password: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let Some(user_doc) = doc else { + return Ok(false); + }; + + let Some(hash) = user_doc.get_str("password_hash").ok() else { + return Ok(false); + }; + + let hash = hash.to_owned(); + Ok(tokio::task::spawn_blocking(move || { + bcrypt::verify(password, &hash).unwrap_or(false) + }) + .await + .unwrap_or(false)) + }) + } + + fn change_user_password( + &self, + account_id: &str, + user_name: &str, + password_hash: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let password_hash = password_hash.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_users"); + let result = coll + .update_one( + doc! { "account_id": &account_id, "user_name": &user_name }, + doc! { "$set": { "password_hash": &password_hash } }, + ) + .await + .map_err(|e| { + tracing::error!("change_user_password failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.matched_count == 0 { + return Err(OpError::NotFound("IAM user not found".to_owned())); + } + Ok(()) + }) + } + + fn tag_user( + &self, + account_id: &str, + user_name: &str, + tags: &[(String, String)], + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let tags = tags.to_vec(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_user_tags"); + for (key, value) in &tags { + let filter = doc! { + "account_id": &account_id, + "user_name": &user_name, + "tag_key": key, + }; + let update = doc! { + "$set": { + "account_id": &account_id, + "user_name": &user_name, + "tag_key": key, + "tag_value": value, + } + }; + let opts = UpdateOptions::builder().upsert(true).build(); + coll.update_one(filter, update) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("tag_user failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + } + Ok(()) + }) + } + + fn untag_user( + &self, + account_id: &str, + user_name: &str, + tag_keys: &[String], + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let tag_keys = tag_keys.to_vec(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_user_tags"); + for key in &tag_keys { + coll.delete_one(doc! { + "account_id": &account_id, + "user_name": &user_name, + "tag_key": key, + }) + .await + .map_err(|e| { + tracing::error!("untag_user failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + } + Ok(()) + }) + } + + fn list_user_tags( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_user_tags"); + let opts = FindOptions::builder().sort(doc! { "tag_key": 1 }).build(); + let cursor = coll + .find(doc! { "account_id": &account_id, "user_name": &user_name }) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("list_user_tags: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_user_tags cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect()) + }) + } + + fn create_group(&self, account_id: &str, group_name: &str) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let group_name = group_name.to_owned(); + Box::pin(async move { + let group_arn = format!("arn:aws:iam::{account_id}:group/{group_name}"); + let coll = self.catalog_db().collection::("iam_groups"); + let result = coll + .insert_one(doc! { + "account_id": &account_id, + "group_name": &group_name, + "group_arn": &group_arn, + "created_at": now_bson(), + }) + .await; + match result { + Ok(_) => Ok(()), + Err(e) if is_duplicate_key(&e) => Err(OpError::AlreadyExists( + "IAM group already exists".to_owned(), + )), + Err(e) => { + tracing::error!("create_group failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + }) + } + + fn delete_group(&self, account_id: &str, group_name: &str) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let group_name = group_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_groups"); + let result = coll + .delete_one(doc! { "account_id": &account_id, "group_name": &group_name }) + .await + .map_err(|e| { + tracing::error!("delete_group failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.deleted_count == 0 { + return Err(OpError::NotFound("IAM group not found".to_owned())); + } + Ok(()) + }) + } + + fn list_groups(&self, account_id: &str) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_groups"); + let opts = FindOptions::builder() + .sort(doc! { "group_name": 1 }) + .build(); + let cursor = coll + .find(doc! { "account_id": &account_id }) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("list_groups: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_groups cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("account_id").ok()?.to_owned(), + d.get_str("group_name").ok()?.to_owned(), + d.get_str("group_arn").ok()?.to_owned(), + to_offset_dt(d.get_datetime("created_at").ok()?.to_owned()), + )) + }) + .collect()) + }) + } + + fn get_group_detail( + &self, + account_id: &str, + group_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let group_name = group_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_groups"); + let exists = coll + .find_one(doc! { "account_id": &account_id, "group_name": &group_name }) + .await + .map_err(|e| { + tracing::error!("get_group_detail exists: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if exists.is_none() { + return Ok(None); + } + + let members_coll = self + .catalog_db() + .collection::("iam_group_members"); + let members_cursor = members_coll + .find(doc! { "account_id": &account_id, "group_name": &group_name }) + .with_options(FindOptions::builder().sort(doc! { "user_name": 1 }).build()) + .await + .map_err(|e| { + tracing::error!("get_group_detail members: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let members: Vec = members_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_group_detail members cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + d.get_str("user_name") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + .collect(); + + let policies_coll = self.catalog_db().collection::("iam_policies"); + let policies_cursor = policies_coll + .find(doc! { + "account_id": &account_id, + "principal_type": "group", + "principal_name": &group_name, + }) + .with_options( + FindOptions::builder() + .sort(doc! { "policy_name": 1 }) + .build(), + ) + .await + .map_err(|e| { + tracing::error!("get_group_detail policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let policies: Vec = policies_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_group_detail policies cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + d.get_str("policy_name") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + .collect(); + + let users_coll = self.catalog_db().collection::("iam_users"); + let all_users_cursor = users_coll + .find(doc! { "account_id": &account_id }) + .with_options(FindOptions::builder().sort(doc! { "user_name": 1 }).build()) + .await + .map_err(|e| { + tracing::error!("get_group_detail all_users: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let all_users: Vec = all_users_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_group_detail all_users cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + d.get_str("user_name") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + .collect(); + + Ok(Some(GroupDetail { + members, + policies, + all_users, + })) + }) + } + + fn add_group_member( + &self, + account_id: &str, + group_name: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let group_name = group_name.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_group_members"); + let result = coll + .insert_one(doc! { + "account_id": &account_id, + "group_name": &group_name, + "user_name": &user_name, + }) + .await; + match result { + Ok(_) => Ok(()), + Err(e) if is_duplicate_key(&e) => Err(OpError::AlreadyExists( + "User is already a member of this group".to_owned(), + )), + Err(e) => { + tracing::error!("add_group_member failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + }) + } + + fn remove_group_member( + &self, + account_id: &str, + group_name: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let group_name = group_name.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_group_members"); + let result = coll + .delete_one(doc! { + "account_id": &account_id, + "group_name": &group_name, + "user_name": &user_name, + }) + .await + .map_err(|e| { + tracing::error!("remove_group_member failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.deleted_count == 0 { + return Err(OpError::NotFound("Membership not found".to_owned())); + } + Ok(()) + }) + } + + fn create_role( + &self, + account_id: &str, + role_name: &str, + trust_policy: &serde_json::Value, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let trust_policy = trust_policy.clone(); + Box::pin(async move { + let role_arn = format!("arn:aws:iam::{account_id}:role/{role_name}"); + let coll = self.catalog_db().collection::("iam_roles"); + let result = coll + .insert_one(doc! { + "account_id": &account_id, + "role_name": &role_name, + "role_arn": &role_arn, + "trust_policy": bson::to_bson(&trust_policy).unwrap_or_default(), + "created_at": now_bson(), + }) + .await; + match result { + Ok(_) => Ok(()), + Err(e) if is_duplicate_key(&e) => { + Err(OpError::AlreadyExists("IAM role already exists".to_owned())) + } + Err(e) => { + tracing::error!("create_role failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + }) + } + + fn delete_role(&self, account_id: &str, role_name: &str) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_roles"); + let result = coll + .delete_one(doc! { "account_id": &account_id, "role_name": &role_name }) + .await + .map_err(|e| { + tracing::error!("delete_role failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.deleted_count == 0 { + return Err(OpError::NotFound("IAM role not found".to_owned())); + } + Ok(()) + }) + } + + fn list_roles(&self, account_id: &str) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_roles"); + let opts = FindOptions::builder().sort(doc! { "role_name": 1 }).build(); + let cursor = coll + .find(doc! { "account_id": &account_id }) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("list_roles: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_roles cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + let tp_bson = d.get("trust_policy")?; + let trust_policy: serde_json::Value = bson::from_bson(tp_bson.clone()).ok()?; + Some(( + d.get_str("account_id").ok()?.to_owned(), + d.get_str("role_name").ok()?.to_owned(), + d.get_str("role_arn").ok()?.to_owned(), + trust_policy, + to_offset_dt(d.get_datetime("created_at").ok()?.to_owned()), + )) + }) + .collect()) + }) + } + + fn get_role_detail( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_roles"); + let role_doc = coll + .find_one(doc! { "account_id": &account_id, "role_name": &role_name }) + .await + .map_err(|e| { + tracing::error!("get_role_detail role: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let Some(role_doc) = role_doc else { + return Ok(None); + }; + + let trust_policy: serde_json::Value = role_doc + .get("trust_policy") + .and_then(|b| bson::from_bson(b.clone()).ok()) + .unwrap_or(serde_json::Value::Null); + + let policies_coll = self.catalog_db().collection::("iam_policies"); + let policies_cursor = policies_coll + .find(doc! { + "account_id": &account_id, + "principal_type": "role", + "principal_name": &role_name, + }) + .with_options( + FindOptions::builder() + .sort(doc! { "policy_name": 1 }) + .build(), + ) + .await + .map_err(|e| { + tracing::error!("get_role_detail policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let policies: Vec = policies_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_role_detail policies cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + d.get_str("policy_name") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + .collect(); + + let tags_coll = self.catalog_db().collection::("iam_role_tags"); + let tags_cursor = tags_coll + .find(doc! { "account_id": &account_id, "role_name": &role_name }) + .with_options(FindOptions::builder().sort(doc! { "tag_key": 1 }).build()) + .await + .map_err(|e| { + tracing::error!("get_role_detail tags: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let tags: Vec<(String, String)> = tags_cursor + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("get_role_detail tags cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })? + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect(); + + Ok(Some(RoleDetail { + trust_policy, + policies, + tags, + })) + }) + } + + fn get_role_trust_policy( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_roles"); + let doc = coll + .find_one(doc! { "account_id": &account_id, "role_name": &role_name }) + .await + .map_err(|e| { + tracing::error!("get_role_trust_policy: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(doc.and_then(|d| { + d.get("trust_policy") + .and_then(|b| bson::from_bson(b.clone()).ok()) + })) + }) + } + + fn tag_role( + &self, + account_id: &str, + role_name: &str, + tags: &[(String, String)], + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let tags = tags.to_vec(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_role_tags"); + for (key, value) in &tags { + let filter = doc! { + "account_id": &account_id, + "role_name": &role_name, + "tag_key": key, + }; + let update = doc! { + "$set": { + "account_id": &account_id, + "role_name": &role_name, + "tag_key": key, + "tag_value": value, + } + }; + let opts = UpdateOptions::builder().upsert(true).build(); + coll.update_one(filter, update) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("tag_role failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + } + Ok(()) + }) + } + + fn untag_role( + &self, + account_id: &str, + role_name: &str, + tag_keys: &[String], + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let tag_keys = tag_keys.to_vec(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_role_tags"); + for key in &tag_keys { + coll.delete_one(doc! { + "account_id": &account_id, + "role_name": &role_name, + "tag_key": key, + }) + .await + .map_err(|e| { + tracing::error!("untag_role failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + } + Ok(()) + }) + } + + fn list_role_tags( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_role_tags"); + let opts = FindOptions::builder().sort(doc! { "tag_key": 1 }).build(); + let cursor = coll + .find(doc! { "account_id": &account_id, "role_name": &role_name }) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("list_role_tags: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_role_tags cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect()) + }) + } + + fn put_policy( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + policy_name: &str, + document: &serde_json::Value, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let principal_type = principal_type.to_owned(); + let principal_name = principal_name.to_owned(); + let policy_name = policy_name.to_owned(); + let document = document.clone(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_policies"); + let filter = doc! { + "account_id": &account_id, + "principal_type": &principal_type, + "principal_name": &principal_name, + "policy_name": &policy_name, + }; + let update = doc! { + "$set": { + "account_id": &account_id, + "principal_type": &principal_type, + "principal_name": &principal_name, + "policy_name": &policy_name, + "policy_document": bson::to_bson(&document).unwrap_or_default(), + }, + "$setOnInsert": { + "created_at": now_bson(), + } + }; + let opts = UpdateOptions::builder().upsert(true).build(); + coll.update_one(filter, update) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("put_policy failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(()) + }) + } + + fn delete_policy( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + policy_name: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let principal_type = principal_type.to_owned(); + let principal_name = principal_name.to_owned(); + let policy_name = policy_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_policies"); + let result = coll + .delete_one(doc! { + "account_id": &account_id, + "principal_type": &principal_type, + "principal_name": &principal_name, + "policy_name": &policy_name, + }) + .await + .map_err(|e| { + tracing::error!("delete_policy failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.deleted_count == 0 { + return Err(OpError::NotFound("Policy not found".to_owned())); + } + Ok(()) + }) + } + + fn list_policies( + &self, + account_id: &str, + principal_type: &str, + principal_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let principal_type = principal_type.to_owned(); + let principal_name = principal_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_policies"); + let opts = FindOptions::builder() + .sort(doc! { "policy_name": 1 }) + .build(); + let cursor = coll + .find(doc! { + "account_id": &account_id, + "principal_type": &principal_type, + "principal_name": &principal_name, + }) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("list_policies: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_policies cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + let policy_name = d.get_str("policy_name").ok()?.to_owned(); + let policy_document: serde_json::Value = d + .get("policy_document") + .and_then(|b| bson::from_bson(b.clone()).ok())?; + let created_at = to_offset_dt(d.get_datetime("created_at").ok()?.to_owned()); + Some((policy_name, policy_document, created_at)) + }) + .collect()) + }) + } + + fn set_user_boundary( + &self, + account_id: &str, + user_name: &str, + document: &serde_json::Value, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let document = document.clone(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_permissions_boundaries"); + let filter = doc! { + "account_id": &account_id, + "principal_type": "user", + "principal_name": &user_name, + }; + let update = doc! { + "$set": { + "account_id": &account_id, + "principal_type": "user", + "principal_name": &user_name, + "policy_document": bson::to_bson(&document).unwrap_or_default(), + } + }; + let opts = UpdateOptions::builder().upsert(true).build(); + coll.update_one(filter, update) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("set_user_boundary failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(()) + }) + } + + fn get_user_boundary( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_permissions_boundaries"); + let doc = coll + .find_one(doc! { + "account_id": &account_id, + "principal_type": "user", + "principal_name": &user_name, + }) + .await + .map_err(|e| { + tracing::error!("get_user_boundary: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(doc.and_then(|d| { + d.get("policy_document") + .and_then(|b| bson::from_bson(b.clone()).ok()) + })) + }) + } + + fn delete_user_boundary( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_permissions_boundaries"); + let result = coll + .delete_one(doc! { + "account_id": &account_id, + "principal_type": "user", + "principal_name": &user_name, + }) + .await + .map_err(|e| { + tracing::error!("delete_user_boundary failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.deleted_count == 0 { + return Err(OpError::NotFound("Permissions boundary not set".to_owned())); + } + Ok(()) + }) + } + + fn set_role_boundary( + &self, + account_id: &str, + role_name: &str, + document: &serde_json::Value, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let document = document.clone(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_permissions_boundaries"); + let filter = doc! { + "account_id": &account_id, + "principal_type": "role", + "principal_name": &role_name, + }; + let update = doc! { + "$set": { + "account_id": &account_id, + "principal_type": "role", + "principal_name": &role_name, + "policy_document": bson::to_bson(&document).unwrap_or_default(), + } + }; + let opts = UpdateOptions::builder().upsert(true).build(); + coll.update_one(filter, update) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("set_role_boundary failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(()) + }) + } + + fn get_role_boundary( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_permissions_boundaries"); + let doc = coll + .find_one(doc! { + "account_id": &account_id, + "principal_type": "role", + "principal_name": &role_name, + }) + .await + .map_err(|e| { + tracing::error!("get_role_boundary: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(doc.and_then(|d| { + d.get("policy_document") + .and_then(|b| bson::from_bson(b.clone()).ok()) + })) + }) + } + + fn delete_role_boundary( + &self, + account_id: &str, + role_name: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + Box::pin(async move { + let coll = self + .catalog_db() + .collection::("iam_permissions_boundaries"); + let result = coll + .delete_one(doc! { + "account_id": &account_id, + "principal_type": "role", + "principal_name": &role_name, + }) + .await + .map_err(|e| { + tracing::error!("delete_role_boundary failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.deleted_count == 0 { + return Err(OpError::NotFound("Permissions boundary not set".to_owned())); + } + Ok(()) + }) + } + + fn create_access_key( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + // Check user exists (MongoDB has no FK constraints) + let users_coll = self.catalog_db().collection::("iam_users"); + let user_exists = users_coll + .find_one(doc! { "account_id": &account_id, "user_name": &user_name }) + .await + .map_err(|e| { + tracing::error!("create_access_key user check: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if user_exists.is_none() { + return Err(OpError::NotFound("User not found".to_owned())); + } + + let enc_key = self.get_encryption_key().await?; + + let access_key_id = generate_access_key_id(); + let secret_key = generate_secret_key(); + let encrypted = encrypt_secret(&secret_key, &enc_key, &access_key_id).map_err(|e| { + tracing::error!("create_access_key encryption: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let coll = self.catalog_db().collection::("access_keys"); + coll.insert_one(doc! { + "access_key_id": &access_key_id, + "account_id": &account_id, + "user_name": &user_name, + "secret_key_encrypted": Binary { subtype: bson::spec::BinarySubtype::Generic, bytes: encrypted }, + "is_active": true, + "created_at": now_bson(), + }) + .await + .map_err(|e| { + if is_duplicate_key(&e) { + OpError::NotFound("User not found".to_owned()) + } else { + tracing::error!("create_access_key failed: {e}"); + OpError::Internal("Database error".to_owned()) + } + })?; + + Ok(AccessKeyCreated { + access_key_id, + secret_access_key: secret_key, + }) + }) + } + + fn delete_access_key( + &self, + account_id: &str, + user_name: &str, + key_id: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let key_id = key_id.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("access_keys"); + let result = coll + .delete_one(doc! { + "access_key_id": &key_id, + "account_id": &account_id, + "user_name": &user_name, + }) + .await + .map_err(|e| { + tracing::error!("delete_access_key failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.deleted_count == 0 { + return Err(OpError::NotFound("Access key not found".to_owned())); + } + Ok(()) + }) + } + + fn list_access_keys( + &self, + account_id: &str, + user_name: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("access_keys"); + let opts = FindOptions::builder() + .sort(doc! { "created_at": 1 }) + .build(); + let cursor = coll + .find(doc! { "account_id": &account_id, "user_name": &user_name }) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("list_access_keys: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_access_keys cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("access_key_id").ok()?.to_owned(), + d.get_bool("is_active").unwrap_or(true), + to_offset_dt(d.get_datetime("created_at").ok()?.to_owned()), + )) + }) + .collect()) + }) + } + + fn import_access_key( + &self, + account_id: &str, + user_name: &str, + access_key_id: &str, + secret_access_key: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let account_id = account_id.to_owned(); + let user_name = user_name.to_owned(); + let access_key_id = access_key_id.to_owned(); + let secret_access_key = secret_access_key.to_owned(); + Box::pin(async move { + let enc_key = self.get_encryption_key().await?; + + let encrypted = + encrypt_secret(&secret_access_key, &enc_key, &access_key_id).map_err(|e| { + tracing::error!("import_access_key encryption: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + + let coll = self.catalog_db().collection::("access_keys"); + let result = coll + .insert_one(doc! { + "access_key_id": &access_key_id, + "account_id": &account_id, + "user_name": &user_name, + "secret_key_encrypted": Binary { subtype: bson::spec::BinarySubtype::Generic, bytes: encrypted }, + "is_active": true, + "created_at": now_bson(), + }) + .await; + match result { + Ok(_) => Ok(()), + Err(e) if is_duplicate_key(&e) => Err(OpError::AlreadyExists( + "Access key ID already exists".to_owned(), + )), + Err(e) => { + tracing::error!("import_access_key failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + }) + } + + fn store_session( + &self, + session_token: &str, + access_key_id: &str, + secret_key_encrypted: &[u8], + account_id: &str, + role_name: &str, + session_name: &str, + session_tags: &Option, + session_policy: &Option, + expires_at: time::OffsetDateTime, + ) -> BoxFuture<'_, OpResult<()>> { + let session_token = session_token.to_owned(); + let access_key_id = access_key_id.to_owned(); + let secret_key_encrypted = secret_key_encrypted.to_vec(); + let account_id = account_id.to_owned(); + let role_name = role_name.to_owned(); + let session_name = session_name.to_owned(); + let session_tags = session_tags.clone(); + let session_policy = session_policy.clone(); + Box::pin(async move { + let coll = self.catalog_db().collection::("iam_sessions"); + let expires_bson = BsonDateTime::from_millis(expires_at.unix_timestamp() * 1000); + + let mut session_doc = doc! { + "session_token": &session_token, + "access_key_id": &access_key_id, + "secret_key_encrypted": Binary { subtype: bson::spec::BinarySubtype::Generic, bytes: secret_key_encrypted }, + "account_id": &account_id, + "role_name": &role_name, + "session_name": &session_name, + "expires_at": expires_bson, + }; + + if let Some(ref tags) = session_tags { + session_doc.insert("session_tags", bson::to_bson(tags).unwrap_or_default()); + } + if let Some(ref policy) = session_policy { + session_doc.insert("session_policy", bson::to_bson(policy).unwrap_or_default()); + } + + coll.insert_one(session_doc).await.map_err(|e| { + tracing::error!("store_session failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(()) + }) + } + + fn fetch_caller_tags( + &self, + account_id: &str, + resource: &str, + ) -> BoxFuture<'_, OpResult>> { + let account_id = account_id.to_owned(); + let resource = resource.to_owned(); + Box::pin(async move { + if let Some(user_name) = resource.strip_prefix("user/") { + let coll = self.catalog_db().collection::("iam_user_tags"); + let cursor = coll + .find(doc! { "account_id": &account_id, "user_name": user_name }) + .await + .map_err(|e| { + tracing::error!("fetch_caller_tags user: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_caller_tags user cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect()) + } else if let Some(role_name) = resource.strip_prefix("role/") { + let coll = self.catalog_db().collection::("iam_role_tags"); + let cursor = coll + .find(doc! { "account_id": &account_id, "role_name": role_name }) + .await + .map_err(|e| { + tracing::error!("fetch_caller_tags role: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_caller_tags role cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect()) + } else if let Some(rest) = resource.strip_prefix("assumed-role/") { + let role_name = rest.split('/').next().unwrap_or(""); + if role_name.is_empty() { + return Ok(Vec::new()); + } + let coll = self.catalog_db().collection::("iam_role_tags"); + let cursor = coll + .find(doc! { "account_id": &account_id, "role_name": role_name }) + .await + .map_err(|e| { + tracing::error!("fetch_caller_tags assumed-role: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("fetch_caller_tags assumed-role cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("tag_key").ok()?.to_owned(), + d.get_str("tag_value").ok()?.to_owned(), + )) + }) + .collect()) + } else { + Ok(Vec::new()) + } + }) + } +} + +// ── SettingsStore ─────────────────────────────────────────────────────── + +impl extenddb_storage::management_store::SettingsStore for MongoCatalogStore { + fn get_setting(&self, key: &str) -> BoxFuture<'_, OpResult>> { + let key = key.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("settings"); + let doc = coll.find_one(doc! { "_id": &key }).await.map_err(|e| { + tracing::error!("get_setting: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(doc.and_then(|d| d.get_str("value").ok().map(std::borrow::ToOwned::to_owned))) + }) + } + + fn set_setting(&self, key: &str, value: &str) -> BoxFuture<'_, OpResult<()>> { + let key = key.to_owned(); + let value = value.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("settings"); + let filter = doc! { "_id": &key }; + let update = doc! { "$set": { "value": &value } }; + let opts = UpdateOptions::builder().upsert(true).build(); + coll.update_one(filter, update) + .with_options(opts) + .await + .map_err(|e| { + tracing::error!("set_setting failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(()) + }) + } + + fn list_settings(&self) -> BoxFuture<'_, OpResult>> { + Box::pin(async { + let coll = self.catalog_db().collection::("settings"); + let opts = FindOptions::builder().sort(doc! { "_id": 1 }).build(); + let cursor = coll.find(doc! {}).with_options(opts).await.map_err(|e| { + tracing::error!("list_settings: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_settings cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(( + d.get_str("_id").ok()?.to_owned(), + d.get_str("value").ok()?.to_owned(), + )) + }) + .collect()) + }) + } + + fn cached_encryption_key(&self) -> Option { + self.encryption_key.clone() + } +} + +// ── MetricsStore ──────────────────────────────────────────────────────── + +impl extenddb_storage::management_store::MetricsStore for MongoCatalogStore { + fn insert_metrics(&self, rows: &[MetricsRow]) -> BoxFuture<'_, OpResult<()>> { + let rows = rows.to_vec(); + Box::pin(async move { + if rows.is_empty() { + return Ok(()); + } + let coll = self.catalog_db().collection::("metrics"); + let docs: Vec = rows + .into_iter() + .map(|r| { + let mut d = doc! { + "bucket": BsonDateTime::from_millis(r.bucket.unix_timestamp() * 1000), + "metric": &r.metric, + "sum": r.sum, + "count": r.count, + "min": r.min, + "max": r.max, + }; + if let Some(ref tn) = r.table_name { + d.insert("table_name", tn.as_str()); + } + if let Some(ref idx) = r.index_name { + d.insert("index_name", idx.as_str()); + } + if let Some(ref op) = r.operation { + d.insert("operation", op.as_str()); + } + d + }) + .collect(); + coll.insert_many(docs).await.map_err(|e| { + tracing::error!("insert_metrics failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(()) + }) + } + + fn query_metrics( + &self, + start: time::OffsetDateTime, + end: time::OffsetDateTime, + table_name: Option<&str>, + metric: Option<&str>, + ) -> BoxFuture<'_, OpResult>> { + let table_name = table_name.map(std::borrow::ToOwned::to_owned); + let metric = metric.map(std::borrow::ToOwned::to_owned); + Box::pin(async move { + let coll = self.catalog_db().collection::("metrics"); + let start_bson = BsonDateTime::from_millis(start.unix_timestamp() * 1000); + let end_bson = BsonDateTime::from_millis(end.unix_timestamp() * 1000); + + let mut filter = doc! { + "bucket": { "$gte": start_bson, "$lte": end_bson } + }; + if let Some(ref tn) = table_name { + filter.insert("table_name", tn.as_str()); + } + if let Some(ref m) = metric { + filter.insert("metric", m.as_str()); + } + + let cursor = coll.find(filter).await.map_err(|e| { + tracing::error!("query_metrics: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("query_metrics cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(MetricsRow { + bucket: to_offset_dt(d.get_datetime("bucket").ok()?.to_owned()), + metric: d.get_str("metric").ok()?.to_owned(), + table_name: d + .get_str("table_name") + .ok() + .map(std::borrow::ToOwned::to_owned), + index_name: d + .get_str("index_name") + .ok() + .map(std::borrow::ToOwned::to_owned), + operation: d + .get_str("operation") + .ok() + .map(std::borrow::ToOwned::to_owned), + sum: d.get_f64("sum").ok()?, + count: d + .get_i64("count") + .ok() + .or_else(|| d.get_i32("count").ok().map(i64::from))?, + min: d.get_f64("min").ok()?, + max: d.get_f64("max").ok()?, + }) + }) + .collect()) + }) + } + + fn prune_metrics(&self, retention: std::time::Duration) -> BoxFuture<'_, OpResult<()>> { + Box::pin(async move { + let coll = self.catalog_db().collection::("metrics"); + let cutoff = time::OffsetDateTime::now_utc() - retention; + let cutoff_bson = BsonDateTime::from_millis(cutoff.unix_timestamp() * 1000); + coll.delete_many(doc! { "bucket": { "$lt": cutoff_bson } }) + .await + .map_err(|e| { + tracing::error!("prune_metrics failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(()) + }) + } +} + +// ── RateLimitStore ────────────────────────────────────────────────────── + +impl extenddb_storage::management_store::RateLimitStore for MongoCatalogStore { + fn count_principal_failures( + &self, + principal: &str, + window_seconds: i64, + ) -> BoxFuture<'_, OpResult> { + let principal = principal.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("failed_logins"); + let cutoff = time::OffsetDateTime::now_utc() + - std::time::Duration::from_secs(window_seconds as u64); + let cutoff_bson = BsonDateTime::from_millis(cutoff.unix_timestamp() * 1000); + let count = coll + .count_documents(doc! { + "principal": &principal, + "attempted_at": { "$gte": cutoff_bson }, + }) + .await + .map_err(|e| { + tracing::error!("count_principal_failures: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(count as i64) + }) + } + + fn count_ip_failures( + &self, + source_ip: &str, + window_seconds: i64, + ) -> BoxFuture<'_, OpResult> { + let source_ip = source_ip.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("failed_logins"); + let cutoff = time::OffsetDateTime::now_utc() + - std::time::Duration::from_secs(window_seconds as u64); + let cutoff_bson = BsonDateTime::from_millis(cutoff.unix_timestamp() * 1000); + let count = coll + .count_documents(doc! { + "source_ip": &source_ip, + "attempted_at": { "$gte": cutoff_bson }, + }) + .await + .map_err(|e| { + tracing::error!("count_ip_failures: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(count as i64) + }) + } + + fn record_failed_login(&self, principal: &str, source_ip: Option<&str>) -> BoxFuture<'_, ()> { + let principal = principal.to_owned(); + let source_ip = source_ip.map(std::borrow::ToOwned::to_owned); + Box::pin(async move { + let coll = self.catalog_db().collection::("failed_logins"); + let mut login_doc = doc! { + "principal": &principal, + "attempted_at": now_bson(), + }; + if let Some(ref ip) = source_ip { + login_doc.insert("source_ip", ip.as_str()); + } + if let Err(e) = coll.insert_one(login_doc).await { + tracing::error!("record_failed_login: {e}"); + } + }) + } + + fn cleanup_old_attempts(&self, max_age_seconds: i64) -> BoxFuture<'_, ()> { + Box::pin(async move { + let coll = self.catalog_db().collection::("failed_logins"); + let cutoff = time::OffsetDateTime::now_utc() + - std::time::Duration::from_secs(max_age_seconds as u64); + let cutoff_bson = BsonDateTime::from_millis(cutoff.unix_timestamp() * 1000); + if let Err(e) = coll + .delete_many(doc! { "attempted_at": { "$lt": cutoff_bson } }) + .await + { + tracing::error!("cleanup_old_attempts: {e}"); + } + }) + } +} + +// ── AdminStore ────────────────────────────────────────────────────────── + +impl extenddb_storage::management_store::AdminStore for MongoCatalogStore { + fn create_admin(&self, admin_name: &str, password_hash: &str) -> BoxFuture<'_, OpResult<()>> { + let admin_name = admin_name.to_owned(); + let password_hash = password_hash.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("admin_users"); + let result = coll + .insert_one(doc! { + "_id": &admin_name, + "password_hash": &password_hash, + "created_at": now_bson(), + }) + .await; + match result { + Ok(_) => Ok(()), + Err(e) if is_duplicate_key(&e) => Err(OpError::AlreadyExists( + "Admin user already exists".to_owned(), + )), + Err(e) => { + tracing::error!("create_admin failed: {e}"); + Err(OpError::Internal("Database error".to_owned())) + } + } + }) + } + + fn list_admins(&self) -> BoxFuture<'_, OpResult>> { + Box::pin(async { + let coll = self.catalog_db().collection::("admin_users"); + let opts = FindOptions::builder().sort(doc! { "_id": 1 }).build(); + let cursor = coll.find(doc! {}).with_options(opts).await.map_err(|e| { + tracing::error!("list_admins: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let docs: Vec = cursor.try_collect().await.map_err(|e| { + tracing::error!("list_admins cursor: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + Ok(docs + .into_iter() + .filter_map(|d| { + Some(AdminEntry { + admin_name: d.get_str("_id").ok()?.to_owned(), + created_at: to_offset_dt(d.get_datetime("created_at").ok()?.to_owned()), + }) + }) + .collect()) + }) + } + + fn delete_admin(&self, admin_name: &str) -> BoxFuture<'_, OpResult<()>> { + let admin_name = admin_name.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("admin_users"); + let result = coll + .delete_one(doc! { "_id": &admin_name }) + .await + .map_err(|e| { + tracing::error!("delete_admin failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.deleted_count == 0 { + return Err(OpError::NotFound("Admin user not found".to_owned())); + } + Ok(()) + }) + } + + fn change_admin_password( + &self, + admin_name: &str, + password_hash: &str, + ) -> BoxFuture<'_, OpResult<()>> { + let admin_name = admin_name.to_owned(); + let password_hash = password_hash.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("admin_users"); + let result = coll + .update_one( + doc! { "_id": &admin_name }, + doc! { "$set": { "password_hash": &password_hash } }, + ) + .await + .map_err(|e| { + tracing::error!("change_admin_password failed: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + if result.matched_count == 0 { + return Err(OpError::NotFound("Admin user not found".to_owned())); + } + Ok(()) + }) + } + + fn verify_admin_password( + &self, + admin_name: &str, + password: &str, + ) -> BoxFuture<'_, OpResult>> { + let admin_name = admin_name.to_owned(); + let password = password.to_owned(); + Box::pin(async move { + let coll = self.catalog_db().collection::("admin_users"); + let doc = coll + .find_one(doc! { "_id": &admin_name }) + .await + .map_err(|e| { + tracing::error!("verify_admin_password: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + let Some(admin_doc) = doc else { + return Ok(None); + }; + let Some(hash) = admin_doc.get_str("password_hash").ok() else { + return Ok(None); + }; + let hash = hash.to_owned(); + Ok(Some( + tokio::task::spawn_blocking(move || { + bcrypt::verify(password, &hash).unwrap_or(false) + }) + .await + .unwrap_or(false), + )) + }) + } +} + +// ── Helper: encryption key retrieval ──────────────────────────────────── + +impl MongoCatalogStore { + async fn get_encryption_key(&self) -> OpResult { + if let Some(ref cached) = self.encryption_key { + return Ok(cached.clone()); + } + let coll = self.catalog_db().collection::("settings"); + let doc = coll + .find_one(doc! { "_id": "encryption_key" }) + .await + .map_err(|e| { + tracing::error!("get_encryption_key: {e}"); + OpError::Internal("Database error".to_owned()) + })?; + doc.and_then(|d| d.get_str("value").ok().map(std::borrow::ToOwned::to_owned)) + .ok_or_else(|| OpError::Internal("Encryption key not configured".to_owned())) + } +} + +// ── Crypto helpers ────────────────────────────────────────────────────── + +fn generate_access_key_id() -> String { + const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + let mut rng = rand::rng(); + let suffix: String = (0..8) + .map(|_| CHARSET[rand::Rng::random_range(&mut rng, 0..CHARSET.len())] as char) + .collect(); + format!("AKIAEXTENDDB{suffix}") +} + +fn generate_secret_key() -> String { + const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut rng = rand::rng(); + let suffix: String = (0..32) + .map(|_| CHARSET[rand::Rng::random_range(&mut rng, 0..CHARSET.len())] as char) + .collect(); + format!("extenddb{suffix}") +} + +fn encrypt_secret(plaintext: &str, key_b64: &str, aad: &str) -> Result, String> { + use aes_gcm::Aes256Gcm; + use aes_gcm::KeyInit; + use aes_gcm::aead::Aead; + use aes_gcm::aead::Payload; + use base64::Engine; + + let key_bytes = base64::engine::general_purpose::STANDARD + .decode(key_b64) + .map_err(|e| format!("decode encryption key: {e}"))?; + + let key = aes_gcm::Key::::from_slice(&key_bytes); + let cipher = Aes256Gcm::new(key); + + let nonce_bytes: [u8; 12] = rand::random(); + let nonce = aes_gcm::Nonce::from_slice(&nonce_bytes); + + let payload = Payload { + msg: plaintext.as_bytes(), + aad: aad.as_bytes(), + }; + let ciphertext = cipher + .encrypt(nonce, payload) + .map_err(|e| format!("encrypt: {e}"))?; + + let mut result = Vec::with_capacity(12 + ciphertext.len()); + result.extend_from_slice(&nonce_bytes); + result.extend_from_slice(&ciphertext); + Ok(result) +} diff --git a/crates/storage-mongodb/src/metadata_engine.rs b/crates/storage-mongodb/src/metadata_engine.rs new file mode 100644 index 00000000..d3606cbe --- /dev/null +++ b/crates/storage-mongodb/src/metadata_engine.rs @@ -0,0 +1,544 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `MetadataEngine` implementation for `MongoDB`. +//! +//! Handles TTL configuration, resource tags, and table size bookkeeping. + +use futures::TryStreamExt; +use futures::future::BoxFuture; +use mongodb::IndexModel; +use mongodb::bson::{Document, doc}; +use mongodb::options::IndexOptions; + +use extenddb_core::types::{Item, Tag, TimeToLiveDescription, TimeToLiveStatus}; +use extenddb_storage::MetadataEngine; +use extenddb_storage::TtlTableInfo; +use extenddb_storage::error::StorageError; + +use crate::MongoEngine; +use crate::data::{data_collection_name, document_to_item}; + +fn extract_id_fields(doc: &Document) -> (String, String) { + let id = doc.get_document("_id").ok(); + let account_id = id + .and_then(|d| d.get_str("account_id").ok()) + .unwrap_or_default() + .to_owned(); + let table_name = id + .and_then(|d| d.get_str("table_name").ok()) + .unwrap_or_default() + .to_owned(); + (account_id, table_name) +} + +impl MetadataEngine for MongoEngine { + fn describe_ttl( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { + let coll = self.catalog_db.collection::("tables"); + let table_doc = coll + .find_one(doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(table_name.clone()))?; + + match table_doc.get_str("ttl_attribute") { + Ok(attr) => Ok(TimeToLiveDescription { + time_to_live_status: TimeToLiveStatus::Enabled, + attribute_name: Some(attr.to_owned()), + }), + Err(_) => Ok(TimeToLiveDescription { + time_to_live_status: TimeToLiveStatus::Disabled, + attribute_name: None, + }), + } + }) + } + + fn update_ttl( + &self, + account_id: &str, + table_name: &str, + attribute_name: &str, + enabled: bool, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + let attribute_name = attribute_name.to_string(); + Box::pin(async move { + let coll = self.catalog_db.collection::("tables"); + + let ttl_val = if enabled { + mongodb::bson::Bson::String(attribute_name) + } else { + mongodb::bson::Bson::Null + }; + + let result = coll + .update_one( + doc! { + "_id": { "account_id": &account_id, "table_name": &table_name }, + "table_status": "ACTIVE", + }, + doc! { + "$set": { + "ttl_attribute": ttl_val, + "ttl_index_ready": false, + } + }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if result.matched_count == 0 { + let exists = coll + .find_one( + doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + return match exists { + None => Err(StorageError::TableNotFound(table_name)), + Some(_) => Err(StorageError::TableNotActive(table_name)), + }; + } + + Ok(()) + }) + } + + fn tag_resource(&self, arn: &str, tags: &[Tag]) -> BoxFuture<'_, Result<(), StorageError>> { + let arn = arn.to_string(); + let tags = tags.to_vec(); + Box::pin(async move { + let coll = self.catalog_db.collection::("tags"); + + for tag in &tags { + coll.update_one( + doc! { "resource_arn": &arn, "tag_key": &tag.key }, + doc! { "$set": { "resource_arn": &arn, "tag_key": &tag.key, "tag_value": &tag.value } }, + ) + .upsert(true) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) + }) + } + + fn untag_resource( + &self, + arn: &str, + tag_keys: &[String], + ) -> BoxFuture<'_, Result<(), StorageError>> { + let arn = arn.to_string(); + let tag_keys = tag_keys.to_vec(); + Box::pin(async move { + let coll = self.catalog_db.collection::("tags"); + + for key in &tag_keys { + coll.delete_one(doc! { "resource_arn": &arn, "tag_key": key }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Ok(()) + }) + } + + fn list_tags(&self, arn: &str) -> BoxFuture<'_, Result, StorageError>> { + let arn = arn.to_string(); + Box::pin(async move { + let coll = self.catalog_db.collection::("tags"); + let mut cursor = coll + .find(doc! { "resource_arn": &arn }) + .sort(doc! { "tag_key": 1 }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut tags = Vec::new(); + while let Some(doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let key = doc.get_str("tag_key").unwrap_or_default().to_owned(); + let value = doc.get_str("tag_value").unwrap_or_default().to_owned(); + tags.push(Tag { key, value }); + } + Ok(tags) + }) + } + + fn tables_with_ttl( + &self, + account_id: &str, + ) -> BoxFuture<'_, Result, StorageError>> { + let account_id = account_id.to_string(); + Box::pin(async move { + let coll = self.catalog_db.collection::("tables"); + let mut cursor = coll + .find(doc! { + "_id.account_id": &account_id, + "ttl_attribute": { "$ne": mongodb::bson::Bson::Null }, + "table_status": "ACTIVE", + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut results = Vec::new(); + while let Some(doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let name = doc + .get_document("_id") + .ok() + .and_then(|id| id.get_str("table_name").ok()) + .unwrap_or_default() + .to_owned(); + let attr = doc.get_str("ttl_attribute").unwrap_or_default().to_owned(); + results.push((name, attr)); + } + Ok(results) + }) + } + + fn all_tables_with_ttl(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let coll = self.catalog_db.collection::("tables"); + let mut cursor = coll + .find(doc! { + "ttl_attribute": { "$ne": mongodb::bson::Bson::Null }, + "table_status": "ACTIVE", + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut results = Vec::new(); + while let Some(doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let (account_id, name) = extract_id_fields(&doc); + let attr = doc.get_str("ttl_attribute").unwrap_or_default().to_owned(); + results.push((account_id, name, attr)); + } + Ok(results) + }) + } + + fn all_tables_with_ttl_index_ready( + &self, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let coll = self.catalog_db.collection::("tables"); + let mut cursor = coll + .find(doc! { + "ttl_attribute": { "$ne": mongodb::bson::Bson::Null }, + "ttl_index_ready": true, + "table_status": "ACTIVE", + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut results = Vec::new(); + while let Some(doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let (account_id, name) = extract_id_fields(&doc); + let attr = doc.get_str("ttl_attribute").unwrap_or_default().to_owned(); + results.push((account_id, name, attr)); + } + Ok(results) + }) + } + + fn create_ttl_index( + &self, + account_id: &str, + table_name: &str, + ttl_attribute: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + let ttl_attribute = ttl_attribute.to_string(); + Box::pin(async move { + let tables_coll = self.catalog_db.collection::("tables"); + let id_filter = + doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }; + let table_doc = tables_coll + .find_one(id_filter.clone()) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(table_name.clone()))?; + + let table_id = table_doc + .get_str("table_id") + .map_err(|_| StorageError::Internal("missing table_id".to_string()))?; + + let coll_name = data_collection_name(table_id); + let data_coll = self.data_db.collection::(&coll_name); + + let index_name = format!("idx_ttl_{ttl_attribute}"); + let index_key = format!("item_data.{ttl_attribute}.N"); + + let index = IndexModel::builder() + .keys(doc! { &index_key: 1 }) + .options( + IndexOptions::builder() + .name(index_name) + .sparse(true) + .build(), + ) + .build(); + + data_coll + .create_index(index) + .await + .map_err(|e| StorageError::Internal(format!("TTL index creation failed: {e}")))?; + + tables_coll + .update_one(id_filter, doc! { "$set": { "ttl_index_ready": true } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(()) + }) + } + + fn drop_ttl_index( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { + let tables_coll = self.catalog_db.collection::("tables"); + let id_filter = + doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }; + + // Mark index as not ready first + tables_coll + .update_one( + id_filter.clone(), + doc! { "$set": { "ttl_index_ready": false } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let table_doc = tables_coll + .find_one(id_filter) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(table_name.clone()))?; + + let table_id = table_doc + .get_str("table_id") + .map_err(|_| StorageError::Internal("missing table_id".to_string()))?; + + let ttl_attribute = table_doc.get_str("ttl_attribute").unwrap_or_default(); + let index_name = format!("idx_ttl_{ttl_attribute}"); + + let coll_name = data_collection_name(table_id); + let data_coll = self.data_db.collection::(&coll_name); + + data_coll + .drop_index(index_name) + .await + .map_err(|e| StorageError::Internal(format!("TTL index drop failed: {e}")))?; + + Ok(()) + }) + } + + fn find_expired_items_indexed( + &self, + account_id: &str, + table_name: &str, + ttl_attribute: &str, + limit: usize, + ) -> BoxFuture<'_, Result, StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + let ttl_attribute = ttl_attribute.to_string(); + Box::pin(async move { + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(table_name.clone()))?; + + let table_id = table_doc + .get_str("table_id") + .map_err(|_| StorageError::Internal("missing table_id".to_string()))?; + + let coll_name = data_collection_name(table_id); + let data_coll = self.data_db.collection::(&coll_name); + + let now_epoch = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64; + + let ttl_field = format!("item_data.{ttl_attribute}.N"); + + // Find items where TTL attribute N value is between 1 and now (expired) + // DynamoDB stores numbers as strings in the N field + let filter = doc! { + &ttl_field: { + "$exists": true, + "$ne": mongodb::bson::Bson::Null, + } + }; + + let mut cursor = data_coll + .find(filter) + .sort(doc! { &ttl_field: 1 }) + .limit(limit as i64 * 2) // over-fetch since we filter in app + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut items = Vec::new(); + while let Some(doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + if items.len() >= limit { + break; + } + // Parse the TTL value and check if expired + if let Ok(item_data) = doc.get_document("item_data") + && let Ok(ttl_obj) = item_data.get_document(&ttl_attribute) + && let Ok(n_str) = ttl_obj.get_str("N") + && let Ok(ttl_val) = n_str.parse::() + && ttl_val >= 1 + && ttl_val <= now_epoch + { + let item = document_to_item(&doc)?; + items.push(item); + } + } + Ok(items) + }) + } + + fn refresh_table_size( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { + let tables_coll = self.catalog_db.collection::("tables"); + let id_filter = + doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }; + let table_doc = tables_coll + .find_one(id_filter.clone()) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(table_name.clone()))?; + + let table_id = table_doc + .get_str("table_id") + .map_err(|_| StorageError::Internal("missing table_id".to_string()))?; + + let coll_name = data_collection_name(table_id); + let data_coll = self.data_db.collection::(&coll_name); + + let item_count = data_coll + .count_documents(doc! {}) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + as i64; + + // Approximate size via collStats + let stats_result = self + .data_db + .run_command(doc! { "collStats": &coll_name }) + .await; + + let table_size = match stats_result { + Ok(stats) => stats.get_i64("size").unwrap_or(0), + Err(_) => 0, + }; + + tables_coll + .update_one( + doc! { "_id": { "account_id": &account_id, "table_name": &table_name }, "table_status": "ACTIVE" }, + doc! { "$set": { "item_count": item_count, "table_size_bytes": table_size } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(()) + }) + } + + fn list_active_table_names( + &self, + account_id: &str, + ) -> BoxFuture<'_, Result, StorageError>> { + let account_id = account_id.to_string(); + Box::pin(async move { + let coll = self.catalog_db.collection::("tables"); + let mut cursor = coll + .find(doc! { "_id.account_id": &account_id, "table_status": "ACTIVE" }) + .sort(doc! { "_id.table_name": 1 }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut names = Vec::new(); + while let Some(doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let name = doc + .get_document("_id") + .ok() + .and_then(|id| id.get_str("table_name").ok()) + .unwrap_or_default() + .to_owned(); + names.push(name); + } + Ok(names) + }) + } + + fn all_active_tables(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let coll = self.catalog_db.collection::("tables"); + let mut cursor = coll + .find(doc! { "table_status": "ACTIVE" }) + .sort(doc! { "_id.account_id": 1, "_id.table_name": 1 }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut results = Vec::new(); + while let Some(doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let (account_id, name) = extract_id_fields(&doc); + results.push((account_id, name)); + } + Ok(results) + }) + } +} diff --git a/crates/storage-mongodb/src/operations.rs b/crates/storage-mongodb/src/operations.rs new file mode 100644 index 00000000..2389a8f1 --- /dev/null +++ b/crates/storage-mongodb/src/operations.rs @@ -0,0 +1,129 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `MongoDB` implementation of `OperationsEngine`. + +use extenddb_storage::error::StorageError; +use extenddb_storage::operations::{ConnectionParts, OperationsEngine}; + +/// `MongoDB` operations engine for extenddb CLI commands. +pub struct MongoOperationsEngine; + +impl OperationsEngine for MongoOperationsEngine { + fn parse_connection_string(&self, s: &str) -> Result { + // Best-effort parse of `mongodb[+srv]://[user[:pass]@]host[:port][/db][?...]`. + // The mongo driver owns full URI validation at connect time; this is only + // for display and CLI-side identifier extraction. + let scheme_end = s + .find("://") + .ok_or_else(|| StorageError::Internal("connection string has no scheme".to_owned()))? + + 3; + let rest = &s[scheme_end..]; + + // Split at first `?` to drop query string. + let (authority_path, _) = rest.split_once('?').unwrap_or((rest, "")); + + // Split userinfo from host by the last `@` before the first `/`. + let path_start = authority_path.find('/').unwrap_or(authority_path.len()); + let authority = &authority_path[..path_start]; + let path = authority_path[path_start..].trim_start_matches('/'); + + let (user, password, hostport) = if let Some(at) = authority.rfind('@') { + let (userinfo, hp) = authority.split_at(at); + let hp = &hp[1..]; + let (u, p) = userinfo + .split_once(':') + .map_or((userinfo, ""), |(u, p)| (u, p)); + (u.to_owned(), p.to_owned(), hp) + } else { + (String::new(), String::new(), authority) + }; + + // hostport may be a comma-separated list for replica sets — take the first + // seed. Port defaults to 27017 (matches mongo driver default). + let first_host = hostport.split(',').next().unwrap_or(hostport); + let (host, port) = if let Some((h, p)) = first_host.rsplit_once(':') { + (h.to_owned(), p.parse::().unwrap_or(27017)) + } else { + (first_host.to_owned(), 27017) + }; + + Ok(ConnectionParts { + host, + port, + user, + password, + database: path.to_owned(), + }) + } + + fn redact_connection_string(&self, s: &str) -> String { + // Redact password from mongodb[+srv]://user:password@host[:port]/... + let Some(scheme_end) = s.find("://") else { + return s.to_owned(); + }; + let after_scheme = scheme_end + 3; + // Only consider `@` before the first `?`. + let query_start = s[after_scheme..] + .find('?') + .map_or(s.len(), |q| after_scheme + q); + let Some(at) = s[after_scheme..query_start].rfind('@') else { + return s.to_owned(); + }; + let at_idx = after_scheme + at; + let userinfo = &s[after_scheme..at_idx]; + let Some(colon) = userinfo.find(':') else { + return s.to_owned(); + }; + let user = &userinfo[..colon]; + format!("{}{user}:***{}", &s[..after_scheme], &s[at_idx..]) + } + + fn validate_identifier(&self, name: &str, label: &str) -> Result<(), StorageError> { + // MongoDB collection/database identifier constraints: + // - no `$` prefix reserved for operators + // - no `.` (used as path separator inside documents) + // - no `\0` (null byte) + // - no non-ASCII + if name.contains('$') { + return Err(StorageError::Internal(format!( + "{label} must not contain '$'" + ))); + } + if name.contains('.') { + return Err(StorageError::Internal(format!( + "{label} must not contain '.'" + ))); + } + if name.contains('\0') { + return Err(StorageError::Internal(format!( + "{label} must not contain null bytes" + ))); + } + if !name.is_ascii() { + return Err(StorageError::Internal(format!( + "{label} must contain only ASCII characters" + ))); + } + Ok(()) + } + + fn catalog_version(&self) -> String { + // Mongo catalog version — matches the constant enforced by + // MongoBootstrapper::expected_catalog_version. + "0.0.2".to_owned() + } + + fn is_sensitive_key(&self, key: &str) -> bool { + let lower = key.to_lowercase(); + [ + "connection_string", + "password", + "secret", + "token", + "encryption_key", + ] + .iter() + .any(|pattern| lower.contains(pattern)) + } +} diff --git a/crates/storage-mongodb/src/pushdown.rs b/crates/storage-mongodb/src/pushdown.rs new file mode 100644 index 00000000..c2c84c42 --- /dev/null +++ b/crates/storage-mongodb/src/pushdown.rs @@ -0,0 +1,523 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Compile-time analyzer for filter-pushdown eligibility. +//! +//! For a DDB [`Expr`] and its accompanying [`ExpressionMaps`], returns +//! [`Pushable::Yes`] when the expression can be safely compiled to a +//! MongoDB filter and evaluated by the storage layer, or +//! [`Pushable::No`] with a reason string when at least one subexpression +//! must fall back to session-scoped in-Rust evaluation. +//! +//! Whole-condition all-or-nothing: if any subexpression is not pushable, +//! the entire expression falls back. You cannot cherry-pick — evaluating +//! part of an `AND` / `OR` in the storage layer and part in the +//! application layer would confuse the composition semantics. +//! +//! **Pushable subset** (see `todo.md` A5 for full rationale): +//! +//! - `attribute_exists(path)`, `attribute_not_exists(path)` +//! - `attribute_type(path, :t)` for any type tag +//! - `begins_with(path, :prefix)` where `:prefix` is `S` +//! - `contains(path, :val)` where `:val` is `S` +//! - `path :v` where `:v` is `S`, or `:v` is `B` and op is `Eq` / `Ne` +//! - `AND`, `OR` of pushable subexpressions +//! - `NOT attribute_exists(path)` / `NOT attribute_not_exists(path)` only +//! +//! **Not pushable** (falls back to in-Rust): +//! +//! - Any operand of type `N` in any position (numbers are stored as +//! strings, so MongoDB comparators evaluate lexicographically — +//! `"10" > "9"` is false string-wise, true numerically) +//! - `size(...)` (MongoDB's `$strLenBytes` and `$strLenCP` don't match +//! DDB's UTF-16 code unit count for strings) +//! - `NOT` around anything except `attribute_exists` / +//! `attribute_not_exists` (three-valued logic on missing paths +//! diverges from MongoDB's `$nor` semantics) +//! - `IN` and `BETWEEN` — the compiler emits them but the analyzer +//! currently marks them non-pushable pending proptest coverage. +//! Restoring them is a follow-up; the in-Rust fallback path is +//! already correct. +//! - `path :v` where `:v` is `B` (base64 string ordering +//! ≠ bytewise byte ordering across mismatched lengths) +//! - Any operand type the analyzer doesn't yet classify + +use extenddb_core::expression::{CompareOp, Expr, ExpressionMaps}; +use extenddb_core::types::AttributeValue; + +/// Outcome of the pushdown analyzer. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Pushable { + /// The expression compiles to a MongoDB filter with semantics that + /// match `extenddb_core::expression::evaluate_condition`. + Yes, + /// At least one subexpression falls outside the pushable subset. + /// The caller must evaluate the whole expression in-Rust. + No(&'static str), +} + +impl Pushable { + pub fn is_yes(&self) -> bool { + matches!(self, Pushable::Yes) + } +} + +/// Decide whether a compiled MongoDB filter for `expr` will agree with +/// `evaluate_condition(expr, item, maps)` for every item. +/// +/// Conservative: unknown constructs return `Pushable::No`. +pub fn is_pushable(expr: &Expr, maps: &ExpressionMaps) -> Pushable { + walk(expr, maps) +} + +fn walk(expr: &Expr, maps: &ExpressionMaps) -> Pushable { + match expr { + Expr::Function { name, args } => match name.to_lowercase().as_str() { + "attribute_exists" | "attribute_not_exists" => { + // Always pushable; missing-path semantics match MongoDB's + // `$exists`. + if args.len() == 1 { + Pushable::Yes + } else { + Pushable::No("attribute_exists/not_exists arity") + } + } + "attribute_type" => { + if args.len() != 2 { + return Pushable::No("attribute_type arity"); + } + // The type argument must resolve to a String whose + // value is exactly one of the DDB type tags. The + // compiler in condition.rs inserts the tag verbatim + // into the mongo field path via + // `format!("{field}.{type_name}")`; without this + // whitelist a placeholder like `":t": {"S": "$ne"}` + // would produce a filter clause with a `$`-prefixed + // "field" that mongo would interpret as an operator. + // Keeping the whitelist in the analyzer means the + // compiler is only ever reached with a known-safe + // tag. RFC-0003 §6.1 / §8 (D-M6). + let Expr::Placeholder(name) = &args[1] else { + return Pushable::No("attribute_type tag not a placeholder"); + }; + let Ok(val) = maps.resolve_value(name) else { + return Pushable::No("attribute_type tag unresolvable"); + }; + match val { + AttributeValue::S(tag) => { + const VALID: &[&str] = + &["S", "N", "B", "BOOL", "NULL", "L", "M", "SS", "NS", "BS"]; + if !VALID.contains(&tag.as_str()) { + return Pushable::No("attribute_type tag not a DDB type name"); + } + } + _ => return Pushable::No("attribute_type non-string tag"), + } + Pushable::Yes + } + "begins_with" => { + if args.len() != 2 { + return Pushable::No("begins_with arity"); + } + if !arg_resolves_to_scalar_type(&args[1], maps, AttrKind::S) { + return Pushable::No("begins_with non-string prefix"); + } + Pushable::Yes + } + "contains" => { + if args.len() != 2 { + return Pushable::No("contains arity"); + } + // Only S operands. B is emittable but not yet + // proptest-covered; N is not pushable because number + // storage is string-based. + match value_kind(&args[1], maps) { + Some(AttrKind::S) => Pushable::Yes, + Some(AttrKind::N) => Pushable::No("contains on N operand"), + Some(AttrKind::B) => Pushable::No("contains on B operand (not yet covered)"), + _ => Pushable::No("contains on unsupported operand type"), + } + } + "size" => Pushable::No("size() — UTF-16 mismatch with MongoDB"), + _ => Pushable::No("unknown function"), + }, + Expr::Compare { left, op, right } => { + // Both operands must be pushable operand types. Numbers + // anywhere → not pushable. Sets / lists / maps in the + // operand position → not pushable in the current subset. + let left_kind = operand_kind(left, maps); + let right_kind = operand_kind(right, maps); + let (Some(lk), Some(rk)) = (left_kind, right_kind) else { + return Pushable::No("Compare with un-inferrable operand kind"); + }; + // Number anywhere disqualifies. + if matches!(lk, AttrKind::N) || matches!(rk, AttrKind::N) { + return Pushable::No("Compare with N operand"); + } + match (lk, rk, op) { + // Field S-value: any comparator OK (lex matches wire form). + (AttrKind::Field, AttrKind::S, _) | (AttrKind::S, AttrKind::Field, _) => { + Pushable::Yes + } + // Field = / <> B-value: OK. Ordering on B not OK + // (base64 string ordering ≠ bytewise). + (AttrKind::Field, AttrKind::B, CompareOp::Eq | CompareOp::Ne) + | (AttrKind::B, AttrKind::Field, CompareOp::Eq | CompareOp::Ne) => Pushable::Yes, + (AttrKind::Field, AttrKind::B, _) | (AttrKind::B, AttrKind::Field, _) => { + Pushable::No("ordering comparator on B operand") + } + // Field = / <> BOOL / NULL: OK. + (AttrKind::Field, AttrKind::Bool, CompareOp::Eq | CompareOp::Ne) + | (AttrKind::Bool, AttrKind::Field, CompareOp::Eq | CompareOp::Ne) + | (AttrKind::Field, AttrKind::Null, CompareOp::Eq | CompareOp::Ne) + | (AttrKind::Null, AttrKind::Field, CompareOp::Eq | CompareOp::Ne) => Pushable::Yes, + (AttrKind::Field, AttrKind::Bool | AttrKind::Null, _) + | (AttrKind::Bool | AttrKind::Null, AttrKind::Field, _) => { + Pushable::No("ordering on BOOL / NULL operand") + } + // Field vs. Field: NOT pushable. A plain field's type is + // unknown at compile time (its AttrKind is just `Field`), so + // the emitted $expr compares the raw tagged subdocuments — + // e.g. two Number fields, stored string-encoded, compare + // lexically ("42" < "9"), giving the wrong answer in both + // directions. Fall back to the in-Rust evaluator, consistent + // with the N/B literal exclusions above. + (AttrKind::Field, AttrKind::Field, _) => { + Pushable::No("Field vs Field — operand types unknown at compile time") + } + // Two literals — pushable but degenerate. + _ => Pushable::No("Compare with unusual operand kinds"), + } + } + Expr::And(l, r) => match (walk(l, maps), walk(r, maps)) { + (Pushable::Yes, Pushable::Yes) => Pushable::Yes, + (Pushable::No(r), _) | (_, Pushable::No(r)) => Pushable::No(r), + }, + Expr::Or(l, r) => match (walk(l, maps), walk(r, maps)) { + (Pushable::Yes, Pushable::Yes) => Pushable::Yes, + (Pushable::No(r), _) | (_, Pushable::No(r)) => Pushable::No(r), + }, + Expr::Not(inner) => { + // Only pushable when inner is exactly an existence check. + // Everything else — comparisons, functions, nested logic — + // is disallowed because MongoDB's $nor on missing paths + // returns true where DDB's three-valued logic returns false. + match inner.as_ref() { + Expr::Function { name, args } if args.len() == 1 => { + let n = name.to_lowercase(); + if n == "attribute_exists" || n == "attribute_not_exists" { + Pushable::Yes + } else { + Pushable::No("NOT around non-existence function") + } + } + _ => Pushable::No("NOT around non-existence expression"), + } + } + Expr::Between { .. } => Pushable::No("BETWEEN — analyzer coverage pending"), + Expr::In { .. } => Pushable::No("IN — analyzer coverage pending"), + Expr::Path(_) | Expr::Placeholder(_) | Expr::Arithmetic { .. } => { + Pushable::No("bare path/placeholder/arithmetic at top level") + } + } +} + +/// The compiler's operand-kind classification, used by the analyzer to +/// reason about type-mixing rules. +#[derive(Debug, Clone, Copy)] +enum AttrKind { + /// Reference to a document field (`Expr::Path`). + Field, + S, + N, + B, + Bool, + Null, +} + +fn operand_kind(expr: &Expr, maps: &ExpressionMaps) -> Option { + match expr { + Expr::Path(_) => Some(AttrKind::Field), + Expr::Placeholder(_) => value_kind(expr, maps), + _ => None, + } +} + +fn value_kind(expr: &Expr, maps: &ExpressionMaps) -> Option { + let Expr::Placeholder(name) = expr else { + return None; + }; + let av = maps.resolve_value(name).ok()?; + Some(match av { + AttributeValue::S(_) => AttrKind::S, + AttributeValue::N(_) => AttrKind::N, + AttributeValue::B(_) => AttrKind::B, + AttributeValue::Bool(_) => AttrKind::Bool, + AttributeValue::Null => AttrKind::Null, + _ => return None, + }) +} + +fn arg_resolves_to_scalar_type(expr: &Expr, maps: &ExpressionMaps, expected: AttrKind) -> bool { + matches!( + (value_kind(expr, maps), expected), + (Some(AttrKind::S), AttrKind::S) + | (Some(AttrKind::N), AttrKind::N) + | (Some(AttrKind::B), AttrKind::B) + | (Some(AttrKind::Bool), AttrKind::Bool) + | (Some(AttrKind::Null), AttrKind::Null) + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use extenddb_core::expression::PathElement; + use std::collections::HashMap; + + fn maps_with(values: &[(&str, AttributeValue)]) -> ExpressionMaps { + let mut m = HashMap::new(); + for (k, v) in values { + m.insert((*k).to_string(), v.clone()); + } + ExpressionMaps::new(HashMap::new(), m) + } + + fn path(name: &str) -> Expr { + Expr::Path(vec![PathElement::Attribute(name.to_string())]) + } + + #[test] + fn attribute_exists_is_pushable() { + let expr = Expr::Function { + name: "attribute_exists".into(), + args: vec![path("a")], + }; + assert_eq!(is_pushable(&expr, &maps_with(&[])), Pushable::Yes); + } + + #[test] + fn size_is_not_pushable() { + let expr = Expr::Function { + name: "size".into(), + args: vec![path("a")], + }; + assert!(!is_pushable(&expr, &maps_with(&[])).is_yes()); + } + + #[test] + fn number_operand_is_not_pushable() { + let expr = Expr::Compare { + left: Box::new(path("a")), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":n".into())), + }; + let maps = maps_with(&[(":n", AttributeValue::N("42".into()))]); + assert!(!is_pushable(&expr, &maps).is_yes()); + } + + #[test] + fn field_vs_field_is_not_pushable() { + // Both operands are plain fields whose runtime types are unknown at + // compile time. Pushing $expr would compare tagged subdocuments + // (lexical for Numbers), so this must fall back to the in-Rust + // evaluator — for every comparator, not just ordering ones. + for op in [ + CompareOp::Eq, + CompareOp::Ne, + CompareOp::Lt, + CompareOp::Le, + CompareOp::Gt, + CompareOp::Ge, + ] { + let expr = Expr::Compare { + left: Box::new(path("counter_a")), + op, + right: Box::new(path("counter_b")), + }; + assert!( + !is_pushable(&expr, &maps_with(&[])).is_yes(), + "Field vs Field must not be pushable for {op:?}" + ); + } + } + + #[test] + fn string_equality_is_pushable() { + let expr = Expr::Compare { + left: Box::new(path("a")), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":s".into())), + }; + let maps = maps_with(&[(":s", AttributeValue::S("x".into()))]); + assert_eq!(is_pushable(&expr, &maps), Pushable::Yes); + } + + #[test] + fn binary_equality_is_pushable() { + let expr = Expr::Compare { + left: Box::new(path("a")), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":b".into())), + }; + let maps = maps_with(&[(":b", AttributeValue::B(vec![0, 1, 2]))]); + assert_eq!(is_pushable(&expr, &maps), Pushable::Yes); + } + + #[test] + fn binary_ordering_is_not_pushable() { + let expr = Expr::Compare { + left: Box::new(path("a")), + op: CompareOp::Lt, + right: Box::new(Expr::Placeholder(":b".into())), + }; + let maps = maps_with(&[(":b", AttributeValue::B(vec![0]))]); + assert!(!is_pushable(&expr, &maps).is_yes()); + } + + #[test] + fn not_attribute_exists_is_pushable() { + let expr = Expr::Not(Box::new(Expr::Function { + name: "attribute_exists".into(), + args: vec![path("a")], + })); + assert_eq!(is_pushable(&expr, &maps_with(&[])), Pushable::Yes); + } + + #[test] + fn not_around_comparison_is_not_pushable() { + let expr = Expr::Not(Box::new(Expr::Compare { + left: Box::new(path("a")), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":s".into())), + })); + let maps = maps_with(&[(":s", AttributeValue::S("x".into()))]); + assert!(!is_pushable(&expr, &maps).is_yes()); + } + + #[test] + fn and_of_two_pushable_is_pushable() { + let expr = Expr::And( + Box::new(Expr::Function { + name: "attribute_exists".into(), + args: vec![path("a")], + }), + Box::new(Expr::Function { + name: "attribute_exists".into(), + args: vec![path("b")], + }), + ); + assert_eq!(is_pushable(&expr, &maps_with(&[])), Pushable::Yes); + } + + #[test] + fn and_taints_on_either_side() { + let expr = Expr::And( + Box::new(Expr::Function { + name: "attribute_exists".into(), + args: vec![path("a")], + }), + Box::new(Expr::Function { + name: "size".into(), + args: vec![path("b")], + }), + ); + assert!(!is_pushable(&expr, &maps_with(&[])).is_yes()); + } + + #[test] + fn between_is_not_pushable() { + let expr = Expr::Between { + operand: Box::new(path("a")), + low: Box::new(Expr::Placeholder(":lo".into())), + high: Box::new(Expr::Placeholder(":hi".into())), + }; + let maps = maps_with(&[ + (":lo", AttributeValue::S("a".into())), + (":hi", AttributeValue::S("z".into())), + ]); + // BETWEEN is currently non-pushable pending analyzer coverage; + // this test locks in that decision. + assert!(!is_pushable(&expr, &maps).is_yes()); + } + + // ── D-M6 exclusion tests ──────────────────────────────────────── + // + // The condition compiler in `condition.rs` has latent correctness + // bugs on numeric operands, set/list/map equality, mixed-type IN + // lists, and unvalidated attribute_type tags. The A5 analyzer is + // supposed to keep every one of those input shapes out of the + // pushdown path. These tests lock in that boundary so a future + // widening of the compiler doesn't accidentally admit a buggy + // shape without the analyzer being updated. + + #[test] + fn numeric_compare_is_not_pushable() { + // Numeric operands would compile to BSON string comparisons — + // the analyzer must reject. + let maps = maps_with(&[(":n", AttributeValue::N("42".into()))]); + let expr = Expr::Compare { + left: Box::new(path("a")), + op: CompareOp::Lt, + right: Box::new(Expr::Placeholder(":n".into())), + }; + assert!(!is_pushable(&expr, &maps).is_yes()); + } + + #[test] + fn in_is_not_pushable() { + // IN with mixed-type list uses the first literal's type for + // all entries — analyzer must reject entirely. + let maps = maps_with(&[ + (":a", AttributeValue::S("x".into())), + (":b", AttributeValue::N("1".into())), + ]); + let expr = Expr::In { + operand: Box::new(path("a")), + list: vec![ + Expr::Placeholder(":a".into()), + Expr::Placeholder(":b".into()), + ], + }; + assert!(!is_pushable(&expr, &maps).is_yes()); + } + + #[test] + fn set_equality_is_not_pushable() { + // Eq/Ne on SS/NS/BS compiles to Bson::Null in the compiler. + // Analyzer classifies as `un-inferrable operand kind` and + // must reject. + let ss: std::collections::BTreeSet = + ["a".to_owned(), "b".to_owned()].into_iter().collect(); + let maps = maps_with(&[(":s", AttributeValue::SS(ss))]); + let expr = Expr::Compare { + left: Box::new(path("tags")), + op: CompareOp::Eq, + right: Box::new(Expr::Placeholder(":s".into())), + }; + assert!(!is_pushable(&expr, &maps).is_yes()); + } + + #[test] + fn attribute_type_valid_tag_is_pushable() { + // "S" is a valid DDB type tag — analyzer admits. + let expr = Expr::Function { + name: "attribute_type".to_owned(), + args: vec![path("a"), Expr::Placeholder(":t".into())], + }; + let maps = maps_with(&[(":t", AttributeValue::S("S".into()))]); + assert!(is_pushable(&expr, &maps).is_yes()); + } + + #[test] + fn attribute_type_invalid_tag_is_not_pushable() { + // An arbitrary string as the tag is refused so the compiler + // never assembles `field.$evil` mongo path fragments. + let expr = Expr::Function { + name: "attribute_type".to_owned(), + args: vec![path("a"), Expr::Placeholder(":t".into())], + }; + let maps = maps_with(&[(":t", AttributeValue::S("$ne".into()))]); + assert!(!is_pushable(&expr, &maps).is_yes()); + } +} diff --git a/crates/storage-mongodb/src/stream_engine.rs b/crates/storage-mongodb/src/stream_engine.rs new file mode 100644 index 00000000..e976a667 --- /dev/null +++ b/crates/storage-mongodb/src/stream_engine.rs @@ -0,0 +1,810 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `StreamEngine` trait implementation for `MongoEngine`. +//! +//! `DynamoDB` Streams are implemented using `MongoDB`'s own stream record storage. +//! Stream records are written to a `stream_records` collection in the data database, +//! grouped by shard. Shards are stored in `stream_shards` in the data database. +//! This approach uses the same storage model as the `PostgreSQL` backend rather than +//! `MongoDB` Change Streams, to maintain behavioral parity (explicit sequence numbers, +//! shard assignment, retention cleanup). + +use futures::TryStreamExt; +use futures::future::BoxFuture; +use mongodb::bson::DateTime as BsonDateTime; +use mongodb::bson::{self, Document, doc}; +use mongodb::options::FindOptions; + +use extenddb_core::types::{ + DescribeStreamInput, SequenceNumberRange, Shard, StreamDescription, StreamEventName, + StreamRecord, StreamStatus, StreamSummary, StreamViewType, +}; +use extenddb_storage::StreamEngine; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{parse_stream_arn, stream_arn}; +use extenddb_storage::{StreamListResult, StreamRecordsResult}; + +use crate::MongoEngine; + +const SHARDS_PER_STREAM: u32 = 4; + +/// Map a `StreamEventName` to its DynamoDB wire-format string. +/// +/// DynamoDB Streams records use uppercase event names (`INSERT`, `MODIFY`, `REMOVE`). +/// The enum's `Debug` output is Rust-cased (`Insert`, ...) so we must not use that. +pub(crate) fn event_name_ddb_str(name: StreamEventName) -> &'static str { + match name { + StreamEventName::Insert => "INSERT", + StreamEventName::Modify => "MODIFY", + StreamEventName::Remove => "REMOVE", + } +} + +/// Build the mongo `stream_shards.shard_id` for a given table_id + shard index. +/// +/// **Security invariant (RFC-0003 §5.3, §8.2):** shard_id must incorporate +/// the table's globally-unique `table_id` (a UUID), not the caller-visible +/// `table_name`. Table names are only unique per-account, so two accounts +/// creating "orders" tables would generate colliding shard_ids under a +/// name-derived scheme, letting one account's `GetRecords(shard_id)` read +/// the other's stream records. `table_id` is a per-table-instance UUID that +/// resets on `DeleteTable + CreateTable` — the recreated table gets fresh +/// shard_ids, so leftover stream records from the deleted table never +/// resurface either. +/// +/// UUIDs are not guessable, so an attacker cannot synthesize a shard_id +/// belonging to another tenant without first observing it (which itself +/// requires an authenticated path scoped to that tenant's account). +pub(crate) fn build_shard_id(table_id: &str, shard_index: u32) -> String { + format!("shardId-{table_id}-{shard_index:012}") +} + +impl MongoEngine { + /// Initialize stream shards for a table. Only creates shard documents; + /// the caller is responsible for setting `stream_label` on the table doc. + /// + /// Uses the table's UUID (`table_id`), not `table_name`, in the shard_id + /// — see `build_shard_id` for the security rationale. + pub(crate) async fn init_stream_shards(&self, table_id: &str) -> Result<(), StorageError> { + let shards_coll = self.data_db.collection::("stream_shards"); + for i in 0..SHARDS_PER_STREAM { + let shard_id = build_shard_id(table_id, i); + let start_seq = format!("{:021}", 0); + let insert_res = shards_coll + .insert_one(doc! { + "shard_id": &shard_id, + "table_id": table_id, + "starting_sequence_number": &start_seq, + "created_at": BsonDateTime::now(), + }) + .await; + if let Err(e) = insert_res { + // Treat a duplicate `shard_id` as idempotent no-op: two + // concurrent `UpdateTable(stream_enabled=true)` calls + // can both observe "no shards yet" under snapshot + // isolation and both reach this insert; the unique + // index on `stream_shards.shard_id` (`bootstrapper.rs`) + // means one of them lands E11000. RFC-0003 §10.3 + // requires repeated `UpdateTable` calls with the same + // specification to not corrupt state — swallowing + // this E11000 makes the redundant call a no-op rather + // than a wire-visible 500. + let is_dup = matches!( + *e.kind, + mongodb::error::ErrorKind::Write(mongodb::error::WriteFailure::WriteError( + mongodb::error::WriteError { code: 11000, .. } + )) + ); + if !is_dup { + return Err(StorageError::Internal(e.to_string())); + } + } + } + Ok(()) + } + + /// Draw the next sequence number for a shard *inside* the given + /// transaction session. + /// + /// Sequence assignment must participate in the same transaction as the + /// stream-record insert, otherwise a fast writer B can obtain seq=6 + /// and commit before a slow writer A (which obtained seq=5) commits. + /// A consumer polling at `after_sequence_number=cursor` between B's + /// commit and A's commit sees seq=6 and advances past it; when A + /// finally commits, seq=5 lands behind the cursor and is never + /// returned. RFC-0003 §5.1 (atomicity with data writes) and §5.2 + /// (per-shard ordering). + /// + /// Placing the counter increment inside the session also serializes + /// concurrent writers on the same shard: two writes racing to + /// $inc the same counter under snapshot isolation will conflict at + /// commit time, so the loser retries — the transaction retry loop + /// upstream in the caller (see D-C3 followup) handles this. + pub(crate) async fn next_sequence_number_in_session( + &self, + shard_id: &str, + session: &mut mongodb::ClientSession, + ) -> Result { + let counters_coll = self.data_db.collection::("counters"); + let opts = mongodb::options::FindOneAndUpdateOptions::builder() + .upsert(true) + .return_document(mongodb::options::ReturnDocument::After) + .build(); + let counter_id = format!("stream_seq:{shard_id}"); + let doc = counters_coll + .find_one_and_update( + doc! { "_id": counter_id }, + doc! { "$inc": { "value": 1_i64 } }, + ) + .with_options(opts) + .session(&mut *session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + StorageError::Internal("Failed to generate sequence number".to_owned()) + })?; + + let seq_val = doc.get_i64("value").unwrap_or(1); + Ok(format!("{seq_val:021}")) + } + + /// Resolve the shard_id for a given (account, table, partition-key) + /// *inside* the given transaction session. Pairs with + /// `next_sequence_number_in_session` so the shard set the write is + /// routed to is read at the same snapshot as the sequence draw. + pub(crate) async fn assign_shard_in_session( + &self, + account_id: &str, + table_name: &str, + partition_key: &str, + session: &mut mongodb::ClientSession, + ) -> Result { + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { "_id": { "account_id": account_id, "table_name": table_name } }) + .session(&mut *session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::Internal(format!("Table {table_name} not found")))?; + let table_id = table_doc.get_str("table_id").unwrap_or_default(); + + let shards_coll = self.data_db.collection::("stream_shards"); + let opts = FindOptions::builder().sort(doc! { "shard_id": 1 }).build(); + let mut cursor = shards_coll + .find(doc! { "table_id": table_id }) + .with_options(opts) + .session(&mut *session) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let shard_docs: Vec = cursor + .stream(&mut *session) + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if shard_docs.is_empty() { + return Err(StorageError::Internal(format!( + "No stream shards for table {table_name}" + ))); + } + + let shard_ids: Vec<&str> = shard_docs + .iter() + .filter_map(|d| d.get_str("shard_id").ok()) + .collect(); + + let hash = crc32fast::hash(partition_key.as_bytes()); + #[allow(clippy::cast_possible_truncation)] + let idx = (hash as usize) % shard_ids.len(); + Ok(shard_ids[idx].to_owned()) + } + + /// Delete every stream_shards document for a given table_id, and every + /// stream_records document written to any of its shards. Invoked from + /// `delete_table_impl` so that a table recreated with the same name + /// (which will get a fresh table_id) cannot inherit the deleted table's + /// stream history. + pub(crate) async fn cleanup_stream_state_for_table( + &self, + table_id: &str, + ) -> Result<(), StorageError> { + let shards_coll = self.data_db.collection::("stream_shards"); + let records_coll = self.data_db.collection::("stream_records"); + + // Collect shard_ids for this table so we can delete their records. + // Records don't carry table_id directly — they're addressed by shard_id. + let cursor = shards_coll + .find(doc! { "table_id": table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let shard_docs: Vec = cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let shard_ids: Vec = shard_docs + .iter() + .filter_map(|d| d.get_str("shard_id").ok().map(str::to_owned)) + .collect(); + + if !shard_ids.is_empty() { + records_coll + .delete_many(doc! { "shard_id": { "$in": &shard_ids } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + shards_coll + .delete_many(doc! { "table_id": table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Sequence-number counters are keyed as "stream_seq:". + let counters_coll = self.data_db.collection::("counters"); + let counter_ids: Vec = shard_ids + .iter() + .map(|sid| format!("stream_seq:{sid}")) + .collect(); + if !counter_ids.is_empty() { + counters_coll + .delete_many(doc! { "_id": { "$in": &counter_ids } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + Ok(()) + } +} + +impl StreamEngine for MongoEngine { + /// Superseded by `MongoEngine::write_stream_inline_in_session` which + /// writes the stream record inside the same session as the data + /// write (D-C2 / RFC-0003 §3.2). The trait method has no callers + /// in the mongo backend after that change — it lives on only + /// because the `StreamEngine` trait still declares it. If invoked + /// externally, it would race against concurrent data writes: this + /// path does not enroll in any transaction and can commit a stream + /// record whose base-table write later rolls back. Return an + /// explicit error rather than performing a subtly-wrong write. + fn write_stream_record( + &self, + _account_id: &str, + _record: &StreamRecord, + _shard_id: &str, + _table_name: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + Box::pin(async move { + Err(StorageError::Internal( + "MongoDB backend: non-transactional write_stream_record is unused; \ + the data-plane path always routes through the in-session variant" + .to_owned(), + )) + }) + } + + fn get_stream_records( + &self, + account_id: &str, + shard_id: &str, + after_sequence: Option<&str>, + limit: i64, + ) -> BoxFuture<'_, StreamRecordsResult> { + let account_id = account_id.to_owned(); + let shard_id = shard_id.to_owned(); + let after_sequence = after_sequence.map(std::borrow::ToOwned::to_owned); + Box::pin(async move { + // Ownership guard: only return records if the shard's backing table + // belongs to the calling account. `stream_shards`/`stream_records` + // live in the data database while the `tables` catalog (which + // carries account_id inside its compound `_id`) lives in the catalog + // database, so ownership is resolved in two steps across the two + // databases: shard_id -> table_id (data db), then table_id + + // account_id (catalog db). Mirrors the postgres backend. + let shards_coll = self.data_db.collection::("stream_shards"); + let shard_doc = shards_coll + .find_one(doc! { "shard_id": &shard_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let owned = match shard_doc.as_ref().and_then(|d| d.get_str("table_id").ok()) { + // Look the table up by its globally-unique `table_id` (a + // top-level field), then compare the owning account_id read out + // of the compound `_id` subdocument in Rust. account_id lives + // only inside the embedded `_id` doc; resolving ownership in + // Rust after a single table_id lookup keeps the check explicit + // and avoids depending on query-time behaviour of a partial + // `_id.account_id` path. + Some(table_id) => self + .catalog_db + .collection::("tables") + .find_one(doc! { "table_id": table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .as_ref() + .and_then(|t| t.get_document("_id").ok()) + .and_then(|id| id.get_str("account_id").ok()) + .is_some_and(|owner| owner == account_id), + None => false, + }; + // A shard iterator that resolves to a shard the caller does not own + // (catalog step fails) or one that does not exist (data step fails) + // is rejected identically. Real DynamoDB returns + // `ValidationException: Invalid ShardIterator` for a GetRecords + // iterator it did not issue, and does not distinguish "exists but not + // yours" from "does not exist" — so neither do we. + if !owned { + return Err(StorageError::Validation("Invalid ShardIterator".to_owned())); + } + + let records_coll = self.data_db.collection::("stream_records"); + + let filter = if let Some(ref after) = after_sequence { + doc! { + "shard_id": &shard_id, + "sequence_number": { "$gt": after }, + } + } else { + doc! { "shard_id": &shard_id } + }; + + let opts = FindOptions::builder() + .sort(doc! { "sequence_number": 1 }) + .limit(limit) + .build(); + + let cursor = records_coll + .find(filter) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let docs: Vec = cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let records: Vec = docs + .into_iter() + .map(|d| { + let record_bson = d + .get("record_data") + .ok_or_else(|| StorageError::Internal("Missing record_data".to_owned()))?; + let json_val: serde_json::Value = bson::from_bson(record_bson.clone()) + .map_err(|e| StorageError::Internal(e.to_string()))?; + serde_json::from_value(json_val) + .map_err(|e| StorageError::Internal(e.to_string())) + }) + .collect::, _>>()?; + + let last_seq = records.last().map(|r| r.dynamodb.sequence_number.clone()); + Ok((records, last_seq)) + }) + } + + fn describe_stream( + &self, + account_id: &str, + input: &DescribeStreamInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let stream_arn_val = input.stream_arn.clone(); + let limit = input.limit; + let exclusive_start_shard_id = input.exclusive_start_shard_id.clone(); + Box::pin(async move { + let (table_name, stream_label) = parse_stream_arn(&stream_arn_val)?; + + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { + "_id": { "account_id": &account_id, "table_name": &table_name }, + "stream_label": &stream_label, + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + StorageError::TableNotFound(format!( + "Requested resource not found: Stream: {stream_arn_val} not found." + )) + })?; + + let key_schema = table_doc + .get("key_schema") + .and_then(|b| bson::from_bson(b.clone()).ok()) + .ok_or_else(|| StorageError::Internal("Missing key_schema".to_owned()))?; + + let stream_view_type = table_doc + .get("stream_specification") + .and_then(|b| { + let json: serde_json::Value = bson::from_bson(b.clone()).ok()?; + json.get("StreamViewType") + .and_then(|sv| serde_json::from_value::(sv.clone()).ok()) + }) + .unwrap_or(StreamViewType::KeysOnly); + + let table_status = table_doc.get_str("table_status").unwrap_or("ACTIVE"); + let table_id = table_doc.get_str("table_id").unwrap_or_default(); + + let limit = limit.unwrap_or(100); + let shards_coll = self.data_db.collection::("stream_shards"); + + let filter = if let Some(ref start) = exclusive_start_shard_id { + doc! { + "table_id": table_id, + "shard_id": { "$gt": start }, + } + } else { + doc! { "table_id": table_id } + }; + + let opts = FindOptions::builder() + .sort(doc! { "shard_id": 1 }) + .limit(limit + 1) + .build(); + + let cursor = shards_coll + .find(filter) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let shard_docs: Vec = cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + #[allow(clippy::cast_sign_loss)] + let limit_usize = limit as usize; + let last_shard = if shard_docs.len() > limit_usize { + shard_docs.get(limit_usize - 1).and_then(|d| { + d.get_str("shard_id") + .ok() + .map(std::borrow::ToOwned::to_owned) + }) + } else { + None + }; + + let shards: Vec = shard_docs + .into_iter() + .take(limit_usize) + .filter_map(|d| { + Some(Shard { + shard_id: d.get_str("shard_id").ok()?.to_owned(), + parent_shard_id: d + .get_str("parent_shard_id") + .ok() + .map(std::borrow::ToOwned::to_owned), + sequence_number_range: SequenceNumberRange { + starting_sequence_number: d + .get_str("starting_sequence_number") + .ok()? + .to_owned(), + ending_sequence_number: d + .get_str("ending_sequence_number") + .ok() + .map(std::borrow::ToOwned::to_owned), + }, + }) + }) + .collect(); + + let stream_status = if table_status == "DELETING" { + StreamStatus::Disabling + } else { + StreamStatus::Enabled + }; + + Ok(StreamDescription { + stream_arn: stream_arn_val, + stream_label, + stream_status, + stream_view_type, + table_name, + key_schema, + shards, + last_evaluated_shard_id: last_shard, + }) + }) + } + + fn list_streams( + &self, + account_id: &str, + table_name: Option<&str>, + limit: i64, + exclusive_start_stream_arn: Option<&str>, + ) -> BoxFuture<'_, StreamListResult> { + let account_id = account_id.to_owned(); + let table_name = table_name.map(std::borrow::ToOwned::to_owned); + let exclusive_start_stream_arn = + exclusive_start_stream_arn.map(std::borrow::ToOwned::to_owned); + Box::pin(async move { + let tables_coll = self.catalog_db.collection::("tables"); + + let mut filter = doc! { + "_id.account_id": &account_id, + "stream_label": { "$ne": null }, + }; + + if let Some(ref tn) = table_name { + filter.insert("_id.table_name", tn.as_str()); + } + + if let Some(ref start_arn) = exclusive_start_stream_arn { + let (start_table, start_label) = parse_stream_arn(start_arn)?; + if table_name.is_some() { + filter.insert("stream_label", doc! { "$gt": &start_label }); + } else { + filter.insert( + "$or", + bson::bson!([ + { "_id.table_name": { "$gt": &start_table } }, + { "_id.table_name": &start_table, "stream_label": { "$gt": &start_label } } + ]), + ); + } + } + + let opts = FindOptions::builder() + .sort(doc! { "_id.table_name": 1, "stream_label": 1 }) + .limit(limit + 1) + .build(); + + let cursor = tables_coll + .find(filter) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let docs: Vec = cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + #[allow(clippy::cast_sign_loss)] + let limit_usize = limit as usize; + + let summaries: Vec = docs + .iter() + .take(limit_usize) + .filter_map(|d| { + let id = d.get_document("_id").ok()?; + let tn = id.get_str("table_name").ok()?; + let label = d.get_str("stream_label").ok()?; + Some(StreamSummary { + stream_arn: stream_arn(&self.region, &account_id, tn, label), + stream_label: label.to_owned(), + table_name: tn.to_owned(), + }) + }) + .collect(); + + let last_arn = if docs.len() > limit_usize { + summaries.last().map(|s| s.stream_arn.clone()) + } else { + None + }; + + Ok((summaries, last_arn)) + }) + } + + fn cleanup_expired_stream_records( + &self, + retention_hours: i64, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { + let records_coll = self.data_db.collection::("stream_records"); + let cutoff = time::OffsetDateTime::now_utc() + - std::time::Duration::from_secs(retention_hours as u64 * 3600); + let cutoff_bson = BsonDateTime::from_millis(cutoff.unix_timestamp() * 1000); + let result = records_coll + .delete_many(doc! { "created_at": { "$lt": cutoff_bson } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(result.deleted_count) + }) + } + + fn assign_shard( + &self, + account_id: &str, + table_name: &str, + partition_key: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_owned(); + let table_name = table_name.to_owned(); + let partition_key = partition_key.to_owned(); + Box::pin(async move { + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { "_id": { "account_id": &account_id, "table_name": &table_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::Internal(format!("Table {table_name} not found")))?; + let table_id = table_doc.get_str("table_id").unwrap_or_default(); + + let shards_coll = self.data_db.collection::("stream_shards"); + let opts = FindOptions::builder().sort(doc! { "shard_id": 1 }).build(); + let cursor = shards_coll + .find(doc! { "table_id": table_id }) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let shard_docs: Vec = cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if shard_docs.is_empty() { + return Err(StorageError::Internal(format!( + "No stream shards for table {table_name}" + ))); + } + + let shard_ids: Vec<&str> = shard_docs + .iter() + .filter_map(|d| d.get_str("shard_id").ok()) + .collect(); + + let hash = crc32fast::hash(partition_key.as_bytes()); + #[allow(clippy::cast_possible_truncation)] + let idx = (hash as usize) % shard_ids.len(); + Ok(shard_ids[idx].to_owned()) + }) + } + + fn next_sequence_number(&self, shard_id: &str) -> BoxFuture<'_, Result> { + let shard_id = shard_id.to_owned(); + Box::pin(async move { + // Per-shard atomic counter. DynamoDB Streams' contract is that + // sequence numbers are strictly monotonic *within a shard* and + // independent *across shards*. A single global counter would + // couple the sequence spaces of unrelated shards — a writer + // pushing records into shard B would advance the counter shard A + // reads back, producing non-contiguous sequence numbers on + // shard A's GetRecords pages. Keying the counter document by + // shard_id preserves the per-shard monotonicity guarantee. + let counters_coll = self.data_db.collection::("counters"); + let opts = mongodb::options::FindOneAndUpdateOptions::builder() + .upsert(true) + .return_document(mongodb::options::ReturnDocument::After) + .build(); + let counter_id = format!("stream_seq:{shard_id}"); + let doc = counters_coll + .find_one_and_update( + doc! { "_id": counter_id }, + doc! { "$inc": { "value": 1_i64 } }, + ) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| { + StorageError::Internal("Failed to generate sequence number".to_owned()) + })?; + + let seq_val = doc.get_i64("value").unwrap_or(1); + Ok(format!("{seq_val:021}")) + }) + } + + fn validate_shard( + &self, + account_id: &str, + stream_arn_val: &str, + shard_id: &str, + ) -> BoxFuture<'_, Result<(), StorageError>> { + let account_id = account_id.to_owned(); + let stream_arn_val = stream_arn_val.to_owned(); + let shard_id = shard_id.to_owned(); + Box::pin(async move { + let (table_name, stream_label) = parse_stream_arn(&stream_arn_val)?; + + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { + "_id": { "account_id": &account_id, "table_name": &table_name }, + "stream_label": &stream_label, + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let Some(table_doc) = table_doc else { + return Err(StorageError::TableNotFound(format!( + "Requested resource not found: Stream: {stream_arn_val} not found." + ))); + }; + + let table_id = table_doc.get_str("table_id").unwrap_or_default(); + + let shards_coll = self.data_db.collection::("stream_shards"); + let exists = shards_coll + .find_one(doc! { "shard_id": &shard_id, "table_id": table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if exists.is_none() { + return Err(StorageError::TableNotFound(format!( + "Requested resource not found: Stream: {stream_arn_val} not found." + ))); + } + Ok(()) + }) + } + + fn latest_sequence_number( + &self, + shard_id: &str, + ) -> BoxFuture<'_, Result, StorageError>> { + let shard_id = shard_id.to_owned(); + Box::pin(async move { + let records_coll = self.data_db.collection::("stream_records"); + let opts = FindOptions::builder() + .sort(doc! { "sequence_number": -1 }) + .limit(1) + .build(); + let cursor = records_coll + .find(doc! { "shard_id": &shard_id }) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let docs: Vec = cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(docs.first().and_then(|d| { + d.get_str("sequence_number") + .ok() + .map(std::borrow::ToOwned::to_owned) + })) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn event_name_maps_to_ddb_wire_format() { + assert_eq!(event_name_ddb_str(StreamEventName::Insert), "INSERT"); + assert_eq!(event_name_ddb_str(StreamEventName::Modify), "MODIFY"); + assert_eq!(event_name_ddb_str(StreamEventName::Remove), "REMOVE"); + } + + #[test] + fn shard_id_embeds_table_id_not_table_name() { + let table_id = "550e8400-e29b-41d4-a716-446655440000"; + assert_eq!( + build_shard_id(table_id, 0), + "shardId-550e8400-e29b-41d4-a716-446655440000-000000000000" + ); + assert_eq!( + build_shard_id(table_id, 3), + "shardId-550e8400-e29b-41d4-a716-446655440000-000000000003" + ); + } + + #[test] + fn shard_ids_for_different_table_ids_do_not_collide() { + // Regression test for RFC-0003 §5.3 (account and table isolation). + // Two tables with the same shard index (0) must have different + // shard_ids so a caller in one tenant cannot address the other's + // shard. + let table_a = "550e8400-e29b-41d4-a716-446655440000"; + let table_b = "6ba7b810-9dad-11d1-80b4-00c04fd430c8"; + assert_ne!(build_shard_id(table_a, 0), build_shard_id(table_b, 0)); + } + + #[test] + fn shard_id_format_is_stable_across_shards_of_same_table() { + // Same table_id, different shard index → deterministic ordering. + let table_id = "550e8400-e29b-41d4-a716-446655440000"; + let s0 = build_shard_id(table_id, 0); + let s1 = build_shard_id(table_id, 1); + let s2 = build_shard_id(table_id, 2); + let s3 = build_shard_id(table_id, 3); + assert!(s0 < s1); + assert!(s1 < s2); + assert!(s2 < s3); + } +} diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs new file mode 100644 index 00000000..2bde96b4 --- /dev/null +++ b/crates/storage-mongodb/src/table_engine.rs @@ -0,0 +1,1433 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `TableEngine` trait implementation for `MongoEngine`. + +use bson::{Document, doc}; +use futures::future::BoxFuture; +use mongodb::IndexModel; +use mongodb::options::{Collation, IndexOptions}; + +use extenddb_core::types::{ + AttributeDefinition, BillingMode, BillingModeSummary, CreateTableInput, DeleteTableInput, + DescribeTableInput, GsiDescription, IndexInfo, IndexType, KeySchemaElement, ListTablesInput, + ListTablesOutput, LsiDescription, OnDemandThroughput, ProvisionedThroughputDescription, + ScalarAttributeType, SseDescription, SseType, TableDescription, TableKeyInfo, TableStatus, + UpdateTableInput, +}; +use extenddb_storage::TableEngine; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::{index_arn, sk_info, stream_arn, table_arn}; + +use crate::MongoEngine; +use crate::data::data_collection_name; + +/// Format a timestamp as a DynamoDB-style stream label: +/// `YYYY-MM-DDThh:mm:ss` (second precision, no timezone). +/// +/// Matches the postgres backend's +/// `to_char(NOW(), 'YYYY-MM-DD"T"HH24:MI:SS')` output byte-for-byte +/// so a stream ARN issued by one backend is parseable by tooling that +/// only ever saw the other. The `time` crate's `Iso8601::DEFAULT` +/// emits nanoseconds with a trailing `Z` — pushing that through AWS- +/// SDK parsers or postgres-shaped tests failed unpredictably. D-m8. +fn format_stream_label(now: time::OffsetDateTime) -> String { + format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}", + now.year(), + u8::from(now.month()), + now.day(), + now.hour(), + now.minute(), + now.second(), + ) +} + +impl TableEngine for MongoEngine { + fn create_table( + &self, + account_id: &str, + input: CreateTableInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + Box::pin(async move { self.create_table_impl(&account_id, input).await }) + } + + fn delete_table( + &self, + account_id: &str, + input: DeleteTableInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + Box::pin(async move { self.delete_table_impl(&account_id, input).await }) + } + + fn describe_table( + &self, + account_id: &str, + input: DescribeTableInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + Box::pin(async move { + self.describe_table_impl(&account_id, &input.table_name) + .await + }) + } + + fn list_tables( + &self, + account_id: &str, + input: ListTablesInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + Box::pin(async move { self.list_tables_impl(&account_id, input).await }) + } + + fn update_table( + &self, + account_id: &str, + input: UpdateTableInput, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + Box::pin(async move { self.update_table_impl(&account_id, input).await }) + } + + fn table_key_info( + &self, + account_id: &str, + table_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + Box::pin(async move { self.table_key_info_impl(&account_id, &table_name).await }) + } + + fn index_info( + &self, + account_id: &str, + table_name: &str, + index_name: &str, + ) -> BoxFuture<'_, Result> { + let account_id = account_id.to_string(); + let table_name = table_name.to_string(); + let index_name = index_name.to_string(); + Box::pin(async move { + self.index_info_impl(&account_id, &table_name, &index_name) + .await + }) + } + + fn index_info_by_table_id( + &self, + table_id: &str, + index_name: &str, + ) -> BoxFuture<'_, Result> { + let table_id = table_id.to_string(); + let index_name = index_name.to_string(); + Box::pin(async move { + self.index_info_by_table_id_impl(&table_id, &index_name) + .await + }) + } +} + +impl MongoEngine { + async fn create_table_impl( + &self, + account_id: &str, + input: CreateTableInput, + ) -> Result { + Self::validate_account_id(account_id)?; + + let table_id = uuid::Uuid::new_v4().to_string(); + let table_arn_val = table_arn(&self.region, account_id, &input.table_name); + let billing_mode = input.billing_mode.unwrap_or(BillingMode::Provisioned); + let deletion_protection = input.deletion_protection_enabled.unwrap_or(false); + + let now = time::OffsetDateTime::now_utc(); + let creation_epoch = now.unix_timestamp() as f64; + + // Build the table metadata document + let key_schema_bson = + bson::to_bson(&input.key_schema).map_err(|e| StorageError::Internal(e.to_string()))?; + let attr_defs_bson = bson::to_bson(&input.attribute_definitions) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let billing_str = match billing_mode { + BillingMode::Provisioned => "PROVISIONED", + BillingMode::PayPerRequest => "PAY_PER_REQUEST", + }; + let pt_bson = input + .provisioned_throughput + .as_ref() + .map(bson::to_bson) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + let stream_bson = input + .stream_specification + .as_ref() + .map(bson::to_bson) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Compute stream label early so it's stored in the table document + let stream_label_opt = if input + .stream_specification + .as_ref() + .is_some_and(|ss| ss.stream_enabled) + { + Some(format_stream_label(now)) + } else { + None + }; + let stream_label_bson = stream_label_opt + .as_ref() + .map_or(bson::Bson::Null, |l| bson::Bson::String(l.clone())); + + // Persist TableClass / SSESpecification / OnDemandThroughput on the + // catalog doc so DescribeTable can return TableClassSummary, + // SSEDescription, and OnDemandThroughput respectively. Mirrors the + // postgres backend at storage-postgres/src/create_table.rs:100-102. + let table_class_bson = input + .table_class + .as_deref() + .map_or(bson::Bson::Null, |tc| bson::Bson::String(tc.to_owned())); + let sse_spec_bson = input.sse_specification.as_ref().map_or_else( + || bson::Bson::Null, + |v| bson::to_bson(v).unwrap_or(bson::Bson::Null), + ); + let on_demand_bson = input.on_demand_throughput.as_ref().map_or_else( + || bson::Bson::Null, + |v| bson::to_bson(v).unwrap_or(bson::Bson::Null), + ); + + // Enter CREATING with a scheduled transition to ACTIVE, unless + // control_plane_delay_seconds is 0 (then go straight to ACTIVE). The + // background control_plane_worker flips CREATING -> ACTIVE once the + // transition time passes; during the window data-plane ops on the + // table return ResourceNotFound, matching DynamoDB and the postgres + // backend. + let delay_secs = self.control_plane_delay_seconds().await; + let (table_status, status_transition_at): (&str, bson::Bson) = if delay_secs <= 0.0 { + ("ACTIVE", bson::Bson::Null) + } else { + let at = bson::DateTime::now().timestamp_millis() + (delay_secs * 1000.0) as i64; + ( + "CREATING", + bson::Bson::DateTime(bson::DateTime::from_millis(at)), + ) + }; + + let table_doc = doc! { + "_id": { "account_id": account_id, "table_name": &input.table_name }, + "key_schema": key_schema_bson, + "attribute_definitions": attr_defs_bson, + "billing_mode": billing_str, + "provisioned_throughput": pt_bson.unwrap_or(bson::Bson::Null), + "stream_specification": stream_bson.unwrap_or(bson::Bson::Null), + "table_status": table_status, + "status_transition_at": status_transition_at, + "creation_date_time": bson::DateTime::from_millis((creation_epoch * 1000.0) as i64), + "table_size_bytes": 0_i64, + "item_count": 0_i64, + "table_arn": &table_arn_val, + "table_id": &table_id, + "deletion_protection_enabled": deletion_protection, + "ttl_attribute": bson::Bson::Null, + "stream_label": stream_label_bson, + "table_class": table_class_bson, + "sse_specification": sse_spec_bson, + "on_demand_throughput": on_demand_bson, + }; + + let tables_coll = self.catalog_db.collection::("tables"); + tables_coll.insert_one(table_doc).await.map_err(|e| { + if e.to_string().contains("E11000") { + StorageError::TableAlreadyExists(input.table_name.clone()) + } else { + StorageError::Internal(e.to_string()) + } + })?; + + // Create the data collection with appropriate indexes + let coll_name = data_collection_name(&table_id); + self.data_db + .create_collection(&coll_name) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let data_coll = self.data_db.collection::(&coll_name); + + // Create index based on sort key type + if let Some((_, sk_type)) = sk_info(&input.key_schema, &input.attribute_definitions) { + let sk_field = match sk_type { + ScalarAttributeType::S => "sk_s", + ScalarAttributeType::N => "sk_n", + ScalarAttributeType::B => "sk_b", + }; + let index_keys = doc! { "pk": 1, sk_field: 1 }; + let mut index_opts = IndexOptions::builder().unique(true).build(); + // Use simple collation for string sort keys (byte-order) + if sk_type == ScalarAttributeType::S { + index_opts.collation = + Some(Collation::builder().locale("simple".to_string()).build()); + } + let index = IndexModel::builder() + .keys(index_keys) + .options(index_opts) + .build(); + data_coll + .create_index(index) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } else { + // PK-only index + let index = IndexModel::builder() + .keys(doc! { "pk": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(); + data_coll + .create_index(index) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + // Initialize stream shards if streaming is enabled. shard_id is + // derived from table_id (UUID), never table_name — see + // stream_engine::build_shard_id for the security rationale. + if stream_label_opt.is_some() { + self.init_stream_shards(&table_id).await?; + } + + // Handle GSI creation + let gsi_descriptions = if let Some(ref gsis) = input.global_secondary_indexes { + let mut descs = Vec::new(); + for gsi in gsis { + let index_id = uuid::Uuid::new_v4().to_string(); + let index_arn_val = + index_arn(&self.region, account_id, &input.table_name, &gsi.index_name); + + // Store index metadata in catalog + let key_schema_bson = bson::to_bson(&gsi.key_schema) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let projection_bson = bson::to_bson(&gsi.projection) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let index_pt_bson = gsi + .provisioned_throughput + .as_ref() + .map(bson::to_bson) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let index_doc = doc! { + "_id": { "table_id": &table_id, "index_name": &gsi.index_name }, + "index_id": &index_id, + "index_type": "GSI", + "key_schema": key_schema_bson, + "projection": projection_bson, + "index_status": "ACTIVE", + "provisioned_throughput": index_pt_bson.unwrap_or(bson::Bson::Null), + }; + + self.catalog_db + .collection::("indexes") + .insert_one(index_doc) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + self.create_index_data_collection( + &index_id, + &gsi.key_schema, + &input.key_schema, + &input.attribute_definitions, + ) + .await?; + + descs.push(GsiDescription { + index_name: gsi.index_name.clone(), + key_schema: gsi.key_schema.clone(), + projection: gsi.projection.clone(), + index_status: "ACTIVE".to_string(), + provisioned_throughput: gsi.provisioned_throughput.as_ref().map(|pt| { + ProvisionedThroughputDescription { + read_capacity_units: pt.read_capacity_units, + write_capacity_units: pt.write_capacity_units, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + } + }), + index_size_bytes: 0, + item_count: 0, + index_arn: index_arn_val, + }); + } + Some(descs) + } else { + None + }; + + // Handle LSI creation + let lsi_descriptions = if let Some(ref lsis) = input.local_secondary_indexes { + let mut descs = Vec::new(); + for lsi in lsis { + let index_id = uuid::Uuid::new_v4().to_string(); + let index_arn_val = + index_arn(&self.region, account_id, &input.table_name, &lsi.index_name); + + let key_schema_bson = bson::to_bson(&lsi.key_schema) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let projection_bson = bson::to_bson(&lsi.projection) + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let index_doc = doc! { + "_id": { "table_id": &table_id, "index_name": &lsi.index_name }, + "index_id": &index_id, + "index_type": "LSI", + "key_schema": key_schema_bson, + "projection": projection_bson, + "index_status": "ACTIVE", + "provisioned_throughput": bson::Bson::Null, + }; + + self.catalog_db + .collection::("indexes") + .insert_one(index_doc) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + self.create_index_data_collection( + &index_id, + &lsi.key_schema, + &input.key_schema, + &input.attribute_definitions, + ) + .await?; + + descs.push(LsiDescription { + index_name: lsi.index_name.clone(), + key_schema: lsi.key_schema.clone(), + projection: lsi.projection.clone(), + index_size_bytes: 0, + item_count: 0, + index_arn: index_arn_val, + }); + } + Some(descs) + } else { + None + }; + + // Build stream ARN from pre-computed label + let stream_arn_opt = stream_label_opt + .as_ref() + .map(|label| stream_arn(&self.region, account_id, &input.table_name, label)); + + let pt_desc = match &input.provisioned_throughput { + Some(pt) => ProvisionedThroughputDescription { + read_capacity_units: pt.read_capacity_units, + write_capacity_units: pt.write_capacity_units, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }, + None => ProvisionedThroughputDescription { + read_capacity_units: 0, + write_capacity_units: 0, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }, + }; + + let billing_summary = if billing_mode == BillingMode::PayPerRequest { + Some(BillingModeSummary { + billing_mode: BillingMode::PayPerRequest, + last_update_to_pay_per_request_date_time: Some(creation_epoch), + }) + } else { + None + }; + + // Store initial tags if provided + if let Some(ref tags) = input.tags { + let tags_coll = self.catalog_db.collection::("tags"); + for tag in tags { + tags_coll + .update_one( + doc! { "resource_arn": &table_arn_val, "tag_key": &tag.key }, + doc! { "$set": { "resource_arn": &table_arn_val, "tag_key": &tag.key, "tag_value": &tag.value } }, + ) + .upsert(true) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + } + + // Derive the SSEDescription from the SSESpecification, mirroring the + // postgres backend (table_helpers.rs:329-346). The specification's + // `Enabled: true` becomes a KMS-status ENABLED description with a + // synthesized ARN. Anything else omits the field. + let sse_description = input.sse_specification.as_ref().and_then(|spec| { + let enabled = spec + .get("Enabled") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + if enabled { + Some(SseDescription { + status: "ENABLED".to_owned(), + sse_type: Some(SseType::KMS), + kms_master_key_arn: Some(format!( + "arn:aws:kms:{}:{}:key/default", + self.region, account_id + )), + }) + } else { + None + } + }); + let table_class_summary = input + .table_class + .as_deref() + .map(|tc| serde_json::json!({ "TableClass": tc })); + + Ok(TableDescription { + table_name: input.table_name, + key_schema: input.key_schema, + attribute_definitions: input.attribute_definitions, + table_status: if table_status == "CREATING" { + TableStatus::Creating + } else { + TableStatus::Active + }, + creation_date_time: creation_epoch, + table_size_bytes: 0, + item_count: 0, + table_arn: table_arn_val, + table_id, + provisioned_throughput: pt_desc, + billing_mode_summary: billing_summary, + global_secondary_indexes: gsi_descriptions, + local_secondary_indexes: lsi_descriptions, + stream_specification: input.stream_specification, + latest_stream_arn: stream_arn_opt, + latest_stream_label: stream_label_opt, + deletion_protection_enabled: deletion_protection, + sse_description, + table_class_summary, + on_demand_throughput: input.on_demand_throughput, + }) + } + + async fn delete_table_impl( + &self, + account_id: &str, + input: DeleteTableInput, + ) -> Result { + Self::validate_account_id(account_id)?; + + // Fetch the table first + let desc = self + .describe_table_impl(account_id, &input.table_name) + .await?; + + // Check deletion protection + if desc.deletion_protection_enabled { + return Err(StorageError::DeletionProtected(input.table_name.clone())); + } + + // Mark as DELETING + let tables_coll = self.catalog_db.collection::("tables"); + tables_coll + .update_one( + doc! { "_id": { "account_id": account_id, "table_name": &input.table_name } }, + doc! { "$set": { "table_status": "DELETING" } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Drop the data collection + let coll_name = data_collection_name(&desc.table_id); + self.data_db + .collection::(&coll_name) + .drop() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Delete index entries + self.catalog_db + .collection::("indexes") + .delete_many(doc! { "_id.table_id": &desc.table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + self.gsi_cache_invalidate(&desc.table_id); + + // Delete stream_shards, stream_records, and their sequence counters + // for this table. Prevents a table recreated with the same name + // (which will get a fresh table_id) from inheriting the deleted + // table's stream history. RFC-0003 §8.2 (table-name reuse). + self.cleanup_stream_state_for_table(&desc.table_id).await?; + + // Delete the table metadata + tables_coll + .delete_one( + doc! { "_id": { "account_id": account_id, "table_name": &input.table_name } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + Ok(TableDescription { + table_status: TableStatus::Deleting, + ..desc + }) + } + + pub(crate) async fn describe_table_impl( + &self, + account_id: &str, + table_name: &str, + ) -> Result { + Self::validate_account_id(account_id)?; + + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { "_id": { "account_id": account_id, "table_name": table_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(table_name.to_string()))?; + + self.doc_to_table_description(&table_doc).await + } + + async fn list_tables_impl( + &self, + account_id: &str, + input: ListTablesInput, + ) -> Result { + Self::validate_account_id(account_id)?; + + use futures::TryStreamExt; + + let limit = i64::from(input.limit.unwrap_or(100)); + let tables_coll = self.catalog_db.collection::("tables"); + + let mut filter = doc! { "_id.account_id": account_id }; + if let Some(ref start) = input.exclusive_start_table_name { + filter.insert("_id.table_name", doc! { "$gt": start }); + } + + let opts = mongodb::options::FindOptions::builder() + .sort(doc! { "_id.table_name": 1 }) + .limit(limit + 1) + .projection(doc! { "_id.table_name": 1 }) + .build(); + + let cursor = tables_coll + .find(filter) + .with_options(opts) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let docs: Vec = cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let names: Vec = docs + .iter() + .filter_map(|d| { + d.get_document("_id") + .ok() + .and_then(|id| id.get_str("table_name").ok()) + .map(std::string::ToString::to_string) + }) + .collect(); + + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + let limit_usize = limit as usize; + + if names.len() > limit_usize { + Ok(ListTablesOutput { + last_evaluated_table_name: Some(names[limit_usize - 1].clone()), + table_names: names[..limit_usize].to_vec(), + }) + } else { + Ok(ListTablesOutput { + table_names: names, + last_evaluated_table_name: None, + }) + } + } + + async fn update_table_impl( + &self, + account_id: &str, + input: UpdateTableInput, + ) -> Result { + Self::validate_account_id(account_id)?; + + let tables_coll = self.catalog_db.collection::("tables"); + + // Reject ProvisionedThroughput when the effective billing mode is + // PAY_PER_REQUEST. The effective mode is the requested billing_mode + // when the request changes it, otherwise the table's current mode. + // Real DynamoDB returns "Neither ReadCapacityUnits nor WriteCapacityUnits + // can be specified when BillingMode is PAY_PER_REQUEST". Postgres does + // this same check under a FOR UPDATE row lock in update_table.rs; mongo + // reads the current billing_mode via find_one and relies on the fact + // that any concurrent billing-mode change would then be rejected by its + // own no-op check (not yet implemented — see R-8 followup). + if input.provisioned_throughput.is_some() { + let effective_ppr = match input.billing_mode { + Some(BillingMode::PayPerRequest) => true, + Some(BillingMode::Provisioned) => false, + None => { + let table_doc = tables_coll + .find_one(doc! { + "_id": { "account_id": account_id, "table_name": &input.table_name }, + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + table_doc + .and_then(|d| d.get_str("billing_mode").ok().map(str::to_owned)) + .as_deref() + == Some("PAY_PER_REQUEST") + } + }; + if effective_ppr { + return Err(StorageError::Validation( + "One or more parameter values were invalid: Neither ReadCapacityUnits nor WriteCapacityUnits can be specified when BillingMode is PAY_PER_REQUEST".to_owned(), + )); + } + } + + // Build update document + let mut update_doc = Document::new(); + + if let Some(billing_mode) = &input.billing_mode { + let billing_str = match billing_mode { + BillingMode::Provisioned => "PROVISIONED", + BillingMode::PayPerRequest => "PAY_PER_REQUEST", + }; + update_doc.insert("billing_mode", billing_str); + } + + if let Some(pt) = &input.provisioned_throughput { + let pt_bson = bson::to_bson(pt).map_err(|e| StorageError::Internal(e.to_string()))?; + update_doc.insert("provisioned_throughput", pt_bson); + } + + if let Some(dp) = input.deletion_protection_enabled { + update_doc.insert("deletion_protection_enabled", dp); + } + + if let Some(tc) = &input.table_class { + update_doc.insert("table_class", tc); + } + + if let Some(odt) = &input.on_demand_throughput { + let odt_bson = bson::to_bson(odt).map_err(|e| StorageError::Internal(e.to_string()))?; + update_doc.insert("on_demand_throughput", odt_bson); + } + + if let Some(ss) = &input.stream_specification { + let ss_bson = bson::to_bson(ss).map_err(|e| StorageError::Internal(e.to_string()))?; + update_doc.insert("stream_specification", ss_bson); + if ss.stream_enabled { + let table_doc = tables_coll + .find_one(doc! { "_id": { "account_id": account_id, "table_name": &input.table_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(input.table_name.clone()))?; + let table_id = table_doc + .get_str("table_id") + .map_err(|_| StorageError::Internal("missing table_id".to_string()))?; + + // Idempotent re-enable: if shards already exist for this + // table, reuse them and preserve the existing + // stream_label. Otherwise a repeat UpdateTable would + // insert duplicate shards (DescribeStream would then + // report N × k) and rotate stream_label, invalidating + // stream ARNs previously handed out to consumers. + let shards_coll = self.data_db.collection::("stream_shards"); + let existing_shard = shards_coll + .find_one(doc! { "table_id": table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if existing_shard.is_none() { + let label = format_stream_label(time::OffsetDateTime::now_utc()); + update_doc.insert("stream_label", &label); + self.init_stream_shards(table_id).await?; + } else if table_doc + .get_str("stream_label") + .ok() + .filter(|s| !s.is_empty()) + .is_none() + { + // Shards exist but the label was cleared by a + // previous disable — restore a fresh label so the + // ARN resolves again. + let label = format_stream_label(time::OffsetDateTime::now_utc()); + update_doc.insert("stream_label", &label); + } + } + } + + if !update_doc.is_empty() { + tables_coll + .update_one( + doc! { "_id": { "account_id": account_id, "table_name": &input.table_name } }, + doc! { "$set": &update_doc }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + + // Handle GSI updates + if let Some(gsi_updates) = &input.global_secondary_index_updates { + for update in gsi_updates { + if let Some(create) = &update.create { + // Fetch table_id + let desc = self + .describe_table_impl(account_id, &input.table_name) + .await?; + let index_id = uuid::Uuid::new_v4().to_string(); + + let key_schema_bson = bson::to_bson(&create.key_schema) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let projection_bson = bson::to_bson(&create.projection) + .map_err(|e| StorageError::Internal(e.to_string()))?; + let pt_bson = create + .provisioned_throughput + .as_ref() + .map(bson::to_bson) + .transpose() + .map_err(|e| StorageError::Internal(e.to_string()))?; + + // Enter CREATING; the background gsi_backfill_worker + // in ttl_worker.rs discovers this row, backfills the + // base table, and flips the status to ACTIVE. Matches + // DDB's async UpdateTable contract (§2.4 in RFC-0003). + // Live writes during the backfill window sync through + // sync_indexes (upserts) so the eventual state is + // convergent regardless of interleaving. + let index_doc = doc! { + "_id": { "table_id": &desc.table_id, "index_name": &create.index_name }, + "index_id": &index_id, + "index_type": "GSI", + "key_schema": key_schema_bson, + "projection": projection_bson, + "index_status": "CREATING", + "provisioned_throughput": pt_bson.unwrap_or(bson::Bson::Null), + }; + + self.catalog_db + .collection::("indexes") + .insert_one(index_doc) + .await + .map_err(|e| { + if e.to_string().contains("E11000") { + StorageError::IndexAlreadyExists(create.index_name.clone()) + } else { + StorageError::Internal(e.to_string()) + } + })?; + + // Pre-create the mongo collection + query indexes + // before the backfill worker starts writing — the + // worker's upserts would work on an un-indexed + // collection but subsequent GetItem/Query traffic + // on the CREATING index would run coll-scans. D-m7. + self.create_index_data_collection( + &index_id, + &create.key_schema, + &desc.key_schema, + &desc.attribute_definitions, + ) + .await?; + + self.gsi_cache_set(&desc.table_id, true); + } + + if let Some(delete) = &update.delete { + let desc = self + .describe_table_impl(account_id, &input.table_name) + .await?; + let result = self.catalog_db.collection::("indexes") + .delete_one(doc! { "_id": { "table_id": &desc.table_id, "index_name": &delete.index_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + if result.deleted_count == 0 { + return Err(StorageError::IndexNotFound(delete.index_name.clone())); + } + + // Invalidate cache — may still have other GSIs + self.gsi_cache_invalidate(&desc.table_id); + } + } + } + + self.describe_table_impl(account_id, &input.table_name) + .await + } + + pub(crate) async fn table_key_info_impl( + &self, + account_id: &str, + table_name: &str, + ) -> Result { + Self::validate_account_id(account_id)?; + + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { "_id": { "account_id": account_id, "table_name": table_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(table_name.to_string()))?; + + self.table_key_info_from_doc(&table_doc, true).await + } + + /// Load `TableKeyInfo` by `table_id`. The tables catalog has a + /// unique index on `table_id`, so this is a single-doc lookup. + /// Used by the GSI backfill worker, which discovers work items + /// keyed by `table_id`. Skips the ACTIVE-status guard so a table + /// that is temporarily in a transient state (CREATING, UPDATING) + /// can still be backfilled — backfill is decoupled from data-plane + /// availability. + pub(crate) async fn table_key_info_by_table_id_impl( + &self, + table_id: &str, + ) -> Result { + let tables_coll = self.catalog_db.collection::("tables"); + let table_doc = tables_coll + .find_one(doc! { "table_id": table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(table_id.to_string()))?; + + self.table_key_info_from_doc(&table_doc, false).await + } + + async fn table_key_info_from_doc( + &self, + table_doc: &Document, + require_active: bool, + ) -> Result { + let id_doc = table_doc + .get_document("_id") + .map_err(|_| StorageError::Internal("missing _id".to_string()))?; + let table_name = id_doc + .get_str("table_name") + .map_err(|_| StorageError::Internal("missing _id.table_name".to_string()))? + .to_string(); + let account_id = id_doc + .get_str("account_id") + .map_err(|_| StorageError::Internal("missing _id.account_id".to_string()))? + .to_string(); + + let status = table_doc.get_str("table_status").unwrap_or("ACTIVE"); + if require_active && status != "ACTIVE" { + // This guard gates data-plane key-schema resolution. DynamoDB + // returns ResourceNotFoundException (not ResourceInUse) for a + // data-plane op against a table that is not yet ACTIVE, matching + // the postgres backend; TableNotActive would map to ResourceInUse. + return Err(StorageError::TableNotFound(table_name)); + } + + let table_id = table_doc + .get_str("table_id") + .map_err(|_| StorageError::Internal("missing table_id".to_string()))? + .to_string(); + + let key_schema_bson = table_doc + .get("key_schema") + .ok_or_else(|| StorageError::Internal("missing key_schema".to_string()))?; + let key_schema: Vec = + bson::from_bson(key_schema_bson.clone()) + .map_err(|e| StorageError::Internal(format!("key_schema parse error: {e}")))?; + + let attr_defs_bson = table_doc + .get("attribute_definitions") + .ok_or_else(|| StorageError::Internal("missing attribute_definitions".to_string()))?; + let attribute_definitions: Vec = + bson::from_bson(attr_defs_bson.clone()) + .map_err(|e| StorageError::Internal(format!("attr_defs parse error: {e}")))?; + + let stream_spec_bson = table_doc.get("stream_specification"); + let stream_specification = stream_spec_bson.and_then(|b| { + if b.as_null().is_some() { + None + } else { + bson::from_bson(b.clone()).ok() + } + }); + + // Load all secondary indexes so per-index consumed capacity can be + // computed from the cached TableKeyInfo without an extra describe_table + // round-trip per write (matches the fields upstream added). + use futures::TryStreamExt; + let indexes_coll = self.catalog_db.collection::("indexes"); + let mut idx_cursor = indexes_coll + .find(doc! { "_id.table_id": &table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let mut global_secondary_indexes = Vec::new(); + let mut local_secondary_indexes = Vec::new(); + while let Some(idx_doc) = idx_cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let info = index_info_from_doc(&idx_doc)?; + match info.index_type { + IndexType::Gsi => global_secondary_indexes.push(info), + IndexType::Lsi => local_secondary_indexes.push(info), + } + } + let has_lsi = !local_secondary_indexes.is_empty(); + + Ok(TableKeyInfo { + table_name, + account_id, + table_id, + base_key_schema: key_schema.clone(), + key_schema, + attribute_definitions, + has_lsi, + global_secondary_indexes, + local_secondary_indexes, + stream_specification, + }) + } + + async fn index_info_impl( + &self, + account_id: &str, + table_name: &str, + index_name: &str, + ) -> Result { + // First, get the table_id + let key_info = self.table_key_info_impl(account_id, table_name).await?; + self.index_info_by_table_id_impl(&key_info.table_id, index_name) + .await + } + + pub(crate) async fn index_info_by_table_id_impl( + &self, + table_id: &str, + index_name: &str, + ) -> Result { + let indexes_coll = self.catalog_db.collection::("indexes"); + let index_doc = indexes_coll + .find_one(doc! { "_id": { "table_id": table_id, "index_name": index_name } }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::IndexNotFound(index_name.to_string()))?; + + let index_id = index_doc + .get_str("index_id") + .map_err(|_| StorageError::Internal("missing index_id".to_string()))? + .to_string(); + let index_type_str = index_doc + .get_str("index_type") + .map_err(|_| StorageError::Internal("missing index_type".to_string()))?; + let index_type = match index_type_str { + "GSI" => IndexType::Gsi, + "LSI" => IndexType::Lsi, + _ => { + return Err(StorageError::Internal(format!( + "unknown index type: {index_type_str}" + ))); + } + }; + + let key_schema_bson = index_doc + .get("key_schema") + .ok_or_else(|| StorageError::Internal("missing key_schema in index".to_string()))?; + let key_schema: Vec = + bson::from_bson(key_schema_bson.clone()) + .map_err(|e| StorageError::Internal(format!("index key_schema parse: {e}")))?; + + let projection_bson = index_doc + .get("projection") + .ok_or_else(|| StorageError::Internal("missing projection in index".to_string()))?; + let projection: extenddb_core::types::Projection = bson::from_bson(projection_bson.clone()) + .map_err(|e| StorageError::Internal(format!("index projection parse: {e}")))?; + + Ok(IndexInfo { + index_name: index_name.to_string(), + index_id, + index_type, + key_schema, + projection, + }) + } + + /// Convert a catalog table document to a `TableDescription`. + async fn doc_to_table_description( + &self, + doc: &Document, + ) -> Result { + let id_doc = doc + .get_document("_id") + .map_err(|_| StorageError::Internal("missing _id".to_string()))?; + let table_name = id_doc + .get_str("table_name") + .map_err(|_| StorageError::Internal("missing table_name".to_string()))? + .to_string(); + let account_id = id_doc + .get_str("account_id") + .map_err(|_| StorageError::Internal("missing account_id".to_string()))?; + + let table_id = doc + .get_str("table_id") + .map_err(|_| StorageError::Internal("missing table_id".to_string()))? + .to_string(); + let table_arn_val = doc + .get_str("table_arn") + .map_err(|_| StorageError::Internal("missing table_arn".to_string()))? + .to_string(); + + let status_str = doc.get_str("table_status").unwrap_or("ACTIVE"); + let table_status = match status_str { + "CREATING" => TableStatus::Creating, + "ACTIVE" => TableStatus::Active, + "DELETING" => TableStatus::Deleting, + "UPDATING" => TableStatus::Updating, + _ => TableStatus::Active, + }; + + let creation_dt = doc + .get_datetime("creation_date_time") + .map(|dt| dt.timestamp_millis() as f64 / 1000.0) + .unwrap_or(0.0); + + let table_size_bytes = doc.get_i64("table_size_bytes").unwrap_or(0); + let item_count = doc.get_i64("item_count").unwrap_or(0); + let deletion_protection = doc.get_bool("deletion_protection_enabled").unwrap_or(false); + + let key_schema_bson = doc + .get("key_schema") + .ok_or_else(|| StorageError::Internal("missing key_schema".to_string()))?; + let key_schema: Vec = + bson::from_bson(key_schema_bson.clone()) + .map_err(|e| StorageError::Internal(format!("key_schema: {e}")))?; + + let attr_defs_bson = doc + .get("attribute_definitions") + .ok_or_else(|| StorageError::Internal("missing attribute_definitions".to_string()))?; + let attribute_definitions: Vec = + bson::from_bson(attr_defs_bson.clone()) + .map_err(|e| StorageError::Internal(format!("attr_defs: {e}")))?; + + let stream_specification = doc.get("stream_specification").and_then(|b| { + if b.as_null().is_some() { + None + } else { + bson::from_bson(b.clone()).ok() + } + }); + + let billing_str = doc.get_str("billing_mode").unwrap_or("PROVISIONED"); + let billing_mode = match billing_str { + "PAY_PER_REQUEST" => BillingMode::PayPerRequest, + _ => BillingMode::Provisioned, + }; + + let pt_desc = doc + .get("provisioned_throughput") + .and_then(|b| { + if b.as_null().is_some() { + None + } else { + bson::from_bson::(b.clone()).ok() + } + }) + .map_or( + ProvisionedThroughputDescription { + read_capacity_units: 0, + write_capacity_units: 0, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }, + |pt| ProvisionedThroughputDescription { + read_capacity_units: pt.read_capacity_units, + write_capacity_units: pt.write_capacity_units, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }, + ); + + let billing_summary = if billing_mode == BillingMode::PayPerRequest { + Some(BillingModeSummary { + billing_mode: BillingMode::PayPerRequest, + last_update_to_pay_per_request_date_time: Some(creation_dt), + }) + } else { + None + }; + + // Fetch indexes + let indexes_coll = self.catalog_db.collection::("indexes"); + use futures::TryStreamExt; + let index_cursor = indexes_coll + .find(doc! { "_id.table_id": &table_id }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + let index_docs: Vec = index_cursor + .try_collect() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + let mut gsis = Vec::new(); + let mut lsis = Vec::new(); + + for idx_doc in &index_docs { + let idx_id_doc = idx_doc + .get_document("_id") + .map_err(|_| StorageError::Internal("missing index _id".to_string()))?; + let idx_name = idx_id_doc + .get_str("index_name") + .map_err(|_| StorageError::Internal("missing index_name".to_string()))? + .to_string(); + let idx_type = idx_doc.get_str("index_type").unwrap_or("GSI"); + + let idx_ks_bson = idx_doc + .get("key_schema") + .ok_or_else(|| StorageError::Internal("missing index key_schema".to_string()))?; + let idx_key_schema: Vec = + bson::from_bson(idx_ks_bson.clone()) + .map_err(|e| StorageError::Internal(format!("index key_schema: {e}")))?; + + let idx_proj_bson = idx_doc + .get("projection") + .ok_or_else(|| StorageError::Internal("missing index projection".to_string()))?; + let idx_projection: extenddb_core::types::Projection = + bson::from_bson(idx_proj_bson.clone()) + .map_err(|e| StorageError::Internal(format!("index projection: {e}")))?; + + let idx_arn = index_arn(&self.region, account_id, &table_name, &idx_name); + + match idx_type { + "GSI" => { + let idx_pt = idx_doc + .get("provisioned_throughput") + .and_then(|b| { + if b.as_null().is_some() { + None + } else { + bson::from_bson::( + b.clone(), + ) + .ok() + } + }) + .map(|pt| ProvisionedThroughputDescription { + read_capacity_units: pt.read_capacity_units, + write_capacity_units: pt.write_capacity_units, + number_of_decreases_today: 0, + last_increase_date_time: None, + last_decrease_date_time: None, + }); + + gsis.push(GsiDescription { + index_name: idx_name, + key_schema: idx_key_schema, + projection: idx_projection, + index_status: idx_doc + .get_str("index_status") + .unwrap_or("ACTIVE") + .to_string(), + provisioned_throughput: idx_pt, + index_size_bytes: 0, + item_count: 0, + index_arn: idx_arn, + }); + } + "LSI" => { + lsis.push(LsiDescription { + index_name: idx_name, + key_schema: idx_key_schema, + projection: idx_projection, + index_size_bytes: 0, + item_count: 0, + index_arn: idx_arn, + }); + } + _ => {} + } + } + + // Stream info + let stream_label = doc + .get_str("stream_label") + .ok() + .map(std::string::ToString::to_string); + let stream_arn_opt = stream_label + .as_ref() + .map(|label| stream_arn(&self.region, account_id, &table_name, label)); + + // TableClass / SSEDescription / OnDemandThroughput — read back the + // fields persisted at CreateTable time. Same shape as postgres' + // table_helpers.rs:329-353. + let table_class_summary = doc + .get_str("table_class") + .ok() + .map(|tc| serde_json::json!({ "TableClass": tc })); + let sse_description = doc.get("sse_specification").and_then(|b| { + let spec: serde_json::Value = bson::from_bson(b.clone()).ok()?; + let enabled = spec + .get("Enabled") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + if enabled { + Some(SseDescription { + status: "ENABLED".to_owned(), + sse_type: Some(SseType::KMS), + kms_master_key_arn: Some(format!( + "arn:aws:kms:{}:{}:key/default", + self.region, account_id + )), + }) + } else { + None + } + }); + let on_demand_throughput: Option = doc + .get("on_demand_throughput") + .and_then(|b| bson::from_bson(b.clone()).ok()); + + Ok(TableDescription { + table_name, + key_schema, + attribute_definitions, + table_status, + creation_date_time: creation_dt, + table_size_bytes, + item_count, + table_arn: table_arn_val, + table_id, + provisioned_throughput: pt_desc, + billing_mode_summary: billing_summary, + global_secondary_indexes: if gsis.is_empty() { None } else { Some(gsis) }, + local_secondary_indexes: if lsis.is_empty() { None } else { Some(lsis) }, + stream_specification, + latest_stream_arn: stream_arn_opt, + latest_stream_label: stream_label, + deletion_protection_enabled: deletion_protection, + sse_description, + table_class_summary, + on_demand_throughput, + }) + } + + /// Create the mongo collection for a GSI/LSI and add the indexes + /// its query path relies on. Every read against the index goes + /// through `find` predicates on `(pk, sk_*, base_pk, base_sk_*)` + /// with sorts on the same fields — without indexes those queries + /// devolve to a collection scan per read. Called from + /// `create_table_impl` (initial GSI/LSI) and the UpdateTable GSI- + /// create path (D-m7). + pub(crate) async fn create_index_data_collection( + &self, + index_id: &str, + index_key_schema: &[KeySchemaElement], + base_key_schema: &[KeySchemaElement], + attribute_definitions: &[AttributeDefinition], + ) -> Result<(), StorageError> { + let coll_name = data_collection_name(index_id); + // create_collection is idempotent on recent MongoDB; a duplicate + // means we retried through a crash after the first success. Log + // + continue rather than surfacing an error to the caller. + if let Err(e) = self.data_db.create_collection(&coll_name).await { + tracing::debug!("index collection {coll_name} pre-exists or race: {e}"); + } + let coll = self.data_db.collection::(&coll_name); + + // Sort/paginate key: (pk, sk?, base_pk, base_sk?). Same tuple + // that scan_impl / query_impl sort by post-D-C1. Not unique — + // GSI keys are non-unique across base items; index docs are + // disambiguated by base-key components in the _id. + let idx_sk_field = sk_info(index_key_schema, attribute_definitions).map(|(_, t)| match t { + ScalarAttributeType::S => ("sk_s", true), + ScalarAttributeType::N => ("sk_n", false), + ScalarAttributeType::B => ("sk_b", false), + }); + let base_sk_field = sk_info(base_key_schema, attribute_definitions).map(|(_, t)| match t { + ScalarAttributeType::S => "base_sk_s", + ScalarAttributeType::N => "base_sk_n", + ScalarAttributeType::B => "base_sk_b", + }); + + let mut keys = doc! { "pk": 1 }; + if let Some((sk_f, _)) = idx_sk_field { + keys.insert(sk_f, 1); + } + keys.insert("base_pk", 1); + if let Some(base_sk_f) = base_sk_field { + keys.insert(base_sk_f, 1); + } + + // String sort keys need the `simple` collation so range + // comparisons behave as byte-wise, matching the query path. + let uses_string_sort = + matches!(idx_sk_field, Some((_, true))) || matches!(base_sk_field, Some("base_sk_s")); + let mut opts = IndexOptions::builder().build(); + if uses_string_sort { + opts.collation = Some(Collation::builder().locale("simple".to_string()).build()); + } + + coll.create_index(IndexModel::builder().keys(keys).options(opts).build()) + .await + .map_err(|e| StorageError::Internal(format!("index-coll index: {e}")))?; + + Ok(()) + } +} + +/// Build an `IndexInfo` from an `indexes` catalog document whose `_id` is +/// `{ table_id, index_name }`. Used to populate the GSI/LSI lists carried on +/// `TableKeyInfo` for per-index consumed-capacity computation. +fn index_info_from_doc(index_doc: &Document) -> Result { + let index_name = index_doc + .get_document("_id") + .ok() + .and_then(|id| id.get_str("index_name").ok()) + .ok_or_else(|| StorageError::Internal("missing _id.index_name".to_string()))? + .to_string(); + let index_id = index_doc + .get_str("index_id") + .map_err(|_| StorageError::Internal("missing index_id".to_string()))? + .to_string(); + let index_type = match index_doc.get_str("index_type") { + Ok("GSI") => IndexType::Gsi, + Ok("LSI") => IndexType::Lsi, + other => { + return Err(StorageError::Internal(format!( + "unknown index type: {other:?}" + ))); + } + }; + let key_schema_bson = index_doc + .get("key_schema") + .ok_or_else(|| StorageError::Internal("missing key_schema in index".to_string()))?; + let key_schema: Vec = + bson::from_bson(key_schema_bson.clone()) + .map_err(|e| StorageError::Internal(format!("index key_schema parse: {e}")))?; + let projection_bson = index_doc + .get("projection") + .ok_or_else(|| StorageError::Internal("missing projection in index".to_string()))?; + let projection: extenddb_core::types::Projection = bson::from_bson(projection_bson.clone()) + .map_err(|e| StorageError::Internal(format!("index projection parse: {e}")))?; + Ok(IndexInfo { + index_name, + index_id, + index_type, + key_schema, + projection, + }) +} diff --git a/crates/storage-mongodb/src/ttl_worker.rs b/crates/storage-mongodb/src/ttl_worker.rs new file mode 100644 index 00000000..5a45ff52 --- /dev/null +++ b/crates/storage-mongodb/src/ttl_worker.rs @@ -0,0 +1,384 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! TTL cleanup background worker for `MongoDB`. + +use std::sync::Arc; +use std::time::Duration; + +use bson::{Document, doc}; +use extenddb_core::metrics::MetricsCollector; +use extenddb_core::types::{KeySchemaElement, Projection, ProjectionType, UserIdentity}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{DataEngine, MetadataEngine, StreamEngine, TableEngine, WorkerStore}; +use futures::TryStreamExt; + +use crate::MongoEngine; + +const SCAN_INTERVAL: Duration = Duration::from_secs(60); +const BATCH_SIZE: usize = 100; +const STREAM_RETENTION_HOURS: i64 = 24; +const STREAM_CLEANUP_INTERVAL: Duration = Duration::from_secs(3600); +const GSI_BACKFILL_INTERVAL: Duration = Duration::from_secs(5); +const GSI_BACKFILL_BATCH: i64 = 500; +/// How often to flip due `CREATING` tables to `ACTIVE`. Short enough that the +/// window closes promptly after `control_plane_delay_seconds` (default 0.25s) +/// without busy-spinning. +const CONTROL_PLANE_POLL_INTERVAL: Duration = Duration::from_millis(250); + +pub(crate) async fn ttl_cleanup_worker(storage: Arc, metrics: Arc) { + let region_arc: Arc = Arc::from(storage.region.as_str()); + + loop { + tokio::time::sleep(SCAN_INTERVAL).await; + retry_pending_indexes(&storage).await; + sweep_expired_items(&storage, &metrics, ®ion_arc).await; + } +} + +/// Defense-in-depth stream-record deletion. A MongoDB TTL index on +/// `stream_records.created_at` already deletes records ~1 minute after +/// created_at + 24h; this loop covers the case where that index is +/// missing (init predates the schema change) or lagging. +pub(crate) async fn stream_record_cleanup_worker(storage: Arc) { + loop { + tokio::time::sleep(STREAM_CLEANUP_INTERVAL).await; + match StreamEngine::cleanup_expired_stream_records(&*storage, STREAM_RETENTION_HOURS).await + { + Ok(0) => {} + Ok(n) => tracing::info!("Stream cleanup worker: deleted {n} expired record(s)"), + Err(e) => tracing::warn!("Stream cleanup worker: delete failed: {e}"), + } + } +} + +/// Background worker that turns CREATING GSIs into ACTIVE ones. +/// +/// UpdateTable's GSI-create path leaves the index in `index_status: +/// "CREATING"` after inserting the catalog document. This worker +/// discovers each such row, iterates the base collection with a +/// persistent cursor, upserts projected items into the index +/// collection, and — once the base is fully scanned — flips the +/// index to `ACTIVE`. Restart-safe: the cursor is persisted after +/// every batch so a mid-backfill crash resumes where it left off. +/// +/// Live writes during the backfill window continue to route through +/// `sync_indexes` / `sync_indexes_in_session`, which write to +/// CREATING indexes too (indexes catalog membership, not status, is +/// what gates the write path). All writes are upserts on the same +/// `_id` shape, so a base item touched by both the backfill and a +/// concurrent write converges regardless of interleaving — +/// RFC-0003 §2.4. +pub(crate) async fn gsi_backfill_worker(storage: Arc) { + loop { + tokio::time::sleep(GSI_BACKFILL_INTERVAL).await; + + let indexes_coll = storage.catalog_db.collection::("indexes"); + let cursor = match indexes_coll + .find(doc! { "index_status": "CREATING", "index_type": "GSI" }) + .await + { + Ok(c) => c, + Err(e) => { + tracing::warn!("GSI backfill worker: list failed: {e}"); + continue; + } + }; + let jobs: Vec = match cursor.try_collect().await { + Ok(j) => j, + Err(e) => { + tracing::warn!("GSI backfill worker: collect failed: {e}"); + continue; + } + }; + + for job in jobs { + if let Err(e) = run_gsi_backfill_job(&storage, &job).await { + tracing::warn!( + "GSI backfill worker: job failed for index_id={}: {e}", + job.get_str("index_id").unwrap_or("?"), + ); + } + } + } +} + +async fn run_gsi_backfill_job(storage: &MongoEngine, job: &Document) -> Result<(), StorageError> { + let index_id = job + .get_str("index_id") + .map_err(|_| StorageError::Internal("missing index_id".to_owned()))? + .to_owned(); + let id_doc = job + .get_document("_id") + .map_err(|_| StorageError::Internal("missing _id".to_owned()))?; + let table_id = id_doc + .get_str("table_id") + .map_err(|_| StorageError::Internal("missing _id.table_id".to_owned()))? + .to_owned(); + + let key_info = storage.table_key_info_by_table_id_impl(&table_id).await?; + + let idx_key_schema_bson = job + .get("key_schema") + .ok_or_else(|| StorageError::Internal("missing key_schema".to_owned()))?; + let idx_key_schema: Vec = bson::from_bson(idx_key_schema_bson.clone()) + .map_err(|e| StorageError::Internal(format!("key_schema parse: {e}")))?; + + let projection: Projection = job + .get("projection") + .and_then(|p| bson::from_bson(p.clone()).ok()) + .unwrap_or(Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }); + + let mut cursor = job.get("backfill_cursor").cloned(); + let indexes_coll = storage.catalog_db.collection::("indexes"); + + loop { + let progress = storage + .backfill_gsi_batch( + &key_info, + &index_id, + &idx_key_schema, + &projection, + cursor.as_ref(), + GSI_BACKFILL_BATCH, + ) + .await?; + + if progress.done { + // Full-scan complete. Flip to ACTIVE and drop the cursor. + indexes_coll + .update_one( + doc! { "index_id": &index_id, "index_status": "CREATING" }, + doc! { + "$set": { "index_status": "ACTIVE" }, + "$unset": { "backfill_cursor": "" }, + }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + tracing::info!( + "GSI backfill worker: index_id={index_id} ACTIVE (last batch scanned {} docs)", + progress.scanned, + ); + return Ok(()); + } + + if let Some(ref last_id) = progress.last_id { + indexes_coll + .update_one( + doc! { "index_id": &index_id }, + doc! { "$set": { "backfill_cursor": last_id.clone() } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + cursor = Some(last_id.clone()); + } else { + // Empty batch but not done — treat as done to avoid an + // infinite loop. Shouldn't happen in practice since + // backfill_gsi_batch marks done when scanned < batch_size. + return Ok(()); + } + } +} + +async fn retry_pending_indexes(storage: &MongoEngine) { + let Ok(pending) = MetadataEngine::all_tables_with_ttl(storage).await else { + return; + }; + let Ok(ready) = MetadataEngine::all_tables_with_ttl_index_ready(storage).await else { + return; + }; + let ready_set: std::collections::HashSet<(&str, &str)> = ready + .iter() + .map(|(a, t, _)| (a.as_str(), t.as_str())) + .collect(); + for (account_id, table_name, ttl_attr) in &pending { + if !ready_set.contains(&(account_id.as_str(), table_name.as_str())) { + if let Err(e) = + MetadataEngine::create_ttl_index(storage, account_id, table_name, ttl_attr).await + { + tracing::debug!("TTL worker: index creation retry failed for {table_name}: {e}"); + } else { + tracing::info!("TTL worker: index created for {table_name}"); + } + } + } +} + +async fn sweep_expired_items(storage: &MongoEngine, metrics: &MetricsCollector, region: &Arc) { + let ttl_identity = UserIdentity { + identity_type: "Service".to_owned(), + principal_id: "dynamodb.amazonaws.com".to_owned(), + }; + + let tables = match MetadataEngine::all_tables_with_ttl_index_ready(storage).await { + Ok(t) => t, + Err(e) => { + tracing::warn!("TTL worker: failed to list tables: {e}"); + return; + } + }; + + let now_epoch = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + + for (account_id, table_name, ttl_attribute) in &tables { + let items = match MetadataEngine::find_expired_items_indexed( + storage, + account_id, + table_name, + ttl_attribute, + BATCH_SIZE, + ) + .await + { + Ok(items) => items, + Err(e) => { + tracing::warn!("TTL worker: find expired failed for {table_name}: {e}"); + continue; + } + }; + + if items.is_empty() { + continue; + } + + let key_info = match TableEngine::table_key_info(storage, account_id, table_name).await { + Ok(ki) => ki, + Err(e) => { + tracing::warn!("TTL worker: key info failed for {table_name}: {e}"); + continue; + } + }; + + let view_type = stream_view_type(&key_info); + let (condition_expr, maps) = build_ttl_condition(ttl_attribute, now_epoch); + + let mut deleted = 0usize; + for item in &items { + let staleness = item + .get(ttl_attribute.as_str()) + .and_then(|av| { + if let extenddb_core::types::AttributeValue::N(n) = av { + n.parse::().ok() + } else { + None + } + }) + .map(|ttl_val| now_epoch.saturating_sub(ttl_val)); + + let key: extenddb_core::types::Item = key_info + .key_schema + .iter() + .filter_map(|ks| { + item.get(&ks.attribute_name) + .map(|v| (ks.attribute_name.clone(), v.clone())) + }) + .collect(); + + let return_old = view_type.is_some(); + let stream = view_type.map(|vt| extenddb_storage::StreamCapture { + view_type: vt, + user_identity: Some(ttl_identity.clone()), + region: region.clone(), + }); + match DataEngine::delete_item( + storage, + &key_info, + &key, + return_old, + Some(&condition_expr), + &maps, + stream.as_ref(), + ) + .await + { + Err(StorageError::ConditionFailed(_)) => {} + Err(e) => { + tracing::warn!("TTL worker: delete failed for {table_name}: {e}"); + } + Ok(_old_item) => { + deleted += 1; + metrics.record_ttl_deletion(table_name); + if let Some(s) = staleness { + #[allow(clippy::cast_precision_loss)] + metrics.record_ttl_staleness(table_name, s as f64); + } + } + } + } + + if deleted > 0 { + tracing::info!("TTL worker: deleted {deleted} expired items from {table_name}"); + } + } +} + +fn stream_view_type( + key_info: &extenddb_core::types::TableKeyInfo, +) -> Option { + key_info.stream_specification.as_ref().and_then(|spec| { + if spec.stream_enabled { + spec.stream_view_type + } else { + None + } + }) +} + +fn build_ttl_condition( + ttl_attribute: &str, + now_epoch: u64, +) -> ( + extenddb_core::expression::Expr, + extenddb_core::expression::ExpressionMaps, +) { + use extenddb_core::expression::{CompareOp, Expr, ExpressionMaps, PathElement}; + use std::collections::HashMap; + + let ttl_path = vec![PathElement::Attribute("#ttl".to_owned())]; + let condition_expr = Expr::And( + Box::new(Expr::Function { + name: "attribute_exists".to_owned(), + args: vec![Expr::Path(ttl_path.clone())], + }), + Box::new(Expr::Compare { + left: Box::new(Expr::Path(ttl_path)), + op: CompareOp::Le, + right: Box::new(Expr::Placeholder("now".to_owned())), + }), + ); + + let mut names = HashMap::new(); + names.insert("ttl".to_owned(), ttl_attribute.to_owned()); + let mut values = HashMap::new(); + values.insert( + "now".to_owned(), + extenddb_core::types::AttributeValue::N(now_epoch.to_string()), + ); + + (condition_expr, ExpressionMaps::new(names, values)) +} + +/// Background poller that flips tables out of the transient `CREATING` state +/// once their scheduled `status_transition_at` has passed. See +/// [`crate::worker_store`] for how rows enter `CREATING`. +pub(crate) async fn control_plane_worker(storage: Arc) { + loop { + tokio::time::sleep(CONTROL_PLANE_POLL_INTERVAL).await; + match WorkerStore::process_control_plane_transitions(&*storage).await { + Ok(t) if t.is_empty() => {} + Ok(transitions) => { + for (name, transition) in &transitions { + tracing::info!("Table '{name}': {transition}"); + } + } + Err(e) => tracing::warn!("Control-plane transition poll failed: {e}"), + } + } +} diff --git a/crates/storage-mongodb/src/worker_store.rs b/crates/storage-mongodb/src/worker_store.rs new file mode 100644 index 00000000..6308f4fd --- /dev/null +++ b/crates/storage-mongodb/src/worker_store.rs @@ -0,0 +1,107 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! `WorkerStore` implementation for `MongoDB`. +//! +//! Processes table control-plane transitions (`CREATING` → `ACTIVE`) as a +//! background job. `create_table_impl` and the restore path write the catalog +//! row as `CREATING` with a `status_transition_at` timestamp when +//! `control_plane_delay_seconds` > 0 (matching the Postgres backend and real +//! DynamoDB, which report `CREATING` before a table becomes usable); this +//! worker flips such rows to `ACTIVE` once their transition time has passed. +//! When `control_plane_delay_seconds` is 0 the create/restore paths write +//! `ACTIVE` directly and this worker has nothing to do. +//! +//! `DeleteTable` remains inline (the catalog row and collections are removed in +//! the request handler), so there is no `DELETING` transient state to reconcile +//! here. GSI create is handled separately by +//! [`ttl_worker::gsi_backfill_worker`] on the `indexes` catalog collection. +//! +//! [`ttl_worker::gsi_backfill_worker`]: crate::ttl_worker::gsi_backfill_worker + +use futures::TryStreamExt; +use futures::future::BoxFuture; +use mongodb::bson::{Document, doc}; + +use extenddb_storage::WorkerStore; +use extenddb_storage::error::StorageError; + +use crate::MongoEngine; + +/// Default control-plane delay (seconds) when the setting is absent or +/// unparseable. Matches the Postgres backend default. +const DEFAULT_CONTROL_PLANE_DELAY_SECS: f64 = 0.25; + +impl MongoEngine { + /// Read `control_plane_delay_seconds` from the settings collection, + /// falling back to the default. A value <= 0 means "no CREATING window" + /// (create/restore write `ACTIVE` synchronously). + pub(crate) async fn control_plane_delay_seconds(&self) -> f64 { + let coll = self.catalog_db.collection::("settings"); + coll.find_one(doc! { "_id": "control_plane_delay_seconds" }) + .await + .ok() + .flatten() + .and_then(|d| d.get_str("value").ok().map(str::to_owned)) + .and_then(|v| v.parse::().ok()) + .filter(|v| *v >= 0.0) + .unwrap_or(DEFAULT_CONTROL_PLANE_DELAY_SECS) + } +} + +impl WorkerStore for MongoEngine { + fn process_control_plane_transitions( + &self, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let mut transitions = Vec::new(); + let tables_coll = self.catalog_db.collection::("tables"); + let now = mongodb::bson::DateTime::now(); + + // CREATING → ACTIVE: tables whose scheduled transition time has + // passed. Each row is updated by its own compound `_id` (the mongo + // catalog stores account_id/table_name inside `_id`, not at the top + // level — the previous impl filtered on flat fields and matched + // nothing). + let filter = doc! { + "table_status": "CREATING", + "status_transition_at": { "$lte": now }, + }; + let mut cursor = tables_coll + .find(filter) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + while let Some(table_doc) = cursor + .try_next() + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + { + let Some(id) = table_doc.get("_id").cloned() else { + continue; + }; + let table_name = table_doc + .get_document("_id") + .ok() + .and_then(|d| d.get_str("table_name").ok()) + .unwrap_or_default() + .to_owned(); + + tables_coll + .update_one( + doc! { "_id": id, "table_status": "CREATING" }, + doc! { + "$set": { "table_status": "ACTIVE" }, + "$unset": { "status_transition_at": "" }, + }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + + transitions.push((table_name, "CREATING → active")); + } + + Ok(transitions) + }) + } +} diff --git a/crates/storage-mongodb/tests/common/mod.rs b/crates/storage-mongodb/tests/common/mod.rs new file mode 100644 index 00000000..581abc76 --- /dev/null +++ b/crates/storage-mongodb/tests/common/mod.rs @@ -0,0 +1,517 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Pure-Rust MongoDB filter interpreter for pushdown-parity tests. +//! +//! Evaluates the subset of MongoDB filter operators emitted by +//! `crates/storage-mongodb/src/condition.rs::condition_to_filter` against a +//! BSON document. Used to differentially test the compiled-filter path +//! against the in-Rust DDB expression evaluator without needing a live +//! MongoDB. +//! +//! This is deliberately not a general-purpose MongoDB query engine — it +//! implements only the operators the compiler can emit. Adding a new +//! operator to the compiler requires adding it here. + +use bson::{Bson, Document}; + +/// Evaluate a MongoDB filter document against a BSON document. +/// +/// Returns `true` if the document matches all clauses in the filter, +/// `false` otherwise. The root document representing the filter is +/// interpreted as an implicit `$and` of its clauses (matching MongoDB's +/// top-level query semantics). +pub fn eval_filter(filter: &Document, doc: &Document) -> bool { + filter.iter().all(|(key, val)| eval_clause(key, val, doc)) +} + +/// Evaluate a single (key, value) clause at the top level of a filter. +/// +/// Handles: +/// - `$and` / `$or` / `$nor` — logical operators over an array of subfilters +/// - `$expr` — expression-based comparison, only for field-vs-field per compiler +/// - `: ` — either a scalar equality or an operator document +fn eval_clause(key: &str, val: &Bson, doc: &Document) -> bool { + match key { + "$and" => as_array(val) + .iter() + .all(|sub| as_doc(sub).is_some_and(|d| eval_filter(d, doc))), + "$or" => as_array(val) + .iter() + .any(|sub| as_doc(sub).is_some_and(|d| eval_filter(d, doc))), + "$nor" => !as_array(val) + .iter() + .any(|sub| as_doc(sub).is_some_and(|d| eval_filter(d, doc))), + "$expr" => eval_expr(val, doc), + _ => { + // : — walk the dotted path, then evaluate the + // predicate against the value found. + let field_val = walk_path(doc, key); + eval_predicate(val, &field_val) + } + } +} + +/// Evaluate a predicate against the value found at the field path. +/// +/// The predicate is either a scalar (implicit `$eq`) or an operator +/// document containing `$eq`, `$ne`, `$lt`, `$lte`, `$gt`, `$gte`, +/// `$exists`, `$in`, `$regex`, or `$type`. +/// +/// If the field is a BSON array and the predicate is a scalar, MongoDB +/// matches if any element of the array equals the scalar (implicit +/// array-match). We support that shape because the compiler relies on it +/// for `contains(SS_field, :s)` and similar. +fn eval_predicate(pred: &Bson, field: &FieldValue<'_>) -> bool { + match pred { + Bson::Document(pred_doc) => { + // Disambiguate: MongoDB treats a document as an operator + // document when at least one of its keys starts with `$`; + // otherwise it's a literal document for equality match. This + // is the same rule the driver uses. The compiler relies on + // literal-match for `contains(L_field, :s)` which emits a + // predicate like `{S: "value"}` — no `$` keys. + let is_operator_doc = pred_doc.keys().any(|k| k.starts_with('$')); + if !is_operator_doc { + return eq_or_array_match(field, pred); + } + // Operator document: every operator must match. MongoDB's + // behavior with multiple operator keys in one doc is that they + // are ANDed together. + pred_doc.iter().all(|(op, arg)| match op.as_str() { + "$eq" => eq_or_array_match(field, arg), + "$ne" => !eq_or_array_match(field, arg), + "$lt" => cmp_field(field, arg, |a, b| { + bson_cmp(a, b) == std::cmp::Ordering::Less + }), + "$lte" => cmp_field(field, arg, |a, b| { + matches!( + bson_cmp(a, b), + std::cmp::Ordering::Less | std::cmp::Ordering::Equal + ) + }), + "$gt" => cmp_field(field, arg, |a, b| { + bson_cmp(a, b) == std::cmp::Ordering::Greater + }), + "$gte" => cmp_field(field, arg, |a, b| { + matches!( + bson_cmp(a, b), + std::cmp::Ordering::Greater | std::cmp::Ordering::Equal + ) + }), + "$exists" => { + let want = as_bool(arg).unwrap_or(true); + field.is_present() == want + } + "$in" => { + let arr = as_array(arg); + match field { + FieldValue::Present(Bson::Array(field_arr)) => field_arr.iter().any(|f| { + arr.iter() + .any(|a| bson_cmp(f, a) == std::cmp::Ordering::Equal) + }), + FieldValue::Present(v) => arr + .iter() + .any(|a| bson_cmp(v, a) == std::cmp::Ordering::Equal), + FieldValue::Missing => false, + } + } + "$regex" => match arg { + Bson::String(pattern) => match field { + FieldValue::Present(Bson::String(s)) => regex_match(pattern, s), + FieldValue::Present(Bson::Array(arr)) => arr + .iter() + .any(|v| matches!(v, Bson::String(s) if regex_match(pattern, s))), + _ => false, + }, + _ => false, + }, + "$type" => match arg { + Bson::String(t) => matches!( + (t.as_str(), field), + ("null", FieldValue::Present(Bson::Null)) + | ("string", FieldValue::Present(Bson::String(_))) + | ("bool", FieldValue::Present(Bson::Boolean(_))) + | ("array", FieldValue::Present(Bson::Array(_))) + | ("object", FieldValue::Present(Bson::Document(_))) + ), + _ => false, + }, + // If the compiler ever emits an operator we don't recognize, + // fail loudly rather than silently pass. + other => panic!("bson filter interpreter: unknown operator {other}"), + }) + } + // Scalar predicate: implicit equality (or array-match). + _ => eq_or_array_match(field, pred), + } +} + +/// Field-vs-field expressions via `$expr`. The compiler only emits +/// `{"$expr": {"$op": ["$left_field", "$right_field"]}}` shapes. +fn eval_expr(val: &Bson, doc: &Document) -> bool { + let Some(expr_doc) = as_doc(val) else { + return false; + }; + for (op, args) in expr_doc { + let arr = as_array(args); + if arr.len() != 2 { + return false; + } + let (Some(lhs_ref), Some(rhs_ref)) = (as_field_ref(&arr[0]), as_field_ref(&arr[1])) else { + return false; + }; + let lhs = walk_path(doc, lhs_ref); + let rhs = walk_path(doc, rhs_ref); + let (FieldValue::Present(l), FieldValue::Present(r)) = (&lhs, &rhs) else { + return false; + }; + let ord = bson_cmp(l, r); + let matched = match op.as_str() { + "$eq" => ord == std::cmp::Ordering::Equal, + "$ne" => ord != std::cmp::Ordering::Equal, + "$lt" => ord == std::cmp::Ordering::Less, + "$lte" => matches!(ord, std::cmp::Ordering::Less | std::cmp::Ordering::Equal), + "$gt" => ord == std::cmp::Ordering::Greater, + "$gte" => matches!(ord, std::cmp::Ordering::Greater | std::cmp::Ordering::Equal), + other => panic!("bson filter interpreter: unknown $expr operator {other}"), + }; + if !matched { + return false; + } + } + true +} + +/// Whether a field's value at a dotted path is Present or Missing. +/// +/// Distinguishing these is required for `$exists` semantics: MongoDB's +/// `{$exists: false}` only matches documents where the field genuinely +/// isn't in the document, not documents where the field is present with +/// value `null`. +#[derive(Debug)] +pub enum FieldValue<'a> { + Present(&'a Bson), + Missing, +} + +impl<'a> FieldValue<'a> { + fn is_present(&self) -> bool { + matches!(self, FieldValue::Present(_)) + } +} + +/// Walk a dotted path like "item_data.address.city" through nested docs +/// (and arrays, when a path component is a numeric index). +pub fn walk_path<'a>(doc: &'a Document, path: &str) -> FieldValue<'a> { + let parts: Vec<&str> = path.split('.').collect(); + if parts.is_empty() { + return FieldValue::Missing; + } + let mut cur: &Bson = match doc.get(parts[0]) { + Some(v) => v, + None => return FieldValue::Missing, + }; + for part in &parts[1..] { + cur = match cur { + Bson::Document(d) => match d.get(*part) { + Some(v) => v, + None => return FieldValue::Missing, + }, + Bson::Array(a) => match part.parse::() { + Ok(idx) => match a.get(idx) { + Some(v) => v, + None => return FieldValue::Missing, + }, + Err(_) => return FieldValue::Missing, + }, + _ => return FieldValue::Missing, + }; + } + FieldValue::Present(cur) +} + +fn eq_or_array_match(field: &FieldValue<'_>, target: &Bson) -> bool { + match field { + FieldValue::Present(Bson::Array(arr)) => arr + .iter() + .any(|v| bson_cmp(v, target) == std::cmp::Ordering::Equal), + FieldValue::Present(v) => bson_cmp(v, target) == std::cmp::Ordering::Equal, + FieldValue::Missing => matches!(target, Bson::Null), + } +} + +fn cmp_field(field: &FieldValue<'_>, target: &Bson, pred: F) -> bool +where + F: Fn(&Bson, &Bson) -> bool, +{ + match field { + FieldValue::Present(v) => pred(v, target), + FieldValue::Missing => false, + } +} + +/// Compare two BSON values with MongoDB's total-ordering rules. +/// +/// This is a small subset — enough for the compiler's operator surface. +/// Numeric types are unified (Int32/Int64/Double/Decimal128 compare by +/// numeric value). Types that don't compare (Document vs. String) return +/// Equal by fallback because the compiler never emits comparisons between +/// mixed types in practice; if a proptest run generates one, the parity +/// check will surface the mismatch. +fn bson_cmp(a: &Bson, b: &Bson) -> std::cmp::Ordering { + use std::cmp::Ordering; + match (a, b) { + (Bson::String(x), Bson::String(y)) => x.cmp(y), + (Bson::Boolean(x), Bson::Boolean(y)) => x.cmp(y), + (Bson::Int32(x), Bson::Int32(y)) => x.cmp(y), + (Bson::Int64(x), Bson::Int64(y)) => x.cmp(y), + (Bson::Int32(x), Bson::Int64(y)) => (*x as i64).cmp(y), + (Bson::Int64(x), Bson::Int32(y)) => x.cmp(&(*y as i64)), + (Bson::Double(x), Bson::Double(y)) => x.partial_cmp(y).unwrap_or(Ordering::Equal), + (Bson::Binary(x), Bson::Binary(y)) => x.bytes.cmp(&y.bytes), + (Bson::Null, Bson::Null) => Ordering::Equal, + (Bson::Array(x), Bson::Array(y)) => { + // Elementwise; documents-of-arrays don't come up in the compiler + // output, but arrays-of-primitives can when a set field is + // compared to another set. + for (xi, yi) in x.iter().zip(y.iter()) { + match bson_cmp(xi, yi) { + Ordering::Equal => continue, + other => return other, + } + } + x.len().cmp(&y.len()) + } + (Bson::Document(x), Bson::Document(y)) => { + // Compare field-by-field in insertion order — matches how BSON + // documents are serialized and how our compiler's `$type` + // comparisons treat them. + for ((xk, xv), (yk, yv)) in x.iter().zip(y.iter()) { + match xk.cmp(yk) { + Ordering::Equal => match bson_cmp(xv, yv) { + Ordering::Equal => continue, + other => return other, + }, + other => return other, + } + } + x.len().cmp(&y.len()) + } + // Mismatched types: fall back to "not equal" ordering. The compiler + // shouldn't produce these — if a proptest generates one, the + // parity harness will report the divergence. + _ => Ordering::Equal, + } +} + +fn as_array(val: &Bson) -> Vec { + match val { + Bson::Array(a) => a.clone(), + _ => Vec::new(), + } +} + +fn as_doc(val: &Bson) -> Option<&Document> { + match val { + Bson::Document(d) => Some(d), + _ => None, + } +} + +fn as_bool(val: &Bson) -> Option { + match val { + Bson::Boolean(b) => Some(*b), + _ => None, + } +} + +/// Convert a `"$fieldname"` string to the field name it refers to, or +/// return None for anything else. Used only for `$expr` argument parsing. +fn as_field_ref(val: &Bson) -> Option<&str> { + match val { + Bson::String(s) => s.strip_prefix('$'), + _ => None, + } +} + +/// Minimal regex matcher — the compiler only emits `^prefix` and plain +/// substring patterns via `regex_escape`, so we don't need a full regex +/// engine. Anchors and escaped literals only. +fn regex_match(pattern: &str, s: &str) -> bool { + // Handle "^prefix" — anchored prefix match. + if let Some(prefix) = pattern.strip_prefix('^') { + // The compiler regex-escapes the prefix, so we treat it as a + // literal string here. Any regex metacharacter present means the + // compiler already escaped it. + let unescaped = unescape_regex(prefix); + return s.starts_with(&unescaped); + } + // Unanchored — substring match. Same treatment. + let unescaped = unescape_regex(pattern); + s.contains(&unescaped) +} + +/// Reverse the `regex_escape` transformation in `condition.rs`. Since our +/// compiler only escapes standard regex metacharacters with backslash, we +/// walk the string and unescape those. +fn unescape_regex(pattern: &str) -> String { + let mut out = String::with_capacity(pattern.len()); + let mut chars = pattern.chars(); + while let Some(c) = chars.next() { + if c == '\\' { + if let Some(next) = chars.next() { + out.push(next); + } + } else { + out.push(c); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use bson::doc; + + fn item() -> Document { + doc! { + "_id": "user#1", + "pk": "user#1", + "item_data": { + "name": { "S": "alice" }, + "age": { "N": "30" }, + "tags": { "SS": ["admin", "beta"] }, + "profile": { + "M": { + "email": { "S": "a@x.com" } + } + } + } + } + } + + #[test] + fn scalar_equality_present() { + let filter = doc! { "item_data.name.S": "alice" }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn scalar_equality_absent() { + let filter = doc! { "item_data.missing.S": "alice" }; + assert!(!eval_filter(&filter, &item())); + } + + #[test] + fn exists_true_present() { + let filter = doc! { "item_data.name": { "$exists": true } }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn exists_false_absent() { + let filter = doc! { "item_data.missing": { "$exists": false } }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn implicit_array_match_on_set() { + // `contains(tags, :s)` compiles to `{"item_data.tags.SS": "admin"}` + // and relies on implicit array-match to succeed when "admin" is a + // set member. + let filter = doc! { "item_data.tags.SS": "admin" }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn implicit_array_match_absent_member() { + let filter = doc! { "item_data.tags.SS": "nonmember" }; + assert!(!eval_filter(&filter, &item())); + } + + #[test] + fn lexicographic_lt() { + let filter = doc! { "item_data.name.S": { "$lt": "bob" } }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn and_of_two_clauses() { + let filter = doc! { "$and": [ + { "item_data.name.S": "alice" }, + { "item_data.age.N": "30" } + ]}; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn or_short_circuit() { + let filter = doc! { "$or": [ + { "item_data.name.S": "wrong" }, + { "item_data.age.N": "30" } + ]}; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn nor_negates() { + let filter = doc! { "$nor": [ { "item_data.name.S": "bob" } ] }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn regex_prefix_match() { + let filter = doc! { "item_data.name.S": { "$regex": "^al" } }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn regex_prefix_no_match() { + let filter = doc! { "item_data.name.S": { "$regex": "^bob" } }; + assert!(!eval_filter(&filter, &item())); + } + + #[test] + fn in_membership() { + let filter = doc! { "item_data.name.S": { "$in": ["alice", "bob"] } }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn in_no_match() { + let filter = doc! { "item_data.name.S": { "$in": ["bob", "carol"] } }; + assert!(!eval_filter(&filter, &item())); + } + + #[test] + fn ne_true_when_different() { + let filter = doc! { "item_data.name.S": { "$ne": "bob" } }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn empty_in_sentinel_never_matches() { + // Compiler emits this for empty IN () — a document must have _id + // and _id must have type "null", which contradicts each other for + // any well-formed item. + let filter = doc! { "$and": [ + { "_id": { "$exists": true } }, + { "_id": { "$type": "null" } } + ]}; + assert!(!eval_filter(&filter, &item())); + } + + #[test] + fn nested_map_path() { + let filter = doc! { "item_data.profile.M.email.S": "a@x.com" }; + assert!(eval_filter(&filter, &item())); + } + + #[test] + fn missing_intermediate_path_short_circuits() { + let filter = doc! { "item_data.profile.M.nonexistent.S": "any" }; + assert!(!eval_filter(&filter, &item())); + } +} diff --git a/crates/storage-mongodb/tests/interpreter_selftests.rs b/crates/storage-mongodb/tests/interpreter_selftests.rs new file mode 100644 index 00000000..e466b47a --- /dev/null +++ b/crates/storage-mongodb/tests/interpreter_selftests.rs @@ -0,0 +1,13 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Runs the interpreter's self-tests as an integration test binary. +//! The interpreter itself lives in `tests/common/mod.rs` — this file +//! exists so cargo compiles the module and runs its `#[cfg(test)]` tests. + +#[allow(dead_code, unused_imports)] +mod common; + +// The interpreter's self-tests live inside `common::tests` (gated by +// `#[cfg(test)]`). Cargo runs them automatically when this binary is +// built for `cargo test`. diff --git a/crates/storage-mongodb/tests/pushdown_parity.proptest-regressions b/crates/storage-mongodb/tests/pushdown_parity.proptest-regressions new file mode 100644 index 00000000..87c79459 --- /dev/null +++ b/crates/storage-mongodb/tests/pushdown_parity.proptest-regressions @@ -0,0 +1,8 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 786e25608dd3707c250764c4ed74630a114a629ebbef777a12f98670ef693a98 # shrinks to item = {}, expr_pair = (Or(Function { name: "contains", args: [Path([Attribute("a")]), Placeholder(":v0")] }, Function { name: "attribute_exists", args: [Path([Attribute("a")])] }), [S("")]) +cc ba22175a8084a07e6dc59ba349772e95cdcce6902424574c1083700f364ba6e9 # shrinks to item = {"c": B([0])}, expr_pair = (Compare { left: Path([Attribute("c")]), op: Eq, right: Placeholder(":v0") }, [B([])]) diff --git a/crates/storage-mongodb/tests/pushdown_parity.rs b/crates/storage-mongodb/tests/pushdown_parity.rs new file mode 100644 index 00000000..6e3cba54 --- /dev/null +++ b/crates/storage-mongodb/tests/pushdown_parity.rs @@ -0,0 +1,455 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Filter-pushdown parity harness. +//! +//! For every generated (item, expression) pair drawn from the "pushable +//! subset" (see todo.md A5), assert that +//! +//! `extenddb_core::expression::evaluate_condition(expr, item, maps)` +//! +//! agrees with evaluating +//! +//! `condition_to_filter(expr, maps)` +//! +//! against the item's BSON representation using the interpreter in +//! `tests/common/mod.rs`. Any divergence indicates the compiler emits a +//! filter whose semantics differ from DDB's — a bug in the compiler. +//! +//! The harness deliberately excludes expression shapes that fall outside +//! the pushable subset: no numeric comparisons, no `size()`, and `NOT` +//! only around `attribute_exists` / `attribute_not_exists`. Those cases +//! are handled by fallback to session-scoped in-Rust evaluation in step 3 +//! of A5, and don't need pushdown parity. + +#[allow(dead_code)] +mod common; + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use bson::{Bson, Document}; +use proptest::prelude::*; +use proptest::sample::select; + +use extenddb_core::expression::{CompareOp, Expr, ExpressionMaps, PathElement, evaluate_condition}; +use extenddb_core::types::AttributeValue; +use extenddb_storage_mongodb::condition::condition_to_filter; +use extenddb_storage_mongodb::pushdown::{Pushable, is_pushable}; + +use common::eval_filter; + +// --------------------------------------------------------------------------- +// Attribute-value strategies +// --------------------------------------------------------------------------- + +/// A small vocabulary of attribute names. Reusing names across items and +/// expressions is what surfaces path-hit / path-miss interactions. +const NAMES: &[&str] = &["a", "b", "c", "x", "y"]; + +/// Constrained string alphabet — short, printable, small alphabet. +/// Keeps proptest shrinking behavior tractable and matches the kinds of +/// values real DDB workloads use for status flags, tags, ETags, etc. +fn arb_short_str() -> impl Strategy { + proptest::string::string_regex("[a-z]{0,4}").unwrap() +} + +fn arb_short_bytes() -> impl Strategy> { + proptest::collection::vec(any::(), 0..4) +} + +/// Attribute values in the "safe subset" — everything except N/NS. +/// The pushable expression grammar never references numeric operands, so +/// items don't need to contain them either. Including L/M would require a +/// recursive strategy; we keep depth flat for now (paths in the grammar +/// are single-name only, so nested M/L values wouldn't be reachable +/// anyway). +fn arb_safe_value() -> impl Strategy { + prop_oneof![ + arb_short_str().prop_map(AttributeValue::S), + arb_short_bytes().prop_map(AttributeValue::B), + any::().prop_map(AttributeValue::Bool), + Just(AttributeValue::Null), + proptest::collection::btree_set(arb_short_str(), 0..3).prop_map(AttributeValue::SS), + proptest::collection::btree_set(arb_short_bytes(), 0..3).prop_map(AttributeValue::BS), + // Lists of strings — the compiler's contains-on-list path takes a + // scalar and matches any element. Mixed-type lists aren't in the + // pushable grammar so we don't generate them. + proptest::collection::vec(arb_short_str().prop_map(AttributeValue::S), 0..3) + .prop_map(AttributeValue::L), + ] +} + +/// A random DDB Item. Not every name is present in every item — that's +/// how the "path missing" edge cases get exercised. +fn arb_item() -> impl Strategy> { + proptest::collection::vec( + (select(NAMES).prop_map(String::from), arb_safe_value()), + 0..NAMES.len(), + ) + .prop_map(|pairs| { + let mut m = BTreeMap::new(); + for (k, v) in pairs { + m.insert(k, v); + } + m + }) +} + +// --------------------------------------------------------------------------- +// Expression strategies +// --------------------------------------------------------------------------- + +/// A random attribute name from the vocabulary. +fn arb_name_expr() -> impl Strategy { + select(NAMES).prop_map(|n| Expr::Path(vec![PathElement::Attribute(n.to_string())])) +} + +/// A random placeholder reference (`:v0`, `:v1`, ...). +fn arb_placeholder_ref(idx: usize) -> Expr { + Expr::Placeholder(format!(":v{idx}")) +} + +/// A leaf comparison expression that is safely pushable. Each returns: +/// - the AST for the expression +/// - the values map required to resolve any placeholders in the AST +/// +/// The placeholders are numbered per-expression starting at :v0. When +/// composing with AND/OR (below), we renumber to keep them globally unique. +/// +/// `.B` operands are re-enabled here as of A5 step 3, which fixed +/// `av_to_bson` in `condition.rs` to emit the base64 string form that +/// matches how storage writes `.B` fields. +#[allow(clippy::redundant_closure)] +fn arb_leaf_expr() -> impl Strategy)> { + // We union several leaf shapes. Each yields (Expr, values-list). + let name = || arb_name_expr(); + let str_val = || arb_short_str().prop_map(AttributeValue::S); + let bytes_val = || arb_short_bytes().prop_map(AttributeValue::B); + + prop_oneof![ + // attribute_exists(name) + name().prop_map(|n| ( + Expr::Function { + name: "attribute_exists".into(), + args: vec![n], + }, + vec![] + )), + // attribute_not_exists(name) + name().prop_map(|n| ( + Expr::Function { + name: "attribute_not_exists".into(), + args: vec![n], + }, + vec![] + )), + // NOT attribute_exists(name) — the only NOT the pushable subset allows + name().prop_map(|n| ( + Expr::Not(Box::new(Expr::Function { + name: "attribute_exists".into(), + args: vec![n], + })), + vec![] + )), + // attribute_type(name, :t) for the S / B / BOOL / NULL / SS / BS / L / M types + ( + name(), + select(&["S", "B", "BOOL", "NULL", "SS", "BS", "L", "M"][..]).prop_map(String::from) + ) + .prop_map(|(n, t)| ( + Expr::Function { + name: "attribute_type".into(), + args: vec![n, arb_placeholder_ref(0)], + }, + vec![AttributeValue::S(t)], + )), + // begins_with(name, :prefix) + (name(), arb_short_str()).prop_map(|(n, p)| ( + Expr::Function { + name: "begins_with".into(), + args: vec![n, arb_placeholder_ref(0)], + }, + vec![AttributeValue::S(p)], + )), + // contains(name, :substr) — matches when name is S (substring) or SS/BS/L (membership) + (name(), str_val()).prop_map(|(n, v)| ( + Expr::Function { + name: "contains".into(), + args: vec![n, arb_placeholder_ref(0)], + }, + vec![v], + )), + // name = :v (S) + (name(), str_val()).prop_map(|(n, v)| ( + Expr::Compare { + left: Box::new(n), + op: CompareOp::Eq, + right: Box::new(arb_placeholder_ref(0)), + }, + vec![v], + )), + // name <> :v (S) + (name(), str_val()).prop_map(|(n, v)| ( + Expr::Compare { + left: Box::new(n), + op: CompareOp::Ne, + right: Box::new(arb_placeholder_ref(0)), + }, + vec![v], + )), + // name < :v (S) + (name(), str_val()).prop_map(|(n, v)| ( + Expr::Compare { + left: Box::new(n), + op: CompareOp::Lt, + right: Box::new(arb_placeholder_ref(0)), + }, + vec![v], + )), + // name > :v (S) + (name(), str_val()).prop_map(|(n, v)| ( + Expr::Compare { + left: Box::new(n), + op: CompareOp::Gt, + right: Box::new(arb_placeholder_ref(0)), + }, + vec![v], + )), + // name = :v (B) — base64-string equality equals underlying-byte + // equality, so pushdown is correct for `.B` under `=`. + (name(), bytes_val()).prop_map(|(n, v)| ( + Expr::Compare { + left: Box::new(n), + op: CompareOp::Eq, + right: Box::new(arb_placeholder_ref(0)), + }, + vec![v], + )), + // name <> :v (B) — same, complement of equality. + (name(), bytes_val()).prop_map(|(n, v)| ( + Expr::Compare { + left: Box::new(n), + op: CompareOp::Ne, + right: Box::new(arb_placeholder_ref(0)), + }, + vec![v], + )), + // Note: `<`/`<=`/`>`/`>=` on `.B` is NOT in the pushable subset. + // The compiler emits a base64 string comparison, but DDB + // compares binary values bytewise. Base64 preserves byte + // ordering only for equal-length inputs; different-length + // inputs can invert the order (e.g. bytes [255] > bytes [0, 0] + // bytewise, but "/w==" < "AAA=" lexicographically). Step 3's + // analyzer marks binary ordering as NotPushable and falls + // back to session-scoped in-Rust evaluation. + ] +} + +/// Compose two leaf expressions with AND or OR. Renumbers the second +/// expression's placeholders so both are addressable in the merged +/// values map. +fn arb_composed_expr() -> impl Strategy)> { + (arb_leaf_expr(), arb_leaf_expr(), any::()).prop_map(|(l, r, is_and)| { + let (lhs, mut lvals) = l; + let (rhs, rvals) = r; + let rhs_offset = lvals.len(); + // Renumber rhs placeholders to `:v`. + let rhs = renumber_placeholders(rhs, rhs_offset); + lvals.extend(rvals); + let composed = if is_and { + Expr::And(Box::new(lhs), Box::new(rhs)) + } else { + Expr::Or(Box::new(lhs), Box::new(rhs)) + }; + (composed, lvals) + }) +} + +fn renumber_placeholders(expr: Expr, offset: usize) -> Expr { + match expr { + Expr::Placeholder(name) => { + if let Some(idx_str) = name.strip_prefix(":v") + && let Ok(idx) = idx_str.parse::() + { + return Expr::Placeholder(format!(":v{}", idx + offset)); + } + Expr::Placeholder(name) + } + Expr::Path(_) => expr, + Expr::Compare { left, op, right } => Expr::Compare { + left: Box::new(renumber_placeholders(*left, offset)), + op, + right: Box::new(renumber_placeholders(*right, offset)), + }, + Expr::And(l, r) => Expr::And( + Box::new(renumber_placeholders(*l, offset)), + Box::new(renumber_placeholders(*r, offset)), + ), + Expr::Or(l, r) => Expr::Or( + Box::new(renumber_placeholders(*l, offset)), + Box::new(renumber_placeholders(*r, offset)), + ), + Expr::Not(inner) => Expr::Not(Box::new(renumber_placeholders(*inner, offset))), + Expr::Function { name, args } => Expr::Function { + name, + args: args + .into_iter() + .map(|a| renumber_placeholders(a, offset)) + .collect(), + }, + Expr::Between { operand, low, high } => Expr::Between { + operand: Box::new(renumber_placeholders(*operand, offset)), + low: Box::new(renumber_placeholders(*low, offset)), + high: Box::new(renumber_placeholders(*high, offset)), + }, + Expr::In { operand, list } => Expr::In { + operand: Box::new(renumber_placeholders(*operand, offset)), + list: list + .into_iter() + .map(|a| renumber_placeholders(a, offset)) + .collect(), + }, + other => other, + } +} + +/// Generate either a leaf or a composed AND/OR expression. +fn arb_expr() -> impl Strategy)> { + prop_oneof![ + 3 => arb_leaf_expr(), + 1 => arb_composed_expr(), + ] +} + +// --------------------------------------------------------------------------- +// Item → BSON conversion for the interpreter side +// --------------------------------------------------------------------------- + +/// Serialize an Item to the BSON shape the compiler assumes: +/// { item_data: { : , ... } } +fn item_to_bson_doc(item: &BTreeMap) -> Document { + let item_data_json = serde_json::to_value(item).expect("item serializes"); + let item_data_bson: Bson = bson::to_bson(&item_data_json).expect("BSON conversion"); + let mut doc = Document::new(); + doc.insert("item_data", item_data_bson); + doc +} + +// --------------------------------------------------------------------------- +// The parity property +// --------------------------------------------------------------------------- + +proptest! { + #![proptest_config(ProptestConfig { + cases: 1024, + max_shrink_iters: 4096, + ..Default::default() + })] + + /// For any pushable expression and any item, the compiled MongoDB + /// filter (evaluated by our BSON interpreter) must agree with the + /// in-Rust DDB evaluator's pass/fail result. + #[test] + fn compiled_filter_matches_ddb_evaluator( + item in arb_item(), + expr_pair in arb_expr(), + ) { + let (expr, values_vec) = expr_pair; + + // Build the ExpressionMaps that the DDB evaluator and the compiler + // both consume. Placeholders are :v0, :v1, ... in insertion order. + let mut values = HashMap::new(); + for (idx, v) in values_vec.iter().enumerate() { + values.insert(format!(":v{idx}"), v.clone()); + } + let maps = ExpressionMaps::new(HashMap::new(), values); + + // Path A: in-Rust DDB evaluator against the logical Item. + let ddb_result = evaluate_condition(&expr, &item, &maps).unwrap_or(false); + + // Path B: compile to MongoDB filter, then evaluate the filter + // against the item's BSON representation using our interpreter. + let filter = match condition_to_filter(&expr, &maps) { + Ok(f) => f, + Err(e) => { + // The compiler rejected this expression. That's fine — it + // means A5's fallback path would kick in. Skip this case + // rather than treating it as a mismatch. + // + // We still assert the compiler doesn't reject something + // the DDB evaluator accepts as trivially-true or trivially- + // false — but the compiler rejecting compilation is + // logically different from evaluating to false. + let _ = e; + return Ok(()); + } + }; + let bson_doc = item_to_bson_doc(&item); + let mongo_result = eval_filter(&filter, &bson_doc); + + prop_assert_eq!( + ddb_result, + mongo_result, + "parity mismatch:\n expr: {:?}\n item: {:?}\n filter: {:?}\n ddb: {}\n mongo: {}", + expr, + item, + filter, + ddb_result, + mongo_result, + ); + } + + /// The analyzer's soundness contract: for every expression it marks + /// `Pushable::Yes`, the compiled filter must agree with the DDB + /// evaluator on the generated item. + /// + /// This is a stricter check than the first property because the + /// analyzer whitelists a specific subset — if it marks Yes on an + /// expression whose compiled filter drifts from DDB, that's an + /// analyzer bug (whitelist too generous). The generator draws from + /// the same pushable grammar as the first property so most cases + /// hit the analyzer's Yes branch; when generation drifts into + /// non-pushable AST shapes (e.g., unhandled operand-type interactions + /// in composed expressions), the analyzer says No and the test + /// short-circuits without a comparison. + #[test] + fn analyzer_yes_implies_filter_parity( + item in arb_item(), + expr_pair in arb_expr(), + ) { + let (expr, values_vec) = expr_pair; + let mut values = HashMap::new(); + for (idx, v) in values_vec.iter().enumerate() { + values.insert(format!(":v{idx}"), v.clone()); + } + let maps = ExpressionMaps::new(HashMap::new(), values); + + if !matches!(is_pushable(&expr, &maps), Pushable::Yes) { + return Ok(()); + } + + let filter = condition_to_filter(&expr, &maps).expect( + "analyzer said Yes but compiler failed — analyzer/compiler drift", + ); + let bson_doc = item_to_bson_doc(&item); + let mongo_result = eval_filter(&filter, &bson_doc); + let ddb_result = evaluate_condition(&expr, &item, &maps).unwrap_or(false); + + prop_assert_eq!( + ddb_result, + mongo_result, + "analyzer said Pushable::Yes but results diverged:\n expr: {:?}\n filter: {:?}\n ddb: {}, mongo: {}", + expr, + filter, + ddb_result, + mongo_result, + ); + } +} + +// Silence unused-import lint from `common` when running only a subset. +#[allow(dead_code)] +fn _keep_common_imported() { + let _ = std::mem::size_of::>(); + let _ = std::mem::size_of::>(); +} diff --git a/crates/storage/src/error.rs b/crates/storage/src/error.rs index bf851968..a2043460 100755 --- a/crates/storage/src/error.rs +++ b/crates/storage/src/error.rs @@ -24,6 +24,13 @@ pub enum StorageError { IdempotentReplay, #[error("Idempotent parameter mismatch")] IdempotentMismatch, + /// A single-item write raced an in-flight `TransactWriteItems` on + /// the same item, and the backend was unable to serialize the two. + /// Maps to `DynamoDbError::TransactionConflictException` at the + /// engine boundary — DynamoDB's canonical error for this case + /// (RFC-0003 §4.3). + #[error("Transaction conflict: {0}")] + TransactionConflict(String), #[error("No-op update: {0}")] NoOpUpdate(String), #[error("Validation error: {0}")] diff --git a/devtools/run-mongodb-tests b/devtools/run-mongodb-tests new file mode 100755 index 00000000..d7dfff23 --- /dev/null +++ b/devtools/run-mongodb-tests @@ -0,0 +1,258 @@ +#!/usr/bin/env bash +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Orchestrated mongo integration-test runner. +# +# Starts a mongo:7 single-node replica set in Docker, initializes and +# serves extenddb against it, then delegates the test workload to +# `devtools/run-tests --backend mongodb`. Tears everything down on exit. +# +# Usage: +# devtools/run-mongodb-tests [OPTIONS] [-- RUN_TESTS_ARGS ...] +# +# Options: +# --port PORT HTTPS port extenddb should bind (default: 18443) +# --mongo-port PORT Host port to publish mongo on (default: 27021) +# --output DIR Directory for logs and generated config +# (default: /tmp/run-mongodb-tests-) +# --keep Do not tear down the mongo container / server +# on exit (useful for post-run inspection) +# -h, --help Show this help +# +# Arguments after `--` are passed to `devtools/run-tests` verbatim. +# Default suite is `--pytest --comprehensive --parallel`. +# +# Examples: +# devtools/run-mongodb-tests +# devtools/run-mongodb-tests -- --pytest --filter test_put_item +# devtools/run-mongodb-tests --keep -- --pytest --filter test_rfc0003 +# +# Prerequisites: +# - Docker running +# - `cargo build --release --features mongodb` already done +# - Python venv activated with pytest installed +# - `~/.extenddb/tls/` populated (from a prior `extenddb init`) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$PROJECT_DIR" + +# --------------------------------------------------------------------------- +# Argument parsing +# --------------------------------------------------------------------------- + +BIND_PORT=18443 +MONGO_PORT=27021 +OUTPUT_DIR="" +KEEP=false +RUN_TESTS_ARGS=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --port) BIND_PORT="$2"; shift 2 ;; + --mongo-port) MONGO_PORT="$2"; shift 2 ;; + --output) OUTPUT_DIR="$2"; shift 2 ;; + --keep) KEEP=true; shift ;; + --) shift; RUN_TESTS_ARGS=("$@"); break ;; + -h|--help) sed -n '5,35p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "error: unknown option: $1" >&2; exit 1 ;; + esac +done + +if [[ ${#RUN_TESTS_ARGS[@]} -eq 0 ]]; then + RUN_TESTS_ARGS=(--pytest --comprehensive --parallel) +fi + +if [[ -z "$OUTPUT_DIR" ]]; then + OUTPUT_DIR="/tmp/run-mongodb-tests-$(date +%Y%m%d-%H%M%S)" +fi +mkdir -p "$OUTPUT_DIR" + +CONFIG="$OUTPUT_DIR/extenddb.toml" +SERVER_LOG="$OUTPUT_DIR/server.log" +CONTAINER_NAME="extenddb-runtests-mongo" +BINARY="./target/release/extenddb" + +if [[ ! -x "$BINARY" ]]; then + echo "error: $BINARY not found. Build first: cargo build --release --features mongodb" >&2 + exit 1 +fi + +# Canonicalized `/tmp` — resolves through the macOS `/tmp -> /private/tmp` +# symlink so the server's import/export path check (which rejects symlink +# components) accepts it. Portable across Linux and macOS. +TMP_CANON=$(realpath /tmp) + +# --------------------------------------------------------------------------- +# Cleanup +# --------------------------------------------------------------------------- + +cleanup() { + if $KEEP; then + echo "" + echo "=== --keep set; leaving mongo + extenddb running ===" + echo " server log (startup): $SERVER_LOG" + echo " mongo container: $CONTAINER_NAME (port $MONGO_PORT)" + echo " tear down manually: $BINARY stop --config $CONFIG; docker rm -f $CONTAINER_NAME" + return + fi + echo "" + echo "=== Cleanup ===" + if [[ -f "$CONFIG" ]] && "$BINARY" stop --config "$CONFIG" >/dev/null 2>&1; then + echo " stopped extenddb server" + fi + if docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1; then + echo " removed docker container $CONTAINER_NAME" + fi +} +trap cleanup EXIT + +# --------------------------------------------------------------------------- +# Start mongo replica set +# --------------------------------------------------------------------------- + +echo "=== Starting MongoDB 7 replica set on host port $MONGO_PORT ===" +docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true +docker run -d --name "$CONTAINER_NAME" -p "$MONGO_PORT:27017" mongo:7 \ + mongod --replSet rs0 --bind_ip_all --wiredTigerCacheSizeGB 1 >/dev/null + +# Wait for mongod to accept pings +for i in $(seq 1 30); do + if docker exec "$CONTAINER_NAME" mongosh --quiet --eval "db.runCommand({ping:1}).ok" 2>/dev/null \ + | grep -q '^1$'; then + echo " mongod responsive after ${i}s" + break + fi + sleep 1 +done + +# Initiate replica set (idempotent — rs.initiate() errors are non-fatal) +docker exec "$CONTAINER_NAME" mongosh --quiet --eval \ + 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "localhost:27017"}]})' >/dev/null 2>&1 || true + +# Wait for PRIMARY election +for i in $(seq 1 30); do + state=$(docker exec "$CONTAINER_NAME" mongosh --quiet --eval 'rs.status().members[0].stateStr' 2>/dev/null | tail -1) + if [[ "$state" == "PRIMARY" ]]; then + echo " replica set PRIMARY after ${i}s" + break + fi + sleep 1 +done + +# --------------------------------------------------------------------------- +# Initialize extenddb +# --------------------------------------------------------------------------- + +echo "" +echo "=== Initializing extenddb ===" + +# `init --backend mongodb` reads the connection string from the config +# file, so write a minimal stub for it first. +cat > "$CONFIG" <&1 | tail -3 + +# `init` regenerates the config with default port 18443 and no +# `[import]` / `[export]` sections. Rewrite with the port + defaults the +# integration suite needs. This overwrite falls away once `extenddb init` +# grows flags for these knobs. +cat > "$CONFIG" <"$SERVER_LOG" 2>&1 + +for i in $(seq 1 30); do + if curl -sk "https://127.0.0.1:$BIND_PORT/health" >/dev/null 2>&1; then + echo " server healthy after ${i}s" + break + fi + sleep 1 +done + +if ! curl -sk "https://127.0.0.1:$BIND_PORT/health" >/dev/null 2>&1; then + echo "error: extenddb server did not become healthy" >&2 + tail -30 "$SERVER_LOG" >&2 + exit 1 +fi + +# --------------------------------------------------------------------------- +# Delegate to run-tests +# --------------------------------------------------------------------------- + +echo "" +echo "=== Running test suite via devtools/run-tests --backend mongodb ===" +echo " passthrough args: ${RUN_TESTS_ARGS[*]}" +echo "" + +export EXTENDDB_TEST_ENDPOINT="https://127.0.0.1:$BIND_PORT" +export EXTENDDB_ADMIN_USER=admin +export EXTENDDB_ADMIN_PASSWORD="$ADMIN_PASSWORD" + +# Some tests read EXTENDDB_CONFIG to invoke `extenddb settings` against +# the running instance. Without this, they fall back to `./extenddb.toml` +# in the repo root, which is a stale dev config that points at a +# postgres instance that isn't running — the CLI hangs on the doomed +# connection and pytest times its subprocess out. +export EXTENDDB_CONFIG="$CONFIG" + +# import/export tests use tempfile.NamedTemporaryFile(), which honors +# $TMPDIR. On macOS $TMPDIR defaults to /var/folders/…/T, which isn't +# inside the server's configured [import]/[export] paths. Point TMPDIR +# at the canonical `/tmp` (matches the paths written above) so the +# tempfiles land in an allowed location on both Linux and macOS. +export TMPDIR="$TMP_CANON" + +RC=0 +devtools/run-tests --extenddb --backend mongodb --config "$CONFIG" "${RUN_TESTS_ARGS[@]}" || RC=$? + +exit $RC diff --git a/devtools/run-tests b/devtools/run-tests index 220f54c5..cfa13c95 100755 --- a/devtools/run-tests +++ b/devtools/run-tests @@ -13,6 +13,7 @@ # devtools/run-tests --extenddb --rust --release # devtools/run-tests --extenddb --filter PATTERN --pytest # devtools/run-tests --extenddb --catalog-check --config PATH +# devtools/run-tests --extenddb --pytest --backend mongodb # # Target flag (exactly one required, mutually exclusive): # --extenddb targeting a local extenddb instance @@ -28,6 +29,10 @@ # --catalog-check post-test catalog integrity check # # Options: +# --backend NAME storage backend the running server uses: +# postgres (default) or mongodb. +# Gates postgres-only paths (pg connection-string +# extraction, test_cli_lifecycle.py). # --release use release build for Rust tests # --filter PATTERN pass to cargo test / pytest -k # --parallel[=N] run pytest in parallel (default: 1/3 of CPU cores) @@ -67,6 +72,7 @@ RELEASE=false FILTER="" CONFIG_PATH="" PARALLEL="" +BACKEND="postgres" usage() { cat <<'EOF' @@ -86,6 +92,8 @@ Suites (at least one required): --catalog-check post-test catalog integrity check Options: + --backend NAME storage backend the running server uses: + postgres (default) or mongodb. --release release build for Rust tests --filter PATTERN filter for cargo test / pytest -k --parallel[=N] run pytest in parallel (default: 1/3 of CPU cores) @@ -112,6 +120,7 @@ while [[ $# -gt 0 ]]; do --filter) FILTER="$2"; shift 2 ;; --catalog-check) RUN_CATALOG_CHECK=true; shift ;; --config) CONFIG_PATH="$2"; shift 2 ;; + --backend) BACKEND="$2"; shift 2 ;; --parallel) # --parallel (no argument) or --parallel=N PARALLEL="auto"; shift @@ -124,6 +133,15 @@ while [[ $# -gt 0 ]]; do esac done +# --- Validate: backend name --- +case "$BACKEND" in + postgres|mongodb) ;; + *) + echo "error: --backend must be 'postgres' or 'mongodb' (got: $BACKEND)" + exit 1 + ;; +esac + # --- Validate: target flag required --- if [[ -z "$TARGET" ]]; then echo "error: target flag is required (--extenddb or --real-dynamodb)" @@ -212,6 +230,7 @@ if [[ "$TARGET" == "real-dynamodb" ]]; then echo " EXTENDDB_TEST_ENDPOINT = (not set — using AWS SDK defaults)" echo " AWS_ACCESS_KEY_ID = ${AWS_ACCESS_KEY_ID:-(from ~/.aws)}" else + echo " backend = $BACKEND" echo " EXTENDDB_TEST_ENDPOINT = ${EXTENDDB_TEST_ENDPOINT:-}" echo " AWS_ACCESS_KEY_ID = ${AWS_ACCESS_KEY_ID:-(will be provisioned)}" fi @@ -345,18 +364,17 @@ if $NEEDS_INTEGRATION && [[ "$TARGET" != "real-dynamodb" ]]; then fi # Export PG connection string for CLI lifecycle tests (PostgreSQL only). - # Extract from config, strip the database name to get the base URL. - # This is only needed for test_cli_lifecycle.py which is PostgreSQL-specific - # and excluded from the main pytest suite. - if [[ -z "${EXTENDDB_TEST_PG_CONNECTION_STRING:-}" && -f "$CONFIG_FOR_SETTINGS" ]]; then - # Only try to extract connection_string for PostgreSQL backend - if grep -q '^[[:space:]]*backend[[:space:]]*=[[:space:]]*"postgres"' "$CONFIG_FOR_SETTINGS" 2>/dev/null; then - FULL_CONN=$(grep 'connection_string' "$CONFIG_FOR_SETTINGS" | sed -n 's/.*connection_string[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' | head -1) - if [[ -n "$FULL_CONN" ]]; then - # Strip trailing /database_name to get base URL - export EXTENDDB_TEST_PG_CONNECTION_STRING="${FULL_CONN%/*}" - echo " ✓ EXTENDDB_TEST_PG_CONNECTION_STRING=${EXTENDDB_TEST_PG_CONNECTION_STRING}" - fi + # test_cli_lifecycle.py is PostgreSQL-specific and excluded from the + # main pytest suite; it opens its own extenddb instances via the + # connection string. Skipped on the mongo backend. + if [[ "$BACKEND" == "postgres" \ + && -z "${EXTENDDB_TEST_PG_CONNECTION_STRING:-}" \ + && -f "$CONFIG_FOR_SETTINGS" ]]; then + FULL_CONN=$(grep 'connection_string' "$CONFIG_FOR_SETTINGS" | sed -n 's/.*connection_string[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' | head -1) + if [[ -n "$FULL_CONN" ]]; then + # Strip trailing /database_name to get base URL + export EXTENDDB_TEST_PG_CONNECTION_STRING="${FULL_CONN%/*}" + echo " ✓ EXTENDDB_TEST_PG_CONNECTION_STRING=${EXTENDDB_TEST_PG_CONNECTION_STRING}" fi fi echo "" @@ -513,8 +531,11 @@ if $RUN_COMPREHENSIVE; then echo "" fi -# --- CLI lifecycle tests (run last — they start/stop their own servers) --- -if $RUN_PYTEST && [[ "$TARGET" != "real-dynamodb" && -n "${EXTENDDB_TEST_PG_CONNECTION_STRING:-}" ]]; then +# --- CLI lifecycle tests (postgres-only; run last — they start/stop their own servers) --- +if $RUN_PYTEST \ + && [[ "$TARGET" != "real-dynamodb" \ + && "$BACKEND" == "postgres" \ + && -n "${EXTENDDB_TEST_PG_CONNECTION_STRING:-}" ]]; then CLI_OUTFILE="discussions/test-cli-${HASH}.txt" echo "=== CLI lifecycle tests → ${CLI_OUTFILE} ===" CLI_ARGS=(python3 -m pytest tests/test_cli_lifecycle.py -v) diff --git a/docs/design/13-storage-mongodb.md b/docs/design/13-storage-mongodb.md new file mode 100644 index 00000000..55ef6ef5 --- /dev/null +++ b/docs/design/13-storage-mongodb.md @@ -0,0 +1,1100 @@ +# Design: MongoDB Storage Backend + +## 1. Overview + +The MongoDB backend (`extenddb-storage-mongodb`) implements the same trait surface as +`extenddb-storage-postgres`: the six engine traits (`TableEngine`, `DataEngine`, +`MetadataEngine`, `StreamEngine`, `BackupEngine`, `WorkerStore`) and the catalog +traits (`ManagementStore`, `AdminStore`, `SettingsStore`, `MetricsStore`, +`RateLimitStore`, `AuthorizationStore`). + +**Driver:** `mongodb` (official Rust driver, async, multi-document ACID +transactions on replica sets). + +**Minimum MongoDB version:** 6.0 (multi-document transactions, snapshot reads). + +**Read preference:** `primary` only. `MongoEngine::new` rejects connection strings +that request `secondary`, `secondaryPreferred`, `primaryPreferred`, or `nearest` — +DynamoDB's `ConsistentRead=true` contract requires linearizable reads, which only +Primary provides. Silently routing reads to replicas would return stale data with +no signal to the caller. + +## 2. Database Layout + +Two databases, mirroring the PostgreSQL backend's catalog / data separation: + +| Database | Purpose | +|----------------------|------------------------------------------------------| +| `extenddb_catalog` | Table metadata, IAM, settings, metrics, backup metadata | +| `extenddb_data` | Per-table item collections, per-index collections, streams, idempotency tokens, backup snapshots | + +DynamoDB Streams are implemented via inline stream-record writes during data +operations. GSI updates are propagated synchronously inline for existing indexes +and asynchronously via a background worker during UpdateTable-driven GSI +creation. + +## 3. Catalog Database Collections + +Collections created by `run_catalog_migrations` in `bootstrapper.rs`. Two +additional collections (`iam_group_members`, `backup_items` — the latter is +no longer used) are auto-created on first insert and not in the migration list. + +### 3.1 `accounts` +```json +{ "_id": "", "account_name": "...", "created_at": ISODate } +``` +Unique index on `account_name`. + +### 3.2 `tables` +```json +{ + "_id": { "account_id": "...", "table_name": "..." }, + "key_schema": [...], + "attribute_definitions": [...], + "billing_mode": "PAY_PER_REQUEST", + "provisioned_throughput": { ... }, + "stream_specification": { ... }, + "table_status": "ACTIVE", + "creation_date_time": ISODate, + "table_size_bytes": NumberLong, + "item_count": NumberLong, + "table_arn": "...", + "table_id": "", + "ttl_attribute": null, + "ttl_index_ready": false, + "deletion_protection_enabled": false, + "status_transition_at": null, + "stream_label": null, + "table_class": null, + "sse_specification": null, + "on_demand_throughput": null +} +``` +Unique index on `table_id`. Partial index on `status_transition_at` where not null. + +### 3.3 `indexes` +```json +{ + "_id": { "table_id": "...", "index_name": "..." }, + "index_id": "", + "index_type": "GSI|LSI", + "key_schema": [...], + "projection": { ... }, + "index_status": "ACTIVE|CREATING", + "provisioned_throughput": { ... }, + "backfill_cursor": // present while index_status = CREATING +} +``` +`backfill_cursor` is written by `gsi_backfill_worker` after every batch. The +field is unset when the index flips to `ACTIVE`. + +### 3.4 `tags` +```json +{ "_id": { "resource_arn": "...", "tag_key": "..." }, "tag_value": "..." } +``` + +### 3.5 `settings` +```json +{ "_id": "", "value": "..." } +``` +Bootstrapped keys include `catalog_version`, `encryption_key` (base64-encoded +256-bit AES-GCM key), `data_database_name`, and `data_connection_string`. + +### 3.6 `admin_users` +```json +{ "_id": "", "password_hash": "", "created_at": ISODate } +``` + +### 3.7 `iam_users` +```json +{ + "_id": { "account_id": "...", "user_name": "..." }, + "user_arn": "...", + "password_hash": null, + "tags": { "": "", ... }, + "created_at": ISODate +} +``` +Unique index on `user_arn`. + +### 3.8 `access_keys` +```json +{ + "access_key_id": "", + "secret_key_encrypted": BinData, + "account_id": "...", + "user_name": "...", + "is_active": true, + "created_at": ISODate +} +``` +Index on `(account_id, user_name)`. `secret_key_encrypted` is AES-256-GCM +ciphertext of the secret access key using the `settings.encryption_key`; +`access_key_id` is bound as additional authenticated data. + +### 3.9 `iam_groups` +```json +{ + "_id": { "account_id": "...", "group_name": "..." }, + "group_arn": "...", + "members": ["user1", "user2"], + "created_at": ISODate +} +``` +Unique index on `group_arn`. + +### 3.10 `iam_roles` +```json +{ + "_id": { "account_id": "...", "role_name": "..." }, + "role_arn": "...", + "trust_policy": { ... }, + "permissions_boundary_arn": null, + "tags": { "": "", ... }, + "created_at": ISODate +} +``` +Unique index on `role_arn`. + +### 3.11 `iam_sessions` +```json +{ + "_id": "", + "access_key_id": "ASIA...", + "secret_key_encrypted": BinData, + "account_id": "...", + "role_name": "...", + "session_name": "...", + "session_tags": { ... }, + "session_policy": { ... }, + "expires_at": ISODate, + "created_at": ISODate +} +``` +Unique index on `access_key_id`. TTL index on `expires_at` (`expireAfterSeconds: 0`). + +### 3.12 `iam_policies` +```json +{ + "account_id": "...", + "principal_type": "user|role|group", + "principal_name": "...", + "policy_name": "...", + "policy_document": { ... }, + "created_at": ISODate +} +``` + +### 3.13 `iam_permissions_boundaries` +```json +{ + "account_id": "...", + "principal_type": "user|role", + "principal_name": "...", + "policy_document": { ... } +} +``` + +### 3.14 `metrics` +```json +{ + "_id": { "bucket": ISODate, "metric": "...", "table_name": "...", "index_name": "...", "operation": "..." }, + "sum": 0.0, + "count": NumberLong(0), + "min": Infinity, + "max": -Infinity +} +``` +Index on `_id.bucket` for pruning. + +### 3.15 `login_attempts` +```json +{ "principal": "...", "attempted_at": ISODate, "success": false, "source_ip": "..." } +``` +Compound index on `(principal, attempted_at)`. + +### 3.16 `backups` +```json +{ + "_id": "", + "backup_id": "", + "backup_name": "...", + "backup_status": "AVAILABLE|DELETED", + "backup_type": "USER", + "table_id": "...", + "table_name": "...", + "table_arn": "...", + "account_id": "...", + "backup_size_bytes": NumberLong, + "item_count": NumberLong, + "key_schema": [...], + "attribute_definitions": [...], + "billing_mode": "PAY_PER_REQUEST", + "table_class": null, + "sse_specification": null, + "on_demand_throughput": null, + "created_at": ISODate, + "table_creation_date_time": +} +``` +Index on `(account_id, table_name)`. The physical backup collection lives in +`extenddb_data` as `_backup_{backup_id}` — see §4.5. + +### 3.17 `continuous_backups` +```json +{ + "account_id": "...", + "table_name": "...", + "pitr_enabled": false, + "earliest_restorable": null, + "latest_restorable": null +} +``` + +### 3.18 `schema_history` +```json +{ "_id": "", "applied_at": ISODate } +``` + +## 4. Data Database Collections + +### 4.1 Per-Table Item Collections: `_ddb_{table_id}` + +Each DynamoDB virtual table maps to a MongoDB collection named +`_ddb_{table_id}` (table_id is a UUID assigned at CreateTable). The +collection-name derivation shields the physical layer from caller-visible +name changes and from characters that are unsafe as MongoDB collection +names. + +**Document structure:** +```json +{ + "_id": "", + "pk": "...", + "sk_s": "...", + "sk_n": Decimal128, + "sk_b": "", + "_v": NumberLong, + "item_data": { ... } +} +``` + +Fields: + +- `_id` — netstring-encoded composite key `:,:,`. + Netstring framing gives an unambiguous boundary between `pk` and `sk` + regardless of their contents. A naive `{pk}#{sk}` delimiter collides + on `pk="a#b",sk="c"` vs `pk="a",sk="b#c"`. PK-only tables use raw `pk` + text as `_id`. +- `pk` — partition key text (matches `composite_pk_to_text` from + `extenddb_storage::util`). +- `sk_s` / `sk_n` / `sk_b` — typed sort key, absent when the schema has + no sort key. See §5.2 for the type-specific encoding. +- `_v` — OCC version counter used by `UpdateItem`'s versioned filter + guard. Absent on freshly-inserted rows (treated as 0); bumped by every + update, including the native fast path. +- `item_data` — full DynamoDB item serialized as BSON via the + `AttributeValue` JSON representation. Non-key attribute values retain + their DynamoDB type tags (`{"S": "hello"}`, `{"N": "42"}`, ...). + +**Indexes:** + +- `{ pk: 1 }` (PK-only tables) or `{ pk: 1, sk_?: 1 }`, unique. + String sort-key indexes use `{ locale: "simple" }` collation for + byte-order comparisons. + +### 4.2 Per-Index Collections: `_ddb_{index_id}` + +Each GSI/LSI has its own collection. The document schema extends §4.1 +with the base table's key attributes as first-class fields: + +```json +{ + "_id": "", + "pk": "...", // index partition key + "sk_s|sk_n|sk_b": ..., // index sort key + "base_pk": "...", // base-table partition key text + "base_sk_s|_n|_b": ..., // base-table sort key, typed + "item_data": { ... } // projected item per the index's Projection +} +``` + +Two reasons for the extra base-key material: + +1. **Unique identity per base item.** GSI keys are non-unique — multiple + base items can share `(index_pk, index_sk)`. If `_id` were only + derived from index keys, two base items with the same GSI-key values + would upsert to the same document; one would silently overwrite the + other. Including the base keys in `_id` and the entry-delete filter + (`index_entry_filter`) makes each entry addressable independently. +2. **Compound pagination cursor.** Index queries and scans sort and + paginate on `(index_sk?, base_pk, base_sk?)` — see §5.3. + Having the base-key components as fields (not buried under + `item_data..S`) lets the sort and cursor filters use per-field + indexes. + +Every index collection carries a compound index on +`(pk, sk_?, base_pk, base_sk_?)` created by `create_index_data_collection`. +Simple collation is used whenever the tuple contains a string component. + +### 4.3 `stream_records` +```json +{ + "sequence_number": "<21-digit zero-padded>", + "shard_id": "shardId-{table_id}-{index:012}", + "table_id": "...", + "event_name": "INSERT|MODIFY|REMOVE", + "record_data": { ... full StreamRecord as BSON ... }, + "created_at": ISODate +} +``` + +- TTL index on `created_at` with `expireAfterSeconds = 24 * 3600` — + primary retention enforcement. +- Compound index on `(shard_id, sequence_number)` — powers `GetRecords` + (`shard_id` equality + `sequence_number > cursor` range with ascending + sort). Without it, every consumer poll runs a full-collection scan. + +### 4.4 `stream_shards` +```json +{ + "shard_id": "shardId-{table_id}-{index:012}", + "table_id": "...", + "starting_sequence_number": "<21-digit>", + "ending_sequence_number": null, + "created_at": ISODate +} +``` + +Unique index on `shard_id`. Four shards per stream-enabled table. + +`shard_id` embeds `table_id` (a UUID) rather than `table_name`. Table +names are only unique per-account; a name-derived scheme would let one +account's `GetRecords(shard_id)` observe another account's records on +same-named tables. `table_id` resets on `DeleteTable + CreateTable`, so +recreated tables get fresh shard_ids and leftover records from the +deleted table cannot resurface. + +### 4.5 `counters` +```json +{ "_id": "stream_seq:", "value": NumberLong } +``` + +One document per shard. `$inc` on `value` inside a session yields the +next sequence number. Per-shard counters (not a single global counter) +preserve DynamoDB Streams' contract that sequence numbers are strictly +monotonic within a shard and independent across shards. + +### 4.6 `idempotency_tokens` +```json +{ + "account_id": "...", + "token": "...", + "fingerprint": "...", + "created_at": ISODate +} +``` + +- TTL index on `created_at` with `expireAfterSeconds = 540`. +- **Unique compound index on `(account_id, token)`.** + +The TTL is 540s (9 min), tighter than DDB's 10-min window. MongoDB's TTL +monitor runs on a ~60s cadence, so worst-case retention with TTL = 540s +is ≤10 min. The data-plane read path (`transact_write_items_impl`) also +filters existing rows by `created_at` age < 600 000 ms so retention is +correct regardless of TTL-monitor timing. + +The unique index closes a race window: two concurrent `TransactWriteItems` +calls with the same token both take snapshot reads that miss the other's +uncommitted insert; without the constraint, both would commit and the +operation would execute twice. With it, the second inserter fails +`E11000` and the write path resolves the winner by re-reading (still +subject to the age filter — if the winner has just expired, the retry +does a fresh insert). + +### 4.7 `_backup_{backup_id}` + +One collection per user-created backup. Populated by a server-side +`[{ $out: "_backup_{backup_id}" }]` aggregation pipeline on the source +data collection, so items are copied server-side without transferring +through the driver. Restored the same way, in reverse. `DeleteBackup` +drops the collection. + +## 5. Key Design Decisions + +### 5.1 Session-scoped conditional writes + +`PutItem`, `DeleteItem`, and `UpdateItem` — when they carry a +`ConditionExpression`, a `StreamCapture`, or write to a table with GSIs — +run inside a MongoDB client session bound to a multi-document transaction +with snapshot read concern and majority write concern. Within the session: + +1. `find_one` the current document. +2. Evaluate the DynamoDB condition in Rust + (`extenddb_core::expression::evaluate_condition`) against the loaded + item. +3. Write (`find_one_and_replace` / `delete_one` / versioned `replace_one`). +4. Synchronize GSIs (`sync_indexes_in_session`). +5. Emit any stream record (`write_stream_inline_in_session`), including + per-shard sequence-number `$inc` — also in the same session. +6. Commit. + +All five happen on the same session, so a concurrent conflicting writer +manifests as a WriteConflict at commit — which the caller retries — not +as a stale-read anomaly. The pre-image loaded in step 1 is reused for +`ReturnValuesOnConditionCheckFailure = ALL_OLD` and for `OldImage` on any +attached stream capture; no follow-up read is needed. + +Update-as-insert (the pre-image was `None`) emits an `INSERT` stream +event with no `OldImage`, not a `MODIFY` with a fabricated key-only stub. + +`UpdateItem` also always fetches the pre-image regardless of the caller's +`ReturnValues` setting — the pre-image is required to compute correct +GSI deltas when the update changes or removes an indexed attribute, and +skipping it leaves stale entries in index collections forever. + +**Native fast path.** For unconditional updates on tables with no streams +and no GSIs (fresh cache says `Some(false)`), the backend collapses the +transaction to a single `find_one_and_update` outside any session using +compiled MongoDB atomic operators (`$set` / `$unset` / plus +`$inc: {_v: 1}`). The `_v` bump is unconditional on this path: without +it, a concurrent session-scoped update running against a stale snapshot +could pass its versioned filter and lost-update over the fast-path +write. + +Implementation: `data_engine.rs::put_item_impl`, `delete_item_impl`, +`update_item_impl`, and `execute_transact_write_op_in_session` for the +TWI arms. + +### 5.2 Filter-pushdown fast path (analyzer-gated) + +An optional pushdown fast path skips the session for conditional writes +that a static analyzer certifies as safe. The path is gated on: + +- `condition` is present. +- `stream` is `None`. +- `gsi_cache_get_fresh(table_id) == Some(false)` (i.e. the cache is + fresh AND says the table has no GSIs). +- `pushdown::is_pushable(cond, maps) == Pushable::Yes`. + +Under those guards, single-document `find_one_and_replace` / +`find_one_and_delete` provides atomicity — no session, no GSI sync, no +stream write. The compiled filter is merged with the primary-key filter +under `$and`. + +The **compiler** (`condition.rs`) is intentionally broader than +production usage: it translates +`attribute_exists`, `attribute_not_exists`, `attribute_type`, +`begins_with`, `contains`, `BETWEEN`, `IN`, `=`, `<>`, `<`, `<=`, `>`, +`>=`, `AND`, `OR`, `NOT`, and `size` into BSON filters. Some of those +translations are correct only for certain operand types. + +The **analyzer** (`pushdown.rs::is_pushable`) is the load-bearing +correctness boundary. It certifies a whole-condition subset that is +provably in agreement with `evaluate_condition`: + +- Existence functions (`attribute_exists`, `attribute_not_exists`) — always + pushable. +- `attribute_type(path, :t)` — pushable when `:t` resolves to a placeholder + whose value is one of the 10 valid DDB type tags + (`S`, `N`, `B`, `BOOL`, `NULL`, `L`, `M`, `SS`, `NS`, `BS`). Without + the whitelist a malicious `:t` could produce a `$`-prefixed pseudo-field. +- `begins_with(path, :S)` — string-only. +- `contains(path, :S)` — string-only. +- `path :S` for any comparator — string operands are stored + verbatim, lex order matches wire order. +- `path = :B` / `path <> :B` — binary equality only. Ordering + comparators on binary are refused because the compiler stores B as + base64 strings inside `item_data`, and base64 lex order diverges from + bytewise order across mismatched lengths. +- `path = / <> :BOOL` and `path = / <> :NULL` — value-only. +- `AND` / `OR` — pushable iff both children are pushable (all-or-nothing; + cherry-picking would confuse composition semantics). +- `NOT attribute_exists(path)` / `NOT attribute_not_exists(path)` — the + only pushable `NOT` forms. Anywhere else, MongoDB's `$nor` semantics + on missing paths diverge from DDB's three-valued logic. + +Not pushable: any operand of type `N` (numbers stored as strings; `"10" +> "9"` is false lex-wise), `size` (MongoDB has no UTF-16 code-unit +count), `BETWEEN` / `IN` (pending proptest coverage; the compiler emits +them, the analyzer refuses them), and `NOT` around anything else. + +Property tests (`tests/pushdown_parity.rs`) generate random items and +expressions, compile the filter, and check that a pure-Rust BSON +interpreter and `evaluate_condition` agree on match/no-match — the +regression harness that lets the analyzer's certification be extended +safely. + +### 5.3 Query and Scan + +**Query key mapping.** +- Partition-key equality: `{ pk: }`. +- Sort-key conditions map to typed filters on `sk_s` / `sk_n` / `sk_b`. +- `BETWEEN` with `low > high` is rejected at the storage boundary with a + `ValidationException`. +- `begins_with(:S)` emits `{ sk_s: { $gte: prefix, $lt: next_string_prefix(prefix) } }`. + `next_string_prefix` computes the exclusive upper bound by incrementing + the rightmost non-`char::MAX` code point (skipping the surrogate gap + via `char::from_u32` retry); if the entire prefix is `char::MAX` it + returns `None` and the caller emits only the `$gte` bound. The + earlier `prefix + char::MAX` scheme excluded stored strings equal to + `s + char::MAX` (or extending past it) that DDB matches. +- `begins_with(:B)` emits the same range shape on the hex-encoded sort + key: `{ sk_b: { $gte: hex(prefix), $lt: hex(increment_bytes(prefix)) } }`. + +**Pagination.** `ExclusiveStartKey` **merges** into the existing sort-key +predicate rather than replacing it. Base-table Query paginates on a +single `$gt` / `$lt` sort-key comparison. Naively inserting +`filter.insert(sk, {$gt: cursor})` drops the caller's original +`BETWEEN` / `begins_with` bound and returns items outside it on page +2+. The merge covers three shapes: + +- No existing sk predicate → insert cursor bound. +- Existing operator map (`{ $gte: X, $lt: Y }`) → merge the cursor bound + into the map. +- Existing equality (`sk = X`) → wrap both under `$and`. + +**Index Query and Scan cursors.** Index-key values are non-unique, so +pagination cannot rely on `(pk, sk)` alone — items with duplicate index +keys would form an unstable page boundary. Instead, index queries +paginate on the compound tuple `(index_sk?, base_pk, base_sk?)` +expressed as a lexicographic `$or`: + +``` +(a > A) OR (a == A AND b > B) OR (a == A AND b == B AND c > C) +``` + +reversed to `$lt` for descending scans. Index Scan paginates on +`(pk, sk?, base_pk, base_sk?)` (index Scan lacks the partition-key +equality that Query has). Sort direction is applied to the whole +tuple so pagination is deterministic across items sharing an index-key +value. `LastEvaluatedKey` carries both the index-key and base-key +components so the next page's `ExclusiveStartKey` can rehydrate the +cursor. + +**Scan** uses lazy cursor iteration and stops when either `limit + 1` +in-segment items are accumulated or the cursor exhausts. It does not +impose a server-side hard limit. `Parallel Scan` filters items in the +application via `crc32(pk) % TotalSegments == Segment`. A hard +`(limit + 1) * TotalSegments` limit combined with post-fetch filtering +silently drops items under hot-key skew — an entire limit window can +land in one segment, terminating the scan with the others empty. +MongoDB batches under the hood (~101 docs), so lazy iteration is +efficient even without a hard limit — at most one extra network batch +beyond what is returned. + +Implementation: `data_engine.rs::query_impl`, `scan_impl`, +`build_sk_filter`, `next_string_prefix`, `increment_bytes`. + +### 5.4 GSI propagation (synchronous inline + async backfill) + +**Live writes** synchronize GSIs in the same session as the base write. +`sync_indexes_in_session` walks the `indexes` catalog for the table_id, +and for each index: + +1. If the old item had the index-key attributes, project it into the + index shape (`project_item`, respecting the `Projection` setting) and + run `delete_one` filtered on both index-key AND base-key components + (`index_entry_filter`). Filtering on index keys alone would delete + every base item's entry sharing those keys. +2. If the new item has the index-key attributes, project and upsert into + the index collection (`index_document` + `replace_one` with + `upsert: true`). + +The `gsi_cache` on `MongoEngine` (`DashMap`) +short-circuits the catalog walk when we know the table has no indexes. +Cache entries expire after `GSI_CACHE_TTL` (60s) so out-of-band GSI +changes on other ExtendDB instances converge within the window. + +**Async backfill.** `UpdateTable` GSI-create inserts the catalog +document with `index_status: "CREATING"` and pre-creates the mongo +index-collection + its compound query index (so live reads on the +CREATING index don't run collection scans). A background +`gsi_backfill_worker` (spawned in `MongoRuntimeHooks::spawn_workers`) +runs every 5 seconds: + +1. `find { index_status: "CREATING", index_type: "GSI" }` on the + `indexes` catalog collection. +2. For each job, read the base collection in batches of 500 items + (`backfill_gsi_batch`) starting from the row's persistent + `backfill_cursor` field. +3. Upsert projected items into the index collection. +4. After every batch, persist `backfill_cursor` back to the catalog + document so a mid-backfill server restart resumes where it left off. +5. When a batch returns fewer docs than the batch size (base fully + scanned), flip the catalog row to + `index_status: "ACTIVE"` and unset `backfill_cursor`. + +Live writes during the backfill window continue to hit `sync_indexes_in_session`, +which writes to CREATING indexes too — index-catalog membership, not +status, is what gates the write path. All writes are upserts on the +same `_id` shape, so a base item touched by both paths converges +regardless of interleaving. + +**Index-key input validation.** `validate_index_keys_for_item` rejects +wrong-type or empty index-key attributes on the item **before** any +write work (post-apply for `UpdateItem`). Without this, +`index_document` would silently skip the typed `sk_?` field when it +sees a type mismatch, leaving the resulting index row un-locatable for +subsequent deletes. Inside `TransactWriteItems`, the failure surfaces +as a per-item `CancellationReason::ValidationError` rather than a +top-level `ValidationException`. + +### 5.5 DynamoDB Streams + +Stream records are written inline during data operations, using the same +storage model as the PostgreSQL backend. This design gives ExtendDB full +control over sequence numbers, shard assignment, and retention — all of +which the DynamoDB Streams API contract tightly specifies. Native +MongoDB Change Streams are not used. + +**Shard model.** Four shards per stream-enabled table, created at +`CreateTable` (or on the first `UpdateTable` stream-enable). Shard IDs +embed the table's UUID: `shardId-{table_id}-{index:012}`. Table names +are only unique per account, so a name-derived scheme would allow +cross-tenant shard-id collisions on same-named tables. `table_id` is +per-instance; a `DeleteTable + CreateTable` sequence produces fresh +shard_ids, and `cleanup_stream_state_for_table` in `delete_table_impl` +removes the deleted table's shards, records, and counters so nothing +resurfaces on recreation. + +**Write path** (`write_stream_inline_in_session`): resolve the event +type from `(old_item, new_item)` presence, build key + old-image + +new-image per `StreamViewType`, hash the pk with CRC32 to select a +shard, draw the next sequence number, insert the record. Both shard +resolution and sequence-number assignment run inside the same session +as the data write. + +**Session-scoped sequence numbers.** `next_sequence_number_in_session` +does `find_one_and_update` with `$inc` on the per-shard counter +document — inside the write session. Without this, a fast writer B +could draw seq=6 and commit before a slow writer A (which drew seq=5) +commits; a consumer polling between B's commit and A's commit sees +seq=6 and advances past it, so when A finally commits, seq=5 lands +behind the cursor and is never returned. Session-scoped assignment +also serializes concurrent writers on the same shard: two `$inc`s +racing under snapshot isolation conflict at commit, and the loser +retries. + +**Per-shard counters.** Counter documents are keyed by +`_id: "stream_seq:"`. A single global counter would couple +the sequence spaces of unrelated shards, so a writer pushing records +into shard B would advance the counter shard A reads back — producing +non-contiguous sequence numbers on shard A's `GetRecords` pages. + +**Event names.** `event_name_ddb_str` emits DynamoDB wire casing +(`INSERT`, `MODIFY`, `REMOVE`). When `UpdateItem` creates an item +(upsert with no pre-image), the stream layer emits an `INSERT`, not a +`MODIFY` with a fabricated key-only `OldImage`. + +**Retention.** 24 hours, enforced by a TTL index on +`stream_records.created_at`. A `stream_record_cleanup_worker` (hourly) +provides defense in depth. + +**`GetRecords` path.** `{ shard_id: , sequence_number: { $gt: after } }` +with ascending sort, backed by the compound index +`(shard_id, sequence_number)`. + +**Non-session `write_stream_record`.** The `StreamEngine::write_stream_record` +trait method is a stub that returns an explicit error — the mongo backend +has no callers for it, and enrolling in the wrong or no session would let +a stream record commit while its base-table write rolls back. + +**`UpdateTable` stream-enable is idempotent.** If shards already exist +for the table, it reuses them and preserves the existing `stream_label`; +only a first-time enable rotates it. A repeat `UpdateTable` +`{ StreamEnabled: true }` would otherwise duplicate the shard set and +invalidate stream ARNs previously handed out to consumers. + +**`stream_label` format.** `YYYY-MM-DDThh:mm:ss` (second precision, no +timezone). Byte-for-byte compatible with the PostgreSQL backend so an +ARN issued by one backend is parseable by tooling that only ever saw +the other. See `format_stream_label` in `table_engine.rs`. + +### 5.6 Write conflict handling + +**Transient-conflict detection.** `is_transient_write_conflict` returns +true for any of: the `TransientTransactionError` label, the +`UnknownTransactionCommitResult` label, or a raw `WriteConflict` (code +112). Under snapshot isolation these all mean "your write lost to a +concurrent writer; retry the whole transaction." + +**Retry loop.** Session-scoped writes (Put / Delete / Update / TWI) wrap +the transaction body in a `for attempt in 0..TRANSIENT_RETRY_ATTEMPTS` +loop (50 attempts). Each attempt starts a fresh transaction, runs the +body, and either commits, aborts and retries (transient), or aborts and +returns (fatal). Retries sleep with jittered exponential backoff +(`backoff_sleep`, base 50 µs). + +**UpdateItem's OCC guard on top.** Even inside the transaction snapshot, +`UpdateItem` uses a `_v` version filter. The transaction guarantees the +snapshot the update was computed from; the versioned replace_one +guarantees the write only commits if the row's `_v` still matches what +we read. If `matched_count == 0` the attempt returns `Stale` (a distinct +signal from `Transient`) and the loop restarts. The native fast path +always emits `$inc: {_v: 1}` so a concurrent slow-path update racing +against a stale snapshot fails its filter and retries. + +**Exhaustion behavior.** Single-item retry exhaustion returns +`StorageError::Internal` (rare in practice; the retry ceiling is high). +`TransactWriteItems` exhaustion surfaces as +`StorageError::TransactionCanceled` with a synthetic per-op +`TransactionConflict` reason so wire consumers see the DDB-canonical +error string instead of a bare HTTP 500. + +**Conditional insert races.** A conditional PutItem on a nonexistent key +that raced against a concurrent inserter can manifest as either an +E11000 duplicate-key (unique-index race) or a WriteConflict (snapshot +race). The write path maps E11000 to `ConditionFailed` after +re-reading the winner outside the session; WriteConflict falls through +the normal retry loop, and the retry re-reads and sees the winner via +the existing-doc branch. + +### 5.7 TTL + +Two TTL surfaces: + +**Storage-native TTL indexes** — configured at bootstrap: +- `idempotency_tokens.created_at` — 540s (§4.6 for the rationale). +- `stream_records.created_at` — 24 h. +- `iam_sessions.expires_at` — `expireAfterSeconds: 0`. + +**Application-level DynamoDB TTL** — user-configured `TimeToLive` +attribute per table. MongoDB's native TTL runs at the storage engine +and cannot emit ExtendDB stream records with the required `Service` +user identity, so the backend maintains its own worker. + +`update_ttl` sets `ttl_attribute` on the table doc. `create_ttl_index` +creates a sparse index on `item_data.{ttl_attribute}.N` and flips +`ttl_index_ready: true`. The `ttl_cleanup_worker` (60s cadence) walks +tables with `ttl_index_ready`, finds expired items in batches of 100 +per table, and issues `DataEngine::delete_item` with a re-check +condition (`attribute_exists(ttl) AND ttl <= now`) to prevent races +with concurrent writes. The delete carries a `StreamCapture` with +`UserIdentity { identity_type: "Service", principal_id: "dynamodb.amazonaws.com" }` +so the stream record matches DynamoDB's format. + +### 5.8 Backups + +`CreateBackup` snapshots the source table by running a server-side +aggregation pipeline `[{ $out: "_backup_" }]` on the data +collection. `$out` writes the target collection server-side without +per-item traffic between the driver and the server. The destination +name is derived from a UUID because the caller-visible `backup_arn` +contains characters (`:`, `/`) that MongoDB does not allow in +collection names. + +`RestoreTableFromBackup` recreates the target table via +`CreateTable` (preserving `TableClass` / `SSESpecification` / +`OnDemandThroughput` from the backup metadata), then clones the backup +collection into the new data collection with the same `$out` pipeline +in reverse. + +`DeleteBackup` drops the physical collection using `backup_id` from +the metadata document and marks the metadata row `DELETED`. + +Implementation: `backup_engine.rs`. + +### 5.9 Account ID validation + +Injection defense on all account-scoped operations (`validate_account_id` +in `lib.rs`). Reject `$` (operator injection), `.` (field-path +traversal), null bytes, and non-ASCII. Runs before any query +construction. + +### 5.10 Catalog version check + +`read_catalog_version` returns the `settings.catalog_version` value; +`expected_catalog_version` returns the compiled-in `0.0.2`. The bin +layer compares them on `extenddb serve` startup — same pattern as the +PostgreSQL backend. + +## 6. Crate Structure + +``` +crates/storage-mongodb/ +├── Cargo.toml +└── src/ + ├── lib.rs # MongoEngine, GSI cache, inventory registrations + ├── config.rs # MongoStorageConfig + ├── operations.rs # OperationsEngine (CLI): connection parsing, redaction + ├── bootstrapper.rs # init / destroy / migrations + ├── table_engine.rs # CreateTable, UpdateTable, DeleteTable, DescribeTable + ├── data_engine.rs # PutItem, GetItem, DeleteItem, UpdateItem, Query, Scan, Transactions, pushdown fast path + ├── data/mod.rs # composite_id, item_to_document, index_document, binary_sk_to_hex + ├── condition.rs # DDB condition Expr → MongoDB filter compiler + ├── pushdown.rs # is_pushable analyzer — pushdown correctness boundary + ├── stream_engine.rs # Shard management, sequence numbers, GetRecords + ├── metadata_engine.rs # TTL configuration, tags, table size bookkeeping + ├── ttl_worker.rs # TTL sweep, stream record cleanup, GSI backfill workers + ├── backup_engine.rs # $out-based backup and restore + ├── management_store.rs # IAM CRUD, settings, metrics, rate limiting + ├── authorization_store.rs # Policy fetching for auth decisions + ├── credential_store.rs # Access-key lookup + AES-GCM decryption + ├── catalog_store.rs # SettingsStore / DiagnosticsStore glue + ├── admin_store.rs # Admin operations (currently thin) + └── worker_store.rs # WorkerStore: CREATING -> ACTIVE table transitions +``` + +## 7. `MongoEngine` Struct + +```rust +pub struct MongoEngine { + client: mongodb::Client, + catalog_db: mongodb::Database, + data_db: mongodb::Database, + region: String, + max_connections: u32, + gsi_cache: dashmap::DashMap, +} + +const GSI_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(60); +``` + +`MongoEngine::new` parses the connection string, rejects non-primary +read preferences, then constructs the client with `max_pool_size = +max_connections`. `catalog_db` and `data_db` are lightweight handles +against the single shared client. + +`gsi_cache` entries carry the observation time so a stale entry +(`elapsed() > GSI_CACHE_TTL`) is treated as a miss and re-read from the +catalog. This keeps writes correct when a GSI is added or dropped on +another ExtendDB instance sharing the catalog. + +## 8. Configuration + +```toml +[storage.mongodb] +connection_string = "mongodb://localhost:27017/?replicaSet=rs0" +max_connections = 50 +max_catalog_connections = 20 +``` + +```rust +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MongoStorageConfig { + pub connection_string: String, + #[serde(default = "default_max_connections")] + pub max_connections: u32, + #[serde(default = "default_max_catalog_connections")] + pub max_catalog_connections: u32, +} +``` + +The connection string must use `readPreference=primary` (the driver +default). Persisting the connection string in +`settings.data_connection_string` uses the raw string as-is; the bin +layer redacts the password from it for display via the +`OperationsEngine::redact_connection_string` hook. + +## 9. Bootstrapper Flow + +**`extenddb init`:** + +1. Materialize both databases with sentinel collections (MongoDB creates + databases lazily on first write, so we explicitly create + `schema_history` in the catalog db and `idempotency_tokens` in the + data db). +2. `run_catalog_migrations` — create the 17 catalog collections and + their indexes; seed `settings.catalog_version = 0.0.2`; record + the migration in `schema_history`. +3. `create_data_db` — create `idempotency_tokens` (TTL + unique index), + `stream_shards` (unique on shard_id), `stream_records` (TTL + compound + query index). +4. `bootstrap_encryption_key` — generate a random 256-bit key, + base64-encode, insert into `settings` (idempotent — E11000 races are + silently ignored). +5. `bootstrap_default_account` — insert `default` account if none + exists. +6. `bootstrap_admin_user` — bcrypt-hash the password, insert into + `admin_users`. +7. `record_data_connection` — record `data_database_name` and + `data_connection_string` in `settings`. + +**`extenddb destroy`:** drop both databases. + +## 10. Inventory Registrations + +Six `inventory::submit!` blocks in `lib.rs`: + +- `BackendRegistration` — factory for `MongoBootstrapper`, called by + `extenddb init`. +- `OperationsEngineRegistration` — CLI operations (connection parsing, + redaction, identifier validation, sensitive-key detection). +- `StorageConfigRegistration` — TOML deserializer for + `[storage.mongodb]`. +- `SettingsStoreRegistration` — factory for `MongoCatalogStore` acting + as `SettingsStore`. +- `DiagnosticsStoreRegistration` — factory for `MongoCatalogStore` + acting as `DiagnosticsStore`. +- `ServerComponentsRegistration` — factory called by `extenddb serve` + that constructs `MongoEngine`, `MongoCatalogStore`, + `MongoCredentialStore`, and `MongoRuntimeHooks` (which spawns the + TTL, stream-cleanup, and GSI-backfill workers). + +No changes to `crates/server/`, `crates/engine/`, or `crates/auth/` +are required — everything flows through the plugin registration +system. + +## 11. Dependencies + +```toml +[dependencies] +mongodb.workspace = true # 3.x, async, tokio-runtime +bson.workspace = true +dashmap.workspace = true # GSI existence cache +tokio.workspace = true +async-trait.workspace = true +futures.workspace = true +serde.workspace = true +serde_json.workspace = true +toml.workspace = true +tracing.workspace = true +time.workspace = true +uuid.workspace = true +base64.workspace = true +rand.workspace = true +bcrypt.workspace = true +aes-gcm.workspace = true +zeroize.workspace = true +thiserror.workspace = true +anyhow.workspace = true +inventory.workspace = true +extenddb-core.workspace = true +extenddb-storage.workspace = true +extenddb-auth.workspace = true +crc32fast.workspace = true + +[dev-dependencies] +proptest = "1" # pushdown parity harness +``` + +## 12. Feature coverage + +The backend implements every trait in `extenddb-storage`: + +**Data plane** +- `TableEngine` — Create/Delete/Describe/List/UpdateTable, including + GSI create with async backfill, LSI create, and idempotent + stream-enable on UpdateTable. +- `DataEngine` — PutItem, GetItem, DeleteItem, UpdateItem, Query, + Scan (including parallel scan), TransactGetItems, TransactWriteItems, + BatchGetItem, BatchWriteItem. Condition expressions run session- + scoped with an analyzer-gated pushdown fast path for a certified + subset. +- `StreamEngine` — session-scoped per-shard sequence numbers, four + shards per stream, CRC32 pk routing, TRIM_HORIZON / LATEST / + AT_SEQUENCE_NUMBER / AFTER_SEQUENCE_NUMBER iterators, 24h retention. +- `MetadataEngine` — TTL lifecycle, tags, table-size tracking. +- `BackupEngine` — CreateBackup, RestoreTableFromBackup, DeleteBackup + via server-side `$out` aggregation. + +**Control plane and catalog** +- `Bootstrapper` — init, destroy, migrate, verify. Creates the + catalog and data databases, seeds encryption key and admin user, + applies index schema. +- `WorkerStore` — `process_control_plane_transitions` flips tables + from `CREATING` to `ACTIVE` once their `status_transition_at` + passes. `create_table_impl` (and restore) write `CREATING` with a + scheduled transition when `control_plane_delay_seconds` > 0, or + `ACTIVE` directly when it is 0. `delete_table_impl` remains inline + (no `DELETING` transient state). +- `ManagementStore`, `AdminStore`, `SettingsStore`, `MetricsStore`, + `RateLimitStore` — the catalog trait surface. +- `AuthorizationStore` — user/group/role/permissions-boundary/session + policy lookup for IAM evaluation. +- `MongoCredentialStore` — SigV4 credential resolution with + AES-GCM-decrypted secret keys. + +**Background workers** — spawned from `MongoRuntimeHooks::spawn_workers`: +- `ttl_cleanup_worker` — sweep expired items every 60 s, emit + service-attributed stream records for the deletes. +- `stream_record_cleanup_worker` — hourly defense-in-depth for the + 24 h retention TTL index. +- `gsi_backfill_worker` — drain `indexes` rows in `CREATING` state, + scan the base collection with a persistent cursor, flip to ACTIVE. +- `control_plane_worker` — flip `tables` rows from `CREATING` to + `ACTIVE` once their scheduled `status_transition_at` passes. + +## 13. Testing Strategy + +- **Unit tests:** netstring `_id` encoding, hex sort-key ordering, + condition compiler, shard-id derivation, next_string_prefix, + Decimal128 rejection, index-doc key disambiguation. +- **Property tests:** `tests/pushdown_parity.rs` — random items and + expressions checked for agreement between the compiled BSON filter + and `evaluate_condition`. +- **Integration tests:** Single-node replica set in Docker + (`mongod --replSet rs0`), full trait coverage. +- **Existing pytest suite:** Passes unchanged (backend-agnostic wire + protocol tests). +- **CI:** GitHub Actions job with MongoDB 7.0 replica set, runs + `cargo test -p extenddb-storage-mongodb` then `devtools/run-tests + --extenddb --pytest --external`. + +## 14. Deployment Requirements + +- MongoDB **6.0+** in **replica set** mode. Standalone nodes reject + multi-document transactions. +- **`readPreference=primary`** on the connection string. Non-primary + is rejected at engine startup. +- Single-node replica set is fine for development / CI. Production: + 3-node replica set. +- Target scale: < 500 DynamoDB tables. At 500 tables with 2 GSIs each + (~1,500 collections), WiredTiger handles the count comfortably with + default settings. Ensure `ulimit -n ≥ 65536`. + +## 15. Design Decisions Summary + +| Decision | Choice | Rationale | +|----------|--------|-----------| +| Conditional writes | Read + evaluate + write inside a MongoDB transaction session | Snapshot atomicity delivers DDB's contract; pre-image reused for `ReturnValuesOnConditionCheckFailure = ALL_OLD` and `OldImage`. | +| Filter pushdown | Analyzer-gated fast path; `is_pushable` certifies a subset | Compiler in `condition.rs` handles broader syntax than production uses; the analyzer is the correctness boundary. | +| GSI live sync | Synchronous inline within the base write's session, gated by a 60s-TTL cache | Strongly-consistent GSI reads; no Change Stream recovery. | +| GSI async backfill | `CREATING` → `ACTIVE` via `gsi_backfill_worker` with persistent `backfill_cursor` | Matches DDB's async UpdateTable contract; restart-safe. | +| Index-doc identity | Composite `_id = netstring(idx_pk, idx_sk, base_pk, base_sk)` + `base_pk` / `base_sk_?` as first-class fields | Base-key disambiguation for non-unique index keys; compound cursor pagination without touching `item_data`. | +| Composite `_id` | Netstring `:,...` | Delimiter-free framing between pk and sk. | +| Binary sort keys | Stored as lowercase hex strings | BSON Binary comparison is length-first-then-content, diverging from DDB unsigned-lex byte order across mismatched lengths. | +| Sort key numbers | Native BSON `Decimal128`; values exceeding 34 sig-digits rejected | Correct numeric ordering; no silent precision loss. | +| DynamoDB Streams | Inline record writes to `stream_records` in the base write's session; per-shard sequence counters | Behavioral parity with PostgreSQL backend; per-shard monotonicity is a contract. | +| Sequence assignment | Inside the write session | Prevents ordering holes where a fast writer commits a higher seq before a slower earlier one does. | +| Stream shard ID | `shardId-{table_id}-{i:012}` | Cross-tenant isolation on same-named tables. | +| Stream retention | 24h TTL index on `stream_records.created_at` + hourly cleanup worker | Primary enforcement at storage; worker is defense in depth. | +| WriteConflict handling | Retry with jittered exponential backoff (50 attempts); TWI exhaustion → `TransactionCanceled` with synthetic `TransactionConflict` reasons | Bounded tail latency; DDB-canonical error surface. | +| UpdateItem concurrency | Snapshot txn + `_v` version filter + retry; native fast-path always `$inc: {_v: 1}` | Prevents lost updates; fast path stays safe against a concurrent slow path. | +| Idempotency retention | Unique `(account_id, token)` index + 540s TTL + 600 ms data-plane age filter | Race safety under snapshot isolation; ≤10-min worst-case retention regardless of TTL-monitor cadence. | +| Backups | Per-backup collection via server-side `$out` aggregation | No per-item driver traffic; metadata schema decoupled from collection name. | +| Parallel scan | Application-side `crc32(pk) % segments` + lazy cursor | Rare feature; server-side bucketing would tax every write. Lazy iteration prevents item-drops under hot-key skew. | +| Read preference | `primary` enforced at engine startup | `ConsistentRead=true` requires linearizable reads. | + +## 16. Performance Characteristics + +**Hot path — single-item conditional write.** One MongoDB transaction +session covers pre-image read, condition eval, base write, GSI sync, +stream insert (per-shard counter `$inc` + document insert). On a local +replica set, session overhead is ~sub-ms over a raw driver call. The +session is what buys the DDB atomicity contract — it is the +compatibility, not overhead. The pushdown fast path collapses this to +a single `find_one_and_*` for the certified subset on tables with no +GSIs / streams. + +**Unconditional single-item update, no GSIs, no streams.** Native +fast path: one `find_one_and_update` with compiled `$set` / `$unset` / +`$inc` outside any session. The `$inc: {_v: 1}` keeps the fast path +safe against a concurrent slow-path update. + +**GSI write overhead.** No GSIs: zero (cached). Has GSIs: one catalog +`find` (cached for subsequent writes on the same table until +`GSI_CACHE_TTL` elapses) + one upsert or delete per index per write, +all inside the base write's session. + +**Stream write overhead.** One counter `$inc` + one `stream_records` +insert per write, inside the base write's session. + +**Query / Scan.** Index lookups on `(pk, sk_?)` for base tables and +`(pk, sk_?, base_pk, base_sk_?)` for index queries. `GetRecords` +uses the compound `(shard_id, sequence_number)` index. + +**TransactWriteItems.** Multi-collection ACID transaction with +snapshot read concern and majority write concern; up to 100 operations +per the DDB spec. Retried on transient conflicts with jittered backoff. diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index 045484a6..f92f4222 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -8,10 +8,11 @@ adaptation when switching between ExtendDB and the real service. | Area | DynamoDB | ExtendDB | |------|----------|------| -| Storage backend | Proprietary distributed storage | PostgreSQL | +| Storage backend | Proprietary distributed storage | PostgreSQL (default) or MongoDB (feature flag) | | Global Tables | CreateGlobalTable, replication | Not implemented (returns UnknownOperationException) | | DAX (Accelerator) | In-memory caching layer | Not applicable | | PartiQL | ExecuteStatement, BatchExecuteStatement | Not implemented (returns UnknownOperationException) | +| Numeric precision on partition/sort keys (MongoDB backend only) | 38 significant digits | 34 significant digits (BSON Decimal128). Values that exceed this precision are rejected at write and query time with a ValidationException rather than silently downcast. PostgreSQL backend supports the full 38 digits. | ## Authentication and Authorization (AWS IAM/STS auth surface used by DynamoDB) diff --git a/docs/getting-started.md b/docs/getting-started.md index 369c9ae7..40721731 100755 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -29,7 +29,9 @@ software on your behalf. After the script completes, continue from ## Prerequisites -- PostgreSQL 14+ running locally (see `docs/local-postgres-setup.md`) +- **Storage backend** (one of): + - PostgreSQL 14+ running locally (see `docs/local-postgres-setup.md`) + - MongoDB 7.0+ with replica set (see `docs/local-mongodb-setup.md`) - Rust toolchain (1.88+) - AWS CLI v2 (for testing) - Python 3.10+ with virtual environment (see [Python Environment Setup](../README.md#python-environment-setup) in the README) @@ -37,7 +39,14 @@ software on your behalf. After the script completes, continue from ## 1. Build extenddb ```bash +# PostgreSQL backend (default) cargo build --release + +# MongoDB backend +cargo build --release --features mongodb + +# Both backends +cargo build --release --features postgres,mongodb ``` The binary is at `target/release/extenddb`. @@ -47,15 +56,17 @@ The binary is at `target/release/extenddb`. Run `extenddb init` to create the catalog and data databases: ```bash +# PostgreSQL (default) ./target/release/extenddb init + +# MongoDB +./target/release/extenddb init --backend mongodb ``` This will: -- Create a `extenddb` PostgreSQL user (if it doesn't exist) - Create the `extenddb_catalog` database (catalog metadata) -- Create the `extenddb` database (user item data) -- Run schema migrations -- Generate an AES-256-GCM encryption key (for future access key storage) +- Create the data database (user item data) +- Generate an AES-256-GCM encryption key (for access key storage) - Create a default account and print the account ID - Create an `admin` user and print the credentials once - Generate a self-signed TLS certificate at `~/.extenddb/tls/` diff --git a/docs/local-mongodb-setup.md b/docs/local-mongodb-setup.md new file mode 100644 index 00000000..d27b02a0 --- /dev/null +++ b/docs/local-mongodb-setup.md @@ -0,0 +1,191 @@ +# Local MongoDB Setup + +## Prerequisites + +- MongoDB 7.0+ (for multi-document transactions) +- A replica set configuration (required even for single-node deployments) + +## Installation + +### macOS (Homebrew) + +```bash +brew tap mongodb/brew +brew install mongodb-community@7.0 +``` + +### Linux (Ubuntu/Debian) + +```bash +curl -fsSL https://www.mongodb.org/static/pgp/server-7.0.asc | \ + sudo gpg -o /usr/share/keyrings/mongodb-server-7.0.gpg --dearmor +echo "deb [ signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] \ + https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | \ + sudo tee /etc/apt/sources.list.d/mongodb-org-7.0.list +sudo apt-get update && sudo apt-get install -y mongodb-org +``` + +### Docker (recommended for development) + +```bash +docker run -d --name extenddb-mongo \ + -p 27017:27017 \ + mongo:7 --replSet rs0 +``` + +## Replica Set Initialization + +MongoDB must run as a replica set for transactions and Change Streams. + +### Single-node replica set (development) + +```bash +# If using Docker: +docker exec extenddb-mongo mongosh --quiet --eval "rs.initiate()" + +# If using a local install: +mongosh --eval "rs.initiate()" +``` + +Wait a few seconds for the replica set to elect a primary, then verify: + +```bash +mongosh --eval "rs.status().ok" +# Should output: 1 +``` + +### Homebrew (macOS) with replica set + +Edit the MongoDB config to add replica set: + +```bash +# Find the config file +brew --prefix mongodb-community@7.0 +# Usually: /opt/homebrew/etc/mongod.conf +``` + +Add to `mongod.conf`: +```yaml +replication: + replSetName: rs0 +``` + +Restart and initiate: +```bash +brew services restart mongodb-community@7.0 +mongosh --eval "rs.initiate()" +``` + +## Connection Details + +| Setting | Value | +|---------|-------| +| Host | `localhost` | +| Port | `27017` | +| Replica set | `rs0` | +| Connection string | `mongodb://localhost:27017/?replicaSet=rs0` | + +For Docker on a non-default port: +``` +mongodb://localhost:27018/?replicaSet=rs0&directConnection=true +``` + +## Building with MongoDB Support + +The MongoDB backend is behind a feature flag: + +```bash +cargo build --release --features mongodb +``` + +To build with both backends: +```bash +cargo build --release --features postgres,mongodb +``` + +## Initializing ExtendDB with MongoDB + +```bash +./target/release/extenddb init --backend mongodb --config extenddb.toml +``` + +This creates: +- `extenddb_catalog` database (table metadata, IAM, settings) +- `extenddb_data` database (per-table item collections) +- Admin user credentials (printed to stdout) +- Self-signed TLS certificate at `~/.extenddb/tls/cert.pem` +- Config file `extenddb.toml` + +The generated config will contain: +```toml +[storage] +backend = "mongodb" + +[storage.mongodb] +connection_string = "mongodb://localhost:27017/?replicaSet=rs0" +``` + +## Starting the Server + +```bash +./target/release/extenddb serve --config extenddb.toml +``` + +## Config Mapping + +```toml +[storage] +backend = "mongodb" + +[storage.mongodb] +connection_string = "mongodb://localhost:27017/?replicaSet=rs0" +# max_pool_size = 20 +``` + +Or via environment variable: +```bash +export EXTENDDB__STORAGE__MONGODB__CONNECTION_STRING="mongodb://localhost:27017/?replicaSet=rs0" +``` + +## Verifying the Connection + +```bash +# Health check +curl --cacert ~/.extenddb/tls/cert.pem https://127.0.0.1:8000/health + +# List tables (should return empty) +aws dynamodb list-tables \ + --endpoint-url https://127.0.0.1:8000 \ + --region us-east-1 +``` + +## Differences from PostgreSQL Backend + +- **Replica set required:** Even single-node MongoDB must be configured as a replica set. +- **DynamoDB Streams:** Inline record writes with atomic sequence numbers. +- **GSI propagation:** Synchronous inline updates during data operations. +- **Concurrency model:** Optimistic versioning with retry on conflict (vs row-level locking in PostgreSQL). + +## Stopping + +```bash +# ExtendDB +./target/release/extenddb stop --config extenddb.toml + +# Docker MongoDB +docker stop extenddb-mongo + +# Homebrew MongoDB +brew services stop mongodb-community@7.0 +``` + +--- + +## License + +Copyright 2026 ExtendDB contributors. Licensed under the Apache License, Version 2.0. +See [LICENSE](../LICENSE) for the full text. + +This software is provided "as is" without warranty of any kind. ExtendDB is not +affiliated with, endorsed by, or sponsored by Amazon Web Services. "DynamoDB" is a trademark +of Amazon.com, Inc. diff --git a/docs/rfcs/0000-mongodb-backend.md b/docs/rfcs/0000-mongodb-backend.md new file mode 100644 index 00000000..248160b8 --- /dev/null +++ b/docs/rfcs/0000-mongodb-backend.md @@ -0,0 +1,305 @@ +# RFC-206: MongoDB Storage Backend + +- Status: Draft +- Author: @diegotoledano95 +- Created: 2026-07-08 +- Tracking issue: #206 + +## Summary + +This RFC proposes adding MongoDB as a backend for ExtendDB. The goal is to let developers run DynamoDB-compatible workloads on MongoDB while preserving ExtendDB's core value: a DynamoDB-compatible API over multiple storage backends. The implementation covers all mandatory traits defined in RFC-0002 and all optional traits, uses ExtendDB's existing `inventory`-based plugin registration system without modifying the server or engine layers, and is maintained by the MongoDB team who commit to ongoing ownership of the backend crate. + +## Motivation + +ExtendDB's core premise is DynamoDB API compatibility over multiple storage backends. The initial reference PostgreSQL backend demonstrates the feasibility of this approach while opening the opportunity for other databases to participate. + +MongoDB is a natural fit as an additional database target: data model alignment; high read/write throughput through horizontal scalability; infrastructure fit. + +DynamoDB and MongoDB share the same data model approach — documents stored as schema-less JSON-like data. MongoDB's document model maps directly to the approach taken by DynamoDB with each item stored as a MongoDB BSON document with no impedance mismatch at the data model level. Unlike relational databases, the translation from JSON to BSON is direct without complicated relational mapping techniques required. + +Customers evaluating DynamoDB and MongoDB often consider scalability as a key requirement. ExtendDB's deployment approach requiring high write throughput is matched by MongoDB's replica set model via horizontal scaling. High read and write throughput across multiple nodes is a core tenet of MongoDB and aligns naturally with the scalability requirement for an ExtendDB customer. + +Organizations running ExtendDB, DynamoDB, and MongoDB have already evaluated the usefulness of a non-relational database approach. These shared customers do not want to run PostgreSQL or other relational databases solely for DynamoDB compatibility. Rather, taking advantage of the infrastructure they already run that aligns with the document model design and scalability requirements they require makes MongoDB a natural fit. + +## Detailed design + +### Scope + +This RFC proposes: + +- A new optional `extenddb-storage-mongodb` crate at `crates/storage-mongodb/` +- A `mongodb` Cargo feature flag on the `extenddb` binary crate +- Backend registration through the existing `inventory`-based plugin system without changes to `crates/engine/`, `crates/server/`, `crates/auth/`, or `crates/core/` +- MongoDB-specific implementations of all mandatory and optional storage traits defined in RFC-0002 +- Setup documentation and sample configuration for MongoDB deployments + +This RFC does not propose: + +- Changes to the DynamoDB wire protocol or API response shapes +- MongoDB Atlas, Atlas Data API, or any hosted MongoDB service as a target +- Sharded cluster support (replica sets only) +- Changing the default backend from PostgreSQL +- Dual-write or online migration from PostgreSQL to MongoDB +- A generic document-store abstraction shared with future document database backends + +### Repository structure + +The backend lives at `crates/storage-mongodb/` in the main ExtendDB repository, following the mono-repo structure prescribed by RFC-0002. It is selected at build time via a `mongodb` Cargo feature flag on the `extenddb` binary crate. + +Feature flag definition: `crates/bin/Cargo.toml` — `[features]` section defines `mongodb = ["extenddb-storage-mongodb"]` with the crate as an optional dependency. The `postgres` feature remains the default. Both features can be enabled together to compile a binary supporting both backends. + +### Plugin registration + +The backend registers itself with ExtendDB's `inventory`-based plugin system without modifying the server, engine, or auth layers. Six `inventory::submit!` calls in `lib.rs` register the backend for: operations engine (CLI commands), bootstrapping (`extenddb init`), config parsing, settings store access, diagnostics store access, and server component construction (`extenddb serve`). + +All registration blocks live in `crates/storage-mongodb/src/lib.rs`. The `ServerComponentsRegistration` block is the critical one — it is the factory function called when `extenddb serve --backend mongodb` is run. No changes are required in `crates/server/`, `crates/engine/`, or `crates/auth/`. + +### Database layout + +The backend uses two MongoDB databases: + +**`extenddb_catalog`** — metadata and management. Created on `extenddb init`. Contains collections for table definitions (`tables`), index metadata (`indexes`), accounts, tags, admin users, IAM users, groups, roles, access keys, IAM sessions, policies, permissions boundaries, settings, metrics, login attempts, backup metadata, continuous backup state, and schema migration history. + +**`extenddb_data`** — item data. One MongoDB collection per DynamoDB table, named `_ddb_{table_id}`. One additional collection per GSI/LSI, named `_ddb_{index_id}`. Shared collections: `stream_records` and `stream_shards` for DynamoDB Streams, `counters` for per-shard sequence-number counters, `idempotency_tokens` for transaction deduplication, and one `_backup_{backup_id}` collection per user-created backup. + +Catalog collection creation and index setup: `crates/storage-mongodb/src/bootstrapper.rs` — `run_catalog_migrations()`. Data-database setup (`idempotency_tokens`, `stream_shards`, `stream_records` and their indexes): `create_data_db()` in the same file. Collection naming: `data/mod.rs` — `data_collection_name()`, shared between base-table and index collections. + +### Document structure for DynamoDB items + +Each DynamoDB item is stored as a MongoDB document: + +``` +{ + _id: "", + pk: "partitionKeyValue", + sk_s: "sortKeyValue", // string sort keys + sk_n: Decimal128(...), // number sort keys, native BSON Decimal128 + sk_b: "aabb...", // binary sort keys, lowercase hex string + _v: NumberLong, // OCC version counter (present on updated docs) + item_data: { ... full DynamoDB item in DynamoDB JSON format ... } +} +``` + +The `_id` is a netstring-encoded composite key of the form `:,:,`. Netstring framing prevents the collision an ad-hoc `"{pk}#{sk}"` scheme suffers from when either component contains the delimiter (e.g. `pk="a#b", sk="c"` and `pk="a", sk="b#c"`). PK-only tables use the raw pk text as `_id`. + +Typed sort-key fields (`sk_s`, `sk_n`, `sk_b`) let MongoDB apply native range comparisons with correct ordering: + +- **String** sort keys use collection-level `{ locale: "simple" }` collation so range comparisons are byte-order, matching DynamoDB. +- **Numeric** sort keys use BSON `Decimal128`. Values whose precision exceeds Decimal128 (34 significant digits) are rejected at write and query time with a ValidationException rather than silently downcast. DynamoDB itself supports 38 significant digits; this is documented in `docs/differences-from-dynamodb.md`. +- **Binary** sort keys are stored as lowercase hex strings. BSON's native Binary comparison is length-first-then-content, which diverges from DynamoDB's unsigned-lex byte order (DynamoDB says `[0x01,0xFF] < [0x02]`; BSON Binary reverses that). Hex-encoded strings preserve DynamoDB byte order under MongoDB's default lexicographic string comparison and let `begins_with` use a plain `$gte` / `$lt` range filter. + +The full item is stored in `item_data` using DynamoDB's own type-tagged format (`{"S": "hello"}`, `{"N": "42"}`), preserving type information for non-key attributes. Item conversion helpers live in `data/mod.rs` (`item_to_document`, `document_to_item`, `composite_id`, `binary_sk_to_hex`). + +**Secondary-index documents** carry a superset of these fields. In addition to the index-key components (`pk`, `sk_?`), each index document also stores the base-table key attributes as first-class fields: `base_pk` (text) and `base_sk_s|n|b` (typed). The `_id` is a 4-tuple netstring `[idx_pk, idx_sk, base_pk, base_sk]`. GSI keys are non-unique across base items, so encoding the base key into `_id` gives each index entry a unique identity keyed to the base item it describes; without this, two base items sharing an index-key value would upsert to the same document and one would silently overwrite the other. The base-key fields also let index pagination form a compound cursor `(index_sk, base_pk, base_sk)` without traversing the JSON `item_data` payload. See `index_document()` and `index_entry_filter()` in `data/mod.rs`. + +### Condition expression evaluation + +Conditional writes (`ConditionExpression` on PutItem, DeleteItem, UpdateItem) run the condition read, evaluation, and write inside a MongoDB client session bound to a multi-document transaction. Within the session, the backend reads the current item, evaluates the DynamoDB condition in Rust against the loaded item (`extenddb_core::expression::evaluate_condition`), and issues the write on the same session. Read and write share the transaction's snapshot isolation, so a concurrent writer's changes either become visible to the condition (in which case the caller sees the same outcome as if the writes were serial) or trigger a WriteConflict at commit (which the write path retries — see Write conflict handling). + +This delivers DynamoDB's atomicity contract for conditional writes and matches `ReturnValuesOnConditionCheckFailure = ALL_OLD` semantics naturally: the item loaded to evaluate the condition is reused directly in the failure response. + +An optional filter-pushdown fast path (`pushdown.rs` + `condition.rs`) skips the session for a restricted subset of conditions on tables that have no GSIs and no stream capture. A compile-time analyzer (`is_pushable`) certifies that a condition's compiled MongoDB filter agrees with `evaluate_condition` on every item; when it says yes, the backend collapses read + check + write into a single `find_one_and_replace` / `find_one_and_delete` or replace with a merged key+condition filter. The analyzer is the correctness boundary: the compiler in `condition.rs` covers a broader syntax (numeric compare, sets, `IN`, `BETWEEN`, `size`, arbitrary `NOT`) than the analyzer certifies, and only the analyzer-approved subset ever reaches production. Anything else falls through to the session-scoped path, which is always authoritative. + +Session-scoped condition path: `data_engine.rs` — `put_item_impl`, `delete_item_impl`, `update_item_impl`, and each `OwnedTransactWriteOp` arm in `execute_transact_write_op_in_session`. Pushdown fast path: `delete_item_pushdown` and `update_item_pushdown` in the same file, gated on `is_pushable(cond, maps) == Yes && stream.is_none() && gsi_cache_get_fresh(table_id) == Some(false)`. + +### Query and Scan + +**Query** translates `KeyConditionExpression` to a MongoDB `find()` filter. Partition key equality maps to `{ pk: "" }`. Sort key conditions map to typed range filters on `sk_s`, `sk_n`, or `sk_b`. `BETWEEN` with `low > high` is rejected upfront with a ValidationException. `begins_with` on strings emits `{ $gte: prefix, $lt: next_string_prefix(prefix) }`, where the upper bound is the least string strictly greater than any prefix-starting string (built by incrementing the rightmost non-`char::MAX` code point). `begins_with` on binary emits the same range shape on the hex-encoded sort key. `ScanIndexForward: false` applies a descending sort. + +Pagination via `ExclusiveStartKey` **merges** the resume bound into the existing sort-key predicate rather than replacing it. Naively inserting `{sk: {$gt: cursor}}` drops the caller's original `BETWEEN` / `begins_with` bound and returns items outside it on page 2 and beyond. The merge covers three cases: no existing sk predicate (insert), existing operator map (merge into it), and existing equality (fall back to `$and`). See `query_impl` in `data_engine.rs`. + +**Index queries** paginate over a compound tuple `(index_sk?, base_pk, base_sk?)`. Index-key values are non-unique — duplicates fall through to the base-key tie-breaker. The cursor is expressed as a lexicographic `$or` of the form `(a > A) OR (a == A AND b > B) OR (a == A AND b == B AND c > C)` (reversed for descending). Sort direction is applied to the same compound tuple so ordering is deterministic across groups of items sharing index keys. `LastEvaluatedKey` carries both index-key and base-key components so the next page's `ExclusiveStartKey` resolves the compound cursor. + +**Scan** performs a full collection scan with lazy cursor iteration. Base-table scans paginate on `_id` (unique after netstring encoding). Index scans paginate on the same compound cursor as index Query. Filter expressions are evaluated after retrieval. + +**Parallel scan** (`Segment` / `TotalSegments`) filters items in the application via `crc32(pk) % TotalSegments == Segment`. Each segment scans the full collection. The scan loop streams the cursor and terminates when either `limit + 1` in-segment items are accumulated or the cursor exhausts, without imposing a server-side hard limit — a hard `limit * total_segments` cap combined with post-fetch segment filtering silently drops items under any hot-key skew. Pre-bucketing documents at write time would avoid the per-segment full scan but adds overhead to every write for a feature that is rarely used in practice. + +### Global and Local Secondary Indexes + +Each secondary index has its own MongoDB collection with a compound index on `(pk, sk_?, base_pk, base_sk_?)` (created by `create_index_data_collection` in `table_engine.rs`). String-sorted index columns use `simple` collation matching the query path. On writes that modify indexed attributes, the backend synchronizes the index collection in the same session as the base write: `sync_indexes_in_session` deletes the old projected entry (filtered on the full index-key + base-key tuple so duplicate index keys don't cross-delete) and upserts the new one. + +A `DashMap` in-memory cache on `MongoEngine` short-circuits the catalog lookup for tables known to have no indexes. Cache entries carry an insertion timestamp and expire after 60 seconds (`GSI_CACHE_TTL`), so out-of-band GSI changes on another ExtendDB instance converge within the TTL window. + +**Async GSI backfill.** `UpdateTable`'s GSI-create path writes the catalog document with `index_status: "CREATING"` and pre-creates the mongo collection and its query index; a background `gsi_backfill_worker` (in `ttl_worker.rs`) discovers `CREATING` rows, iterates the base collection in batches keyed by a persistent `backfill_cursor` field on the index document, upserts projected items into the index collection via `backfill_gsi_batch`, and flips the status to `ACTIVE` when the base is fully scanned. The cursor is persisted between batches so a mid-backfill server restart resumes where it left off. Live writes during the backfill window continue to route through `sync_indexes_in_session`, which writes to CREATING indexes too — all writes are upserts on the same `_id` shape, so a base item touched by both paths converges regardless of interleaving. + +Before every put and update, `validate_index_keys_for_item` rejects wrong-type or empty index-key attributes as a top-level `ValidationException` (or a per-item `CancellationReason` inside `TransactWriteItems`). Without this check, a mismatched-type index-key attribute would be silently dropped from the index doc, leaving the row un-locatable for subsequent deletes. + +### Transactions + +`TransactWriteItems` runs all operations inside a single MongoDB multi-document ACID transaction with snapshot read concern and majority write concern. Each operation's condition evaluation, base-row write, GSI synchronization, and stream record insert happens on the same `ClientSession` (`sync_indexes_in_session` + `write_stream_inline_in_session` are called inline from each `OwnedTransactWriteOp` arm). Without this, a transactional write to a GSI-bearing or streams-enabled table would commit the base row while silently dropping its dependent side effects. + +Idempotency tokens live in the `idempotency_tokens` collection in `extenddb_data`. A unique compound index on `(account_id, token)` catches races between concurrent transacts under snapshot isolation — an inserter that races through the pre-check gets an `E11000` on insert, resolved by re-reading the winner and returning `IdempotentReplay` (fingerprints match) or `IdempotentMismatch` (fingerprints differ). Retention is enforced by a 540-second MongoDB TTL index plus a data-plane age filter (`created_at` within 600 000 ms) so worst-case retention stays ≤10 minutes regardless of the TTL monitor's ~60s cadence. + +`TransactGetItems` performs a consistent snapshot read using a `ClientSession` with snapshot read concern. + +Transaction implementation: `data_engine.rs` — `transact_write_items_impl`, `transact_get_items_impl`, `execute_transact_write_op_in_session`. + +### Write conflict handling + +**Session-scoped writes** (PutItem, DeleteItem, UpdateItem, TransactWriteItems) detect transient MongoDB conflicts via `is_transient_write_conflict`, which returns true for any of: the `TransientTransactionError` label, the `UnknownTransactionCommitResult` label, or a raw `WriteConflict` (code 112). Conflicts trigger a retry loop with jittered exponential backoff (`backoff_sleep`, base 50 µs) up to `TRANSIENT_RETRY_ATTEMPTS` (50). Exhausted retries on single-item operations return a `StorageError::Internal`; on `TransactWriteItems`, they surface as a `TransactionCanceled` with a synthetic per-op `TransactionConflict` cancellation reason so wire consumers see the DDB-canonical error string instead of a bare HTTP 500. + +**UpdateItem** additionally uses an optimistic-concurrency version guard on top of the transaction. A `_v` counter is stored on each document. The write path reads the current `_v` under the transaction snapshot, applies the update expression in memory, sets `_v = current_version + 1`, then executes `replace_one` filtered on both the primary key AND the expected `_v`. If `matched_count == 0`, a concurrent writer committed a higher version between the snapshot read and the replace; the attempt aborts and the outer retry loop re-reads. The version guard doubles up when a native-fast-path update (unconditional, no stream, no GSI) is possible — that path uses a single `find_one_and_update` with `$inc: {_v: 1}` outside a transaction, and always bumps the counter so a concurrent session-scoped update against a stale snapshot fails its versioned filter and retries. + +**PutItem** with an existence guard on a new document maps duplicate-key errors (`E11000`) to `ConditionFailed` after re-reading the winner. This is the runtime signature of a conditional-put race the transaction snapshot didn't see. + +### DynamoDB Streams + +DynamoDB Streams are implemented using explicit stream record storage in MongoDB collections, not MongoDB's native Change Streams feature. The explicit approach maintains behavioral parity with the PostgreSQL backend and retains full application control over sequence-number generation, shard assignment, and record retention — all of which the DynamoDB Streams API contract tightly specifies. + +Each stream-enabled table is assigned 4 shards at creation time. Shard identifiers embed the table's globally-unique `table_id` UUID rather than the caller-visible `table_name` (`build_shard_id` in `stream_engine.rs`): `shardId-{table_id}-{i:012}`. Table names are only unique per-account, so a name-derived shard_id would let one account's `GetRecords` read another's records on same-named tables. `table_id` is per-instance, so a `DeleteTable + CreateTable` sequence produces fresh shard_ids; leftover stream records from the deleted table are cleaned up in `delete_table_impl` (`cleanup_stream_state_for_table`). A unique index on `stream_shards.shard_id` rules out duplicate insertions structurally. + +On each data write with streams enabled, `write_stream_inline_in_session` runs inside the same session as the base write: + +1. Resolve the shard for the item's partition key by reading the table's shard set under the session (`assign_shard_in_session`) and hashing the pk with CRC32. +2. Draw the next sequence number by `$inc`-ing the per-shard counter at `_id: "stream_seq:"` in the `counters` collection — also under the session. +3. Insert the stream record into `stream_records`. + +Per-shard counters preserve DynamoDB Streams' contract that sequence numbers are strictly monotonic within a shard and independent across shards. A single global counter would couple unrelated shards' sequence spaces. Session-scoped assignment closes an ordering hole: without it, a fast writer B can draw seq=6 and commit before a slow writer A (which drew seq=5) commits, and a consumer polling between B's commit and A's commit would advance past seq=6 and never see seq=5. With the counter increment inside the write transaction, two writers racing on the same shard conflict at commit time and the loser retries. + +Stream event names use DynamoDB wire casing (`INSERT`, `MODIFY`, `REMOVE`) via `event_name_ddb_str`. When `UpdateItem` creates an item that didn't exist (upsert case), the stream layer emits `INSERT`, not `MODIFY` with a fabricated key-only `OldImage`. + +`GetRecords` paginates using `{ sequence_number: { $gt: after } }` range queries with ascending sort, backed by a compound index on `(shard_id, sequence_number)`. Retention is 24 hours: a TTL index on `stream_records.created_at` (24 h) drives primary enforcement; a background `stream_record_cleanup_worker` runs hourly as defense in depth. `UpdateTable` stream-enable is idempotent: if shards already exist for the table, it reuses them and preserves the existing `stream_label` rather than rotating it (which would invalidate ARNs previously handed out to consumers). `stream_label` uses `YYYY-MM-DDThh:mm:ss` (second precision, no timezone), byte-for-byte compatible with the PostgreSQL backend. + +The `StreamEngine::write_stream_record` trait method is not used on this backend; it returns an explicit error so a caller who invokes it doesn't get a subtly-wrong write outside any transaction session. + +### Time to Live (TTL) + +When TTL is enabled on a table, the backend creates a sparse MongoDB index on `item_data.{ttl_attribute}.N` and marks `ttl_index_ready: true` on the table doc. A background TTL worker (spawned at server startup by `MongoRuntimeHooks::spawn_workers`) sweeps expired items every 60 seconds in batches of 100 per table. Each deletion goes through `DataEngine::delete_item` with a condition expression re-checking expiry, preventing races with concurrent writes. TTL deletions carry `UserIdentity { type: "Service", principalId: "dynamodb.amazonaws.com" }` on their stream records, matching DynamoDB's TTL stream record format. + +TTL index creation: `metadata_engine.rs` — `create_ttl_index`. Background worker and stream/GSI companions: `ttl_worker.rs` — `ttl_cleanup_worker`, `stream_record_cleanup_worker`, `gsi_backfill_worker`. Worker spawn: `lib.rs` — `MongoRuntimeHooks::spawn_workers`. + +### Control plane state transitions + +`CreateTable` (and `RestoreTableFromBackup`) write the catalog row and create the data collection with its indexes before returning. When `control_plane_delay_seconds` > 0 (the default is 0.25) the row is written with `TableStatus: CREATING` and a `status_transition_at` timestamp, and the returned `TableDescription` carries `TableStatus: CREATING`; a background `control_plane_worker` (`ttl_worker.rs`) flips the row to `ACTIVE` once the transition time passes. During the window, data-plane operations against the table return `ResourceNotFoundException`, matching DynamoDB and the PostgreSQL backend. When `control_plane_delay_seconds` is 0, the row is written `ACTIVE` directly and the worker has nothing to do. `DeleteTable` remains inline: it removes the catalog row, drops the data + index collections, deletes tags, and cleans up stream shards / records / counters via `cleanup_stream_state_for_table` (`table_engine.rs`), all before returning — there is no transient `DELETING` state. + +GSI creation on `UpdateTable` is the one control-plane operation that does need asynchronous work — a background worker drains index rows in `CREATING` state, backfills the base collection, and flips the row to `ACTIVE`. See the Global and Local Secondary Indexes section for the state machine. + +### Authentication and authorization + +ExtendDB's mandatory SigV4 authentication is fully supported. Access-key secrets are stored AES-GCM encrypted in `extenddb_catalog.access_keys`. The encryption key is a 256-bit random key generated during `extenddb init`, base64-encoded, and stored in `extenddb_catalog.settings` under `_id: "encryption_key"`. Admin passwords are bcrypt-hashed before storage in `extenddb_catalog.admin_users`. + +IAM policy evaluation fetches user-attached policies, group-attached policies (via `iam_groups.members` → `iam_policies` join), role policies, permissions boundaries, and session policies from the catalog. + +`MongoEngine::new` rejects connection strings that specify a non-primary read preference (`secondary`, `secondaryPreferred`, `nearest`, `primaryPreferred`). DynamoDB's `ConsistentRead=true` requires linearizable reads; only MongoDB's Primary read preference provides that. A connection string that routes reads to a replica would silently return stale data — a fidelity violation the caller has no way to detect. The check fails at engine construction so misconfiguration surfaces at `extenddb serve` startup, not at request time. + +Encryption key bootstrap: `bootstrapper.rs::bootstrap_encryption_key`. Admin password hashing: same file, `bootstrap_admin_user`. Access-key decryption: `credential_store.rs`. IAM policy fetching: `authorization_store.rs`. + +### Backup + +`CreateBackup` snapshots the source table by running a server-side aggregation pipeline `[{ $out: "_backup_" }]` on the data collection. MongoDB copies items server-side without transferring them through the driver, and the destination is a per-backup collection in `extenddb_data` whose name derives from a UUID (never the caller-visible ARN, which contains characters MongoDB doesn't allow in collection names). Backup metadata (arn, backup_id, table, timestamps, key schema, table class, SSE, on-demand throughput, status) is stored in `extenddb_catalog.backups`. + +`RestoreTableFromBackup` recreates the table via the normal CreateTable path (preserving TableClass, SSESpecification, OnDemandThroughput from the backup metadata) and clones the backup collection into the new data collection with the same `$out` stage. `DeleteBackup` drops the backup collection and marks the metadata row `DELETED`. + +Backup implementation: `backup_engine.rs`. + +### Operational requirements + +**Minimum MongoDB version: 7.0.** Required for multi-document ACID transactions and snapshot reads. The MongoDB Rust driver 3.x is technically compatible with earlier server versions; this backend targets 7.0 as the minimum supported. + +**Replica set required.** MongoDB must be configured as a replica set before running `extenddb init`. A standalone node does not support multi-document transactions. A single-node replica set is sufficient for development and CI; production deployments should use a 3-node replica set for high availability. + +**Primary read preference.** Connection strings must use `readPreference=primary` (the driver default). Non-primary preferences are rejected at engine startup. + +**File descriptor limit.** Each MongoDB collection maps to one WiredTiger file. At 500 DynamoDB tables with 2 GSIs each (~1,500 collections), ensure `ulimit -n ≥ 65536` on the MongoDB host. See `docs/local-mongodb-setup.md` for platform-specific instructions. + +**Target scale.** This backend is designed for deployments of up to ~500 DynamoDB tables. At that scale, WiredTiger handles the collection count comfortably with default settings. Deployments significantly beyond this range have not been validated. + +Configuration is added under `[storage.mongodb]` in `extenddb.toml`: + +```toml +backend = "mongodb" + +[storage.mongodb] +connection_string = "mongodb://localhost:27017/?replicaSet=rs0" +max_connections = 50 +max_catalog_connections = 20 +``` + +Configuration struct: `crates/storage-mongodb/src/config.rs`. Sample configuration: `extenddb.sample.toml` — `[storage.mongodb]` section. Setup guide: `docs/local-mongodb-setup.md`. + +### Implementation summary + +| Crate modified | Change | +|---|---| +| `crates/storage-mongodb/` | New crate — full backend implementation | +| `crates/bin/Cargo.toml` | Added `mongodb` optional feature flag | +| `crates/bin/src/main.rs` | Added `#[cfg(feature = "mongodb")] extern crate` | +| `crates/bin/src/cmd_serve.rs` | Generalized the supported-backend gate from a hard-coded `"postgres"` check to a compile-time list built from enabled features | +| `Cargo.toml` (workspace) | Added crate to members; added `mongodb`, `bson`, `dashmap` workspace dependencies | + +No changes to `crates/engine/`, `crates/server/`, `crates/storage/` (trait definitions), `crates/auth/`, or `crates/core/`. + +### Design decisions summary + +| Decision | Choice | Rationale | +|---|---|---| +| Conditional writes | Read + evaluate + write inside a MongoDB transaction session | Snapshot atomicity gives DynamoDB's contract; loaded item reused for `ReturnValuesOnConditionCheckFailure = ALL_OLD` without a follow-up read. Analyzer-gated pushdown fast path skips the session for a certified subset on tables with no GSIs / streams. | +| UpdateItem concurrency | `_v` version guard inside snapshot txn + WriteConflict retry with jittered exponential backoff | Prevents lost updates; retry ceiling (50) bounds tail latency under sustained contention. | +| WriteConflict handling | Detect via `TransientTransactionError` label, `UnknownTransactionCommitResult` label, or raw code 112; retry with backoff | Converts a raw HTTP 500 into a retryable operation; TWI exhaustion surfaces as `TransactionCanceled` with per-op `TransactionConflict` reasons. | +| GSI updates | Synchronous inline within the base write's session, with 60-second TTL cache short-circuit for tables with no GSIs; async worker-driven backfill on UpdateTable | No Change Stream recovery; GSI reads are strongly consistent; UpdateTable matches DDB's async CREATING → ACTIVE contract. | +| GSI/LSI index docs | Composite `_id` includes both index and base keys; `base_pk` / `base_sk_?` stored as first-class fields | GSI keys are non-unique; base-key disambiguation prevents cross-item overwrite. Base keys as fields let index pagination form a compound cursor without traversing item_data. | +| Composite `_id` | Netstring-encoded (`:,...`) | Unambiguous boundary between pk and sk regardless of content. | +| Binary sort keys | Stored as lowercase hex strings | MongoDB's BSON Binary sort order diverges from DDB's unsigned-lex byte order across mismatched lengths. Hex-encoded strings preserve DDB order under default string comparison and make `begins_with` a plain range filter. | +| Sort key numbers | Native BSON `Decimal128` | Correct ordering by value. Values exceeding Decimal128's 34-digit precision are rejected. | +| DynamoDB Streams | Inline writes to `stream_records` inside the base write's session; per-shard sequence counters | Behavioral parity with PostgreSQL backend; sequence-number monotonicity within a shard is a contract. Session-scoped assignment prevents ordering holes under concurrent writes. | +| Stream shard ID | `shardId-{table_id}-{i:012}` | Table names are only account-unique; `table_id` UUID prevents cross-tenant shard address collisions. | +| Stream retention | TTL index on `stream_records.created_at` (24h) + hourly worker as defense in depth | Primary enforcement is at the storage layer; worker covers TTL-monitor lag or missing index. | +| Idempotency tokens | Unique compound index on `(account_id, token)` + 540s TTL + 600 ms data-plane age filter | Race safety under snapshot isolation; worst-case retention stays ≤10 min regardless of TTL-monitor cadence. | +| Backups | Per-backup collection via server-side `$out` aggregation | No per-item traffic between driver and server; backup metadata schema decouples from collection naming; ARN characters are unsafe as collection names. | +| Parallel scan | Application-side `crc32(pk) % segments` filter with lazy cursor iteration | Avoids per-document write overhead; lazy iteration prevents item-drops on hot-key skew that a hard server-side limit would cause. | +| Non-primary read preference | Rejected at engine startup | `ConsistentRead=true` requires linearizable reads; only Primary provides that. | + +### Performance characteristics + +**Single-item conditional writes.** One transaction session covers the pre-image read, condition evaluation, base write, GSI synchronization, and stream record insert. On a local replica set this adds ~sub-millisecond of session-start/commit overhead over a raw driver call. The session wrap is what gives DynamoDB's atomicity contract on conditional writes — it is the compatibility, not overhead. The analyzer-gated pushdown fast path collapses this to a single `find_one_and_*` call for the narrow case of certified pushable conditions on tables with no GSIs and no streams. + +**Unconditional single-item updates on GSI-free / stream-free tables.** A native-fast-path `find_one_and_update` runs outside any transaction. It always includes `$inc: {_v: 1}` so a concurrent slow-path update cannot pass its versioned filter against a stale snapshot. + +**GSI write overhead.** For tables with no GSIs, the `gsi_cache` short-circuits to zero overhead (no catalog query, no I/O) — refreshed at most once per `GSI_CACHE_TTL` window per table. For tables with GSIs, one catalog query fetches the index definitions (cached for subsequent writes) and one upsert or delete runs per index collection per write, all within the base write's session. + +**Stream write overhead.** When streams are enabled, each write adds one atomic per-shard counter `$inc` and one document insert into `stream_records`, both within the base write's session. + +**Query and Scan.** Direct index lookups on `(pk, sk_?)` for base tables; compound `(pk, sk_?, base_pk, base_sk_?)` lookups for index queries. `GetRecords` uses the compound `(shard_id, sequence_number)` index. + +**TransactWriteItems.** Multi-collection ACID transaction; up to 100 operations per the DDB spec. Uncommon in practice — most workloads are single-item operations. + +### Testing + +Testing is organized in three layers. + +**Unit tests** cover pure logic without a live MongoDB instance: netstring composite `_id` encoding, hex sort-key ordering, condition filter compilation, pushdown-analyzer decisions, sequence-number formatting, stream shard-id derivation. Property tests (`crates/storage-mongodb/tests/pushdown_parity.rs`) exercise the parity between the pushdown compiler and the in-Rust `evaluate_condition` reference over randomly generated items and expressions. + +**Integration tests** run against a single-node replica set (`mongod --replSet rs0`) covering the full table lifecycle, all item operations (conditional and unconditional), query and scan pagination (base and index), transactions, TTL worker behavior, stream record writes and consumer pagination, GSI propagation and async backfill, backup and restore, and all catalog and IAM operations. These execute as `cargo test -p extenddb-storage-mongodb`. + +**End-to-end tests** run the existing ExtendDB pytest suite (`tests/`) unchanged against a MongoDB-backed ExtendDB server. The pytest suite speaks the DynamoDB wire protocol and has no backend awareness — a passing run against MongoDB is equivalent to a passing run against PostgreSQL. This is the conformance test baseline required by RFC-0002. + +CI spins up a single-node MongoDB 7.0 replica set, builds ExtendDB with `--features mongodb`, runs `cargo test -p extenddb-storage-mongodb`, then runs `devtools/run-tests --extenddb --pytest` and `devtools/run-tests --extenddb --external` against the MongoDB-backed server. + +## Drawbacks + +**Replica set requirement.** MongoDB must be run as a replica set for multi-document transactions. Users who run standalone MongoDB will receive a runtime error on transactional operations. This is a MongoDB architectural constraint, not an ExtendDB limitation, and is documented in setup guides. + +**TTL throughput at scale.** DynamoDB TTL deletions must emit stream records with a specific service identity. MongoDB's native TTL indexes operate at the storage-engine level with no awareness of ExtendDB's stream system, so this implementation uses an application-level background worker that owns the full deletion lifecycle. The worker runs every 60 seconds and processes 100 expired items per table per pass. This is sufficient for ExtendDB's target deployment contexts. At very high sustained expiration rates the worker will fall behind, and the backlog will grow. This is a known scale limitation, not a correctness issue — DynamoDB's own contract only guarantees expiration within 48 hours, not immediately (see `docs/differences-from-dynamodb.md`, TTL row). + +## Alternatives + +### Use MongoDB Change Streams for DynamoDB Streams + +MongoDB has a native change-data-capture feature (Change Streams) that could back DynamoDB Streams. The implementation instead adopted the explicit `stream_records` collection approach used by the PostgreSQL backend, which gives ExtendDB full control over sequence-number generation, shard assignment, record retention, and iterator behavior — all of which the DynamoDB Streams API contract tightly specifies. Reviewers are invited to weigh in on whether a comparative evaluation of the Change Streams approach should be documented before acceptance. + +## Prior art + +**MongoDB document model and DynamoDB.** MongoDB's flexible document model has been noted as a natural fit for DynamoDB-style workloads in multiple independent analyses. Amazon DocumentDB (MongoDB-compatible) demonstrates AWS's own recognition of this overlap. The key difference in this implementation is that ExtendDB provides the full DynamoDB API layer — clients using the AWS SDK do not need to know they are talking to MongoDB. + +**Condition pushdown pattern.** Compiling application-level filter expressions into storage-native query operators is a well-established pattern in query engines (Apache Arrow DataFusion, Spark, Presto all implement predicate pushdown). The pushdown-analyzer / compiler split in this implementation applies the same principle at the storage backend level, with the analyzer serving as the correctness boundary between the two. + +--- + +## License + +Copyright 2026 ExtendDB contributors. Licensed under the Apache License, Version 2.0. +See [LICENSE](../../LICENSE) for the full text. + +This software is provided "as is" without warranty of any kind. ExtendDB is not +affiliated with, endorsed by, or sponsored by Amazon Web Services. "DynamoDB" is +a trademark of Amazon.com, Inc. diff --git a/extenddb-mongo.toml b/extenddb-mongo.toml new file mode 100644 index 00000000..e3423540 --- /dev/null +++ b/extenddb-mongo.toml @@ -0,0 +1,32 @@ +# ExtendDB config for MongoDB backend (integration testing) + +[server] +bind_addr = "127.0.0.1" +port = 8100 +region = "us-east-1" + +[server.tls] +cert_path = "~/.extenddb/tls/cert.pem" +key_path = "~/.extenddb/tls/key.pem" + +[storage] +backend = "mongodb" + +[storage.mongodb] +connection_string = "mongodb://localhost:27017/?replicaSet=rs0&directConnection=true" + +[limits] +enforce_reserved_keywords = true + +[auth] +provider = "builtin" + +[logging] +level = "debug" +format = "pretty" + +[import] +paths = ["/private/tmp", "/tmp"] + +[export] +paths = ["/private/tmp", "/tmp"] diff --git a/extenddb.sample.toml b/extenddb.sample.toml index 2c3ba608..d533ac47 100755 --- a/extenddb.sample.toml +++ b/extenddb.sample.toml @@ -29,7 +29,7 @@ # run_dir = "~/.extenddb/run" # Directory for PID file (~ is expanded to $HOME) [storage] -# backend = "postgres" # Storage backend (only "postgres" supported) +# backend = "postgres" # Storage backend: "postgres" or "mongodb" [storage.postgres] # Connection string points to the CATALOG database. @@ -52,6 +52,11 @@ # DynamoDB request makes concurrent authz queries # — size this to match expected concurrency. +[storage.mongodb] +# MongoDB connection string. Requires a replica set (even single-node). +# connection_string = "mongodb://localhost:27017/?replicaSet=rs0" +# max_pool_size = 20 # Maximum concurrent connections to MongoDB. + [auth] # provider = "builtin" # Auth provider: # "builtin" — SigV4 verification with local credential diff --git a/tests/python/test_rfc0003_concurrency.py b/tests/python/test_rfc0003_concurrency.py new file mode 100644 index 00000000..d8e3c455 --- /dev/null +++ b/tests/python/test_rfc0003_concurrency.py @@ -0,0 +1,244 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""RFC-0003 §4 concurrency conformance — strict, no client-side retries. + +Every scenario asserts that the backend surfaces the correct DDB error +class (or no error at all) even under sustained contention. If the +backend produces `InternalServerError` under any of these workloads, +the test fails — DDB is not permitted to surface internal concurrency- +control mechanisms as client errors, and neither is a conformant +backend. + +The RFC-0003 stress-test scenarios covered here: + +- §4.1 Two concurrent unconditional `PutItem` on the same key must + both succeed (last-writer-wins). +- §4.1 Concurrent conditional `PutItem` with `attribute_not_exists`: + exactly one succeeds, the rest fail with + `ConditionalCheckFailedException` — nothing internal. +- §4.4 Concurrent `UpdateItem ADD counter :one` on the same item must + all succeed; the final counter value must equal the total + increments applied (no lost updates, no internal errors). +- §4.1 Concurrent unconditional `DeleteItem` on the same key must all + return without error (last-writer-wins semantics for delete). + +These tests deliberately use a boto3 client with `retries={"max_attempts": 0}` +so any InternalServerError surfaces immediately instead of being masked +by the SDK's retry policy. +""" + +from __future__ import annotations + +import uuid +from concurrent.futures import ThreadPoolExecutor, as_completed + +import pytest +from botocore.exceptions import ClientError + +from helpers import unique_name, wait_for_active, wait_for_deleted + + +NUM_THREADS = 50 +INCREMENTS_PER_THREAD = 20 # 50 * 20 = 1_000 increments + + +@pytest.fixture() +def counter_table(dynamodb_client): + """A plain HASH-keyed table for the RFC-0003 §4.x scenarios.""" + name = unique_name("rfc4x") + dynamodb_client.create_table( + TableName=name, + AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(dynamodb_client, name) + yield name + dynamodb_client.delete_table(TableName=name) + wait_for_deleted(dynamodb_client, name) + + +def _classify(exc: ClientError) -> str: + """Return the ClientError's DynamoDB error code.""" + return exc.response.get("Error", {}).get("Code", "") + + +class TestRfc0003UnconditionalPutOnHotKey: + """RFC-0003 §4.1 — concurrent unconditional PutItem on the same key. + + Both must succeed; DDB never surfaces contention as a client error + for unconditional writes. This is the scenario the pre-Phase-6 + backend violated by returning `InternalServerError` after 50 retry + attempts of a snapshot-txn WriteConflict loop. + """ + + def test_all_writers_succeed(self, dynamodb_client, counter_table): + key = f"hot-{uuid.uuid4().hex[:8]}" + errors: list[str] = [] + + def _write(thread_id: int) -> None: + try: + dynamodb_client.put_item( + TableName=counter_table, + Item={ + "pk": {"S": key}, + "writer": {"N": str(thread_id)}, + }, + ) + except ClientError as e: + errors.append(_classify(e)) + + with ThreadPoolExecutor(max_workers=NUM_THREADS) as pool: + futs = [pool.submit(_write, tid) for tid in range(NUM_THREADS)] + for f in as_completed(futs): + f.result() + + # DDB contract: all writes succeed. Any error is a conformance failure. + assert errors == [], f"unexpected errors: {errors}" + + # And the item exists with *some* writer's value — last-writer-wins, + # so we don't assert which one, just that the item is there. + resp = dynamodb_client.get_item( + TableName=counter_table, + Key={"pk": {"S": key}}, + ConsistentRead=True, + ) + assert "Item" in resp + + +class TestRfc0003ConditionalPutOnHotKey: + """RFC-0003 §4.1 — concurrent conditional PutItem with attribute_not_exists. + + Exactly one writer wins (item created). Everyone else must fail with + `ConditionalCheckFailedException`, not `InternalServerError`. + """ + + def test_one_winner_rest_ccf(self, dynamodb_client, counter_table): + key = f"race-{uuid.uuid4().hex[:8]}" + outcomes: list[tuple[str, str]] = [] # (result, error_code) + + def _conditional_put(thread_id: int) -> None: + try: + dynamodb_client.put_item( + TableName=counter_table, + Item={ + "pk": {"S": key}, + "winner": {"N": str(thread_id)}, + }, + ConditionExpression="attribute_not_exists(pk)", + ) + outcomes.append(("ok", "")) + except ClientError as e: + outcomes.append(("err", _classify(e))) + + with ThreadPoolExecutor(max_workers=NUM_THREADS) as pool: + futs = [pool.submit(_conditional_put, tid) for tid in range(NUM_THREADS)] + for f in as_completed(futs): + f.result() + + winners = [o for o in outcomes if o[0] == "ok"] + losers = [o for o in outcomes if o[0] == "err"] + + assert len(winners) == 1, f"expected exactly one winner, got {len(winners)}" + assert len(losers) == NUM_THREADS - 1 + + # All losers must have ConditionalCheckFailedException — nothing + # else. Any InternalServerError is a conformance failure. + for _, code in losers: + assert code == "ConditionalCheckFailedException", ( + f"loser returned {code!r} instead of ConditionalCheckFailedException" + ) + + +class TestRfc0003AtomicCounterAdd: + """RFC-0003 §4.4 — concurrent `UpdateItem ADD counter :one`. + + Every increment must apply cumulatively; the final counter equals + NUM_THREADS * INCREMENTS_PER_THREAD. Every UpdateItem call must + succeed — no InternalServerError, no retries at the client. + + The mongo backend uses an aggregation-pipeline update + (`$toString` of `$add` of `$toDecimal`) so 50+ concurrent ADD + calls converge at MongoDB's doc-lock level without OCC retries. + """ + + def test_all_increments_apply(self, dynamodb_client, counter_table): + key = f"counter-{uuid.uuid4().hex[:8]}" + dynamodb_client.put_item( + TableName=counter_table, + Item={"pk": {"S": key}, "counter": {"N": "0"}}, + ) + errors: list[str] = [] + + def _increment(thread_id: int) -> int: + done = 0 + for _ in range(INCREMENTS_PER_THREAD): + try: + dynamodb_client.update_item( + TableName=counter_table, + Key={"pk": {"S": key}}, + UpdateExpression="ADD #c :one", + ExpressionAttributeNames={"#c": "counter"}, + ExpressionAttributeValues={":one": {"N": "1"}}, + ) + done += 1 + except ClientError as e: + errors.append(_classify(e)) + return done + + with ThreadPoolExecutor(max_workers=NUM_THREADS) as pool: + futs = [pool.submit(_increment, tid) for tid in range(NUM_THREADS)] + total_done = sum(f.result() for f in as_completed(futs)) + + assert errors == [], f"unexpected errors: {errors}" + assert total_done == NUM_THREADS * INCREMENTS_PER_THREAD + + # The counter must equal every increment applied. No lost updates. + resp = dynamodb_client.get_item( + TableName=counter_table, + Key={"pk": {"S": key}}, + ConsistentRead=True, + ) + final = int(resp["Item"]["counter"]["N"]) + assert final == NUM_THREADS * INCREMENTS_PER_THREAD + + +class TestRfc0003UnconditionalDeleteOnHotKey: + """RFC-0003 §4.1 — concurrent unconditional DeleteItem on the same key. + + All succeed. If the item exists, one delete removes it and the rest + are no-ops; if it doesn't, all are no-ops. Never an error. + """ + + def test_all_deletes_succeed(self, dynamodb_client, counter_table): + key = f"delkey-{uuid.uuid4().hex[:8]}" + dynamodb_client.put_item( + TableName=counter_table, + Item={"pk": {"S": key}, "val": {"S": "seed"}}, + ) + errors: list[str] = [] + + def _delete(_thread_id: int) -> None: + try: + dynamodb_client.delete_item( + TableName=counter_table, + Key={"pk": {"S": key}}, + ) + except ClientError as e: + errors.append(_classify(e)) + + with ThreadPoolExecutor(max_workers=NUM_THREADS) as pool: + futs = [pool.submit(_delete, tid) for tid in range(NUM_THREADS)] + for f in as_completed(futs): + f.result() + + assert errors == [], f"unexpected errors: {errors}" + + # Item must be gone. + resp = dynamodb_client.get_item( + TableName=counter_table, + Key={"pk": {"S": key}}, + ConsistentRead=True, + ) + assert "Item" not in resp diff --git a/tests/rust/src/main.rs b/tests/rust/src/main.rs index 9ff7cfa4..1482275c 100755 --- a/tests/rust/src/main.rs +++ b/tests/rust/src/main.rs @@ -70,6 +70,8 @@ mod query_more; #[cfg(test)] mod raw_http; #[cfg(test)] +mod restore_active_completeness; +#[cfg(test)] mod scan; #[cfg(test)] mod select_projection_validation; diff --git a/tests/rust/src/restore_active_completeness.rs b/tests/rust/src/restore_active_completeness.rs new file mode 100644 index 00000000..4eb81b91 --- /dev/null +++ b/tests/rust/src/restore_active_completeness.rs @@ -0,0 +1,181 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! A restored table must not report ACTIVE before its data copy completes. +//! +//! DynamoDB's contract: when `DescribeTable` on a restore target first returns +//! ACTIVE, the restored data is fully present. A restore that flips ACTIVE on a +//! control-plane timer decoupled from the copy exposes an empty or partial +//! table to a client that waits-for-ACTIVE. +//! +//! This is a race detector, deliberately. `RestoreTableFromBackup` in this +//! backend blocks until the copy completes, so the ACTIVE-before-data window is +//! only observable from a second client, and whether it is observed depends on +//! whether the copy or the transition timer wins. Two consequences worth +//! knowing before reading a result: +//! +//! - It cannot fail when the ordering is correct. If ACTIVE is only set after +//! the copy drains, the observed count is always complete. +//! - A PASS is not proof the ordering is correct. On an idle server the `$out` +//! copy can finish inside the transition window, and the race is simply not +//! observed. Failures are meaningful; passes are weak evidence. +//! +//! The dataset is sized so the copy takes longer than the transition window on +//! a loaded server. On very fast or idle hardware, raising `ITEMS` widens the +//! window. + +use crate::test_base::*; +use aws_sdk_dynamodb::types::{ + AttributeDefinition, AttributeValue, BillingMode, KeySchemaElement, KeyType, PutRequest, + ScalarAttributeType, Select, WriteRequest, +}; + +const ITEMS: usize = 40000; + +/// Bound on the observer's wait for first ACTIVE, so a failed restore fails the +/// test rather than hanging it. 10ms per attempt, so this is a 60s ceiling. +const OBSERVER_MAX_ATTEMPTS: usize = 6000; + +#[tokio::test] +async fn restored_table_has_all_items_when_first_active() { + let c = client(); + let src = format!("RestoreRaceSrc_{}", ts()); + c.create_table() + .table_name(&src) + .key_schema( + KeySchemaElement::builder() + .attribute_name("pk") + .key_type(KeyType::Hash) + .build() + .unwrap(), + ) + .attribute_definitions( + AttributeDefinition::builder() + .attribute_name("pk") + .attribute_type(ScalarAttributeType::S) + .build() + .unwrap(), + ) + .billing_mode(BillingMode::PayPerRequest) + .send() + .await + .unwrap(); + wait_for_active(&c, &src).await; + + // Enough data that the restore copy takes longer than the control-plane + // transition delay, so a timer-driven ACTIVE flip would win the race. + let pad = "x".repeat(2000); + for chunk in (0..ITEMS).collect::>().chunks(25) { + let reqs: Vec = chunk + .iter() + .map(|i| { + WriteRequest::builder() + .put_request( + PutRequest::builder() + .item("pk", AttributeValue::S(format!("k{i:06}"))) + .item("d", AttributeValue::S(pad.clone())) + .build() + .unwrap(), + ) + .build() + }) + .collect(); + c.batch_write_item() + .request_items(&src, reqs) + .send() + .await + .unwrap(); + } + + let backup = c + .create_backup() + .table_name(&src) + .backup_name("restore-race-probe") + .send() + .await + .unwrap(); + let arn = backup.backup_details().unwrap().backup_arn().to_string(); + // Wait until the backup is AVAILABLE. + for _ in 0..240 { + let d = c.describe_backup().backup_arn(&arn).send().await.unwrap(); + if d.backup_description() + .and_then(|b| b.backup_details()) + .map(|b| b.backup_status().as_str() == "AVAILABLE") + .unwrap_or(false) + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + + let dst = format!("RestoreRaceDst_{}", ts()); + // Race an observer against the in-flight restore: the moment it sees + // ACTIVE, it counts. Returns None if ACTIVE never arrives within the + // bound, which means the restore itself failed. + let observer = { + let c2 = client().clone(); + let dst2 = dst.clone(); + tokio::spawn(async move { + let mut saw_active = false; + for _ in 0..OBSERVER_MAX_ATTEMPTS { + if let Ok(out) = c2.describe_table().table_name(&dst2).send().await { + let status = out + .table() + .and_then(|t| t.table_status()) + .map(|s| s.as_str().to_owned()); + if status.as_deref() == Some("ACTIVE") { + saw_active = true; + break; + } + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + if !saw_active { + return None; + } + // First ACTIVE observation: count items immediately. + let mut count = 0usize; + let mut start_key = None; + loop { + let mut req = c2.scan().table_name(&dst2).select(Select::Count); + if let Some(k) = start_key.take() { + req = req.set_exclusive_start_key(Some(k)); + } + let resp = match req.send().await { + Ok(r) => r, + Err(_) => break, + }; + count += resp.count() as usize; + match resp.last_evaluated_key() { + Some(k) if !k.is_empty() => start_key = Some(k.clone()), + _ => break, + } + } + Some(count) + }) + }; + + c.restore_table_from_backup() + .target_table_name(&dst) + .backup_arn(&arn) + .send() + .await + .unwrap(); + + // The concurrent observer counted at first-ACTIVE while the restore call + // above was still in flight (or just after, if the copy was fast). + let observed = observer.await.unwrap(); + + c.delete_table().table_name(&src).send().await.ok(); + c.delete_table().table_name(&dst).send().await.ok(); + + let count = observed.expect( + "restore target never reported ACTIVE within the observer bound, \ + so the restore itself did not complete", + ); + assert_eq!( + count, ITEMS, + "restored table reported ACTIVE with {count}/{ITEMS} items present. \ + ACTIVE must imply the restore copy is complete" + ); +}