From 206c83a6a2e5ba8b13e5b8d4254e7429498692e6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 17 Jun 2026 13:24:57 +0000 Subject: [PATCH 1/2] feat(storage): add S3 Object Annotations backend (s3annotations feature) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a second satirical-but-functional storage backend, modeled on the Route 53 backend from PR #54. Items are stored as named annotations on a sentinel S3 object (the "table", default key `.well-actually`): the annotation name encodes the partition/sort key and the value carries the JSON item body. - `encoding`: real, round-trip-tested item<->annotation mapping. Bodies are base64-encoded and chunked across sibling annotations (`#0001`, ...) to respect the 1 MB per-annotation limit, reassembled on read — the direct analog of #54's 255-byte TXT-string spillover. Five passing round-trip tests, including a multi-MB item that exercises chunking. - `S3AnnotationsBootstrapper`: the `Bootstrapper` trait, registered with the backend inventory under `s3annotations`. Every method returns `OpError::Internal` annotated (error string + inline comment) with the S3 Object Annotations API call a real implementation would issue, giving a future implementer a porting map. Gated behind the `s3annotations` cargo feature on the `extenddb` binary; `cargo build` is unchanged and the backend is not registered. With `--features s3annotations`, `extenddb init --backend s3annotations` reaches the bootstrapper and returns the porting-map error. Includes PR_DESCRIPTION.md. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_014bbGFdvhY4Hi6qBykooFYA --- Cargo.lock | 15 + Cargo.toml | 2 + crates/bin/Cargo.toml | 2 + crates/bin/src/main.rs | 7 + crates/storage-s3annotations/Cargo.toml | 20 ++ .../storage-s3annotations/PR_DESCRIPTION.md | 167 +++++++++ .../storage-s3annotations/src/bootstrapper.rs | 198 ++++++++++ crates/storage-s3annotations/src/encoding.rs | 337 ++++++++++++++++++ crates/storage-s3annotations/src/lib.rs | 41 +++ 9 files changed, 789 insertions(+) create mode 100644 crates/storage-s3annotations/Cargo.toml create mode 100644 crates/storage-s3annotations/PR_DESCRIPTION.md create mode 100644 crates/storage-s3annotations/src/bootstrapper.rs create mode 100644 crates/storage-s3annotations/src/encoding.rs create mode 100644 crates/storage-s3annotations/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index c74e1077..c91677b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -852,6 +852,7 @@ dependencies = [ "extenddb-server", "extenddb-storage", "extenddb-storage-postgres", + "extenddb-storage-s3annotations", "libc", "rand 0.9.4", "rcgen", @@ -994,6 +995,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "extenddb-storage-s3annotations" +version = "0.1.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "extenddb-core", + "extenddb-storage", + "inventory", + "serde_json", + "thiserror 2.0.18", + "tracing", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" diff --git a/Cargo.toml b/Cargo.toml index b5581126..7fc90291 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/engine", "crates/storage", "crates/storage-postgres", + "crates/storage-s3annotations", "crates/auth", "crates/server", "crates/bin", @@ -24,6 +25,7 @@ extenddb-core = { path = "crates/core" } extenddb-engine = { path = "crates/engine" } extenddb-storage = { path = "crates/storage" } extenddb-storage-postgres = { path = "crates/storage-postgres" } +extenddb-storage-s3annotations = { path = "crates/storage-s3annotations" } extenddb-auth = { path = "crates/auth" } extenddb-server = { path = "crates/server" } diff --git a/crates/bin/Cargo.toml b/crates/bin/Cargo.toml index 2a2c646a..0b42f132 100755 --- a/crates/bin/Cargo.toml +++ b/crates/bin/Cargo.toml @@ -14,12 +14,14 @@ path = "src/main.rs" [features] default = ["postgres"] postgres = ["extenddb-storage-postgres"] +s3annotations = ["extenddb-storage-s3annotations"] [dependencies] extenddb-core = { workspace = true } extenddb-engine = { workspace = true } extenddb-storage = { workspace = true } extenddb-storage-postgres = { workspace = true, optional = true } +extenddb-storage-s3annotations = { workspace = true, optional = true } extenddb-auth = { workspace = true } extenddb-server = { workspace = true } tokio = { workspace = true } diff --git a/crates/bin/src/main.rs b/crates/bin/src/main.rs index b4f61bf0..1a5eafa3 100755 --- a/crates/bin/src/main.rs +++ b/crates/bin/src/main.rs @@ -25,6 +25,13 @@ mod serve_helpers; mod util; mod workers; +// The S3 Annotations backend registers itself via `inventory` at link time. +// Nothing in this binary names the crate by path, so `extern crate` is what +// forces the linker to include it — and thus its `BackendRegistration` — when +// the `s3annotations` feature is enabled. +#[cfg(feature = "s3annotations")] +extern crate extenddb_storage_s3annotations; + use clap::{Parser, Subcommand}; #[derive(Parser)] diff --git a/crates/storage-s3annotations/Cargo.toml b/crates/storage-s3annotations/Cargo.toml new file mode 100644 index 00000000..84340fb8 --- /dev/null +++ b/crates/storage-s3annotations/Cargo.toml @@ -0,0 +1,20 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +[package] +name = "extenddb-storage-s3annotations" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +extenddb-core = { workspace = true } +extenddb-storage = { workspace = true } +async-trait = { workspace = true } +inventory = { workspace = true } +base64 = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +serde_json = { workspace = true } diff --git a/crates/storage-s3annotations/PR_DESCRIPTION.md b/crates/storage-s3annotations/PR_DESCRIPTION.md new file mode 100644 index 00000000..db5d916e --- /dev/null +++ b/crates/storage-s3annotations/PR_DESCRIPTION.md @@ -0,0 +1,167 @@ +# feat(storage): add S3 Object Annotations backend (named annotations, base64-chunked items, AWS ships the SELECT) + +### Forward + +This is the second time. [PR #54](https://github.com/ExtendDB/extenddb/pull/54) added a +Route 53 storage backend, on the standing argument that Route 53 is a database. That PR +made the case by force: it bent TXT records into a key-value store and dared the reviewer +to say it wasn't one. + +S3 Object Annotations — [launched 2026-06-16](https://aws.amazon.com/blogs/aws/amazon-s3-annotations-attach-rich-queryable-context-directly-to-your-objects/) — +is a more honest fit than Route 53 was, and I want to be clear about why. With Route 53 +I had to supply the query path myself; DNS does not come with a `WHERE` clause. With S3 +Annotations, AWS shipped the query path. They built an Apache Iceberg table that auto-indexes +every annotation, wired it to Athena, gave it a journal of change records, and then +described the result as "rich, queryable context." They built a database and declined to +call it one. This PR calls it one. + +### What's in here + +A new `extenddb-storage-s3annotations` crate (behind an `s3annotations` cargo feature) +stores items as named annotations on a sentinel S3 object. The object is the table — the +structural analog of #54's hosted zone. Its default key is `.well-actually`. + +- **`encoding`** — the real, round-trip-tested item↔annotation mapping. Each item is one + logical annotation: the annotation **name** encodes the partition/sort key, the annotation + **value** is the JSON-serialized item body. Bodies are base64-encoded and chunked into + ≤ 1 MB pieces, one annotation per chunk, named `#0001`, `#0002`, … and + reassembled on read. This is the direct analog of #54's 255-byte TXT-string spillover; + the constraint moved from 255 bytes to 1 MB but the mechanism is identical. +- **`S3AnnotationsBootstrapper`** — the `Bootstrapper` trait, registered with the backend + inventory under the name `s3annotations`. Every method returns `OpError::Internal` + annotated — in both the error string and an inline source comment — with the S3 Object + Annotations API call a real implementation would issue. It is a porting map with a + non-zero exit code. + +The encoding module has five passing round-trip tests, including a small single-annotation +item and a multi-megabyte item that exercises chunking. `cargo test -p extenddb-storage-s3annotations` +is green. + +### Why this is not as deranged as it sounds + +**Properties that make the substrate look reasonable:** + +- AWS provides the `SELECT` path. Athena over the Iceberg annotation table, the + `text_value` column, and the S3 Tables MCP server are all theirs. The query engine + ExtendDB would otherwise have to build is simply free. +- Annotations update in place without rewriting the object. An `UpdateItem` does not pay to + rewrite the whole item the way a DynamoDB write effectively does. +- Annotations move with the object on copy and replication, and are deleted with the object. + That is cascade-delete and referential integrity, for free, enforced by the storage layer. +- The annotation table is an asynchronously-built secondary index that you did not provision, + do not manage, and are not billed to maintain as a GSI. +- Objects in S3 Glacier remain queryable through the annotation table without a restore. The + rows can sit in cold storage while the index stays hot. + +**Properties that make it indefensible:** + +- Annotation tables refresh within an hour and backfill takes "hours to days," so the + queryable index has an eventual-consistency window measured in business days. +- The point API (`GetObjectAnnotation`) is strongly consistent, but the SQL path lags. Read- + your-writes therefore holds only if you never use the query engine that is the entire point + of the backend. +- Annotation storage bills at S3 Standard rates regardless of the parent object's storage + class. The cold-storage-rows trick above costs Standard rates on the metadata, so the + savings are imaginary. +- 1,000 annotations per object caps items per table at 1,000. +- Every Athena query is a full table scan billed per TB scanned. There is no point-read price; + there is only the scan. + +I am genuinely unsure which list is more interesting. I have included both for the reviewer's +enjoyment. + +### Pricing + +| Component | DynamoDB on-demand | S3 Annotations backend | +|-----------|--------------------|------------------------| +| Storage | Per GB-month, by storage class | Per GB-month at **S3 Standard**, regardless of the object's class | +| Writes | Per WCU-second | Per `PutObjectAnnotation` call | +| Reads (consistent point) | Per RCU | Per `GetObjectAnnotation` call | +| Reads (analytical) | Per RCU (Query/Scan) | Per **TB scanned** in Athena — every query is a full table scan | + +Point-read-heavy workloads map cleanly onto `GetObjectAnnotation`. Analytical workloads get a +real SQL engine they did not have to build, billed by the terabyte regardless of how few rows +they wanted. + +### Streams + +ExtendDB streams map onto the S3 Metadata **journal table**, which is a change log AWS already +maintains in near real time. The streams implementation tails `CREATE_ANNOTATION` and +`DELETE_ANNOTATION` records: + +- `record_timestamp` → `ApproximateCreationDateTime` +- `CREATE_ANNOTATION` → `INSERT` +- `DELETE_ANNOTATION` → `REMOVE` + +This is cleaner than #54's streams. There, I had to poll Route 53's `GetChange` for +propagation state and synthesize records when changes reached `INSYNC`. Here AWS ships an +actual change log, so there is nothing to poll and nothing to synthesize — you read the +journal. + +### Build matrix + +| Build | Behavior | +|-------|----------| +| `cargo build` | Unchanged; `s3annotations` not registered | +| `cargo build --features s3annotations` | `extenddb init --backend s3annotations` reaches the bootstrapper with an S3 Annotations error | +| `cargo test -p extenddb-storage-s3annotations` | Five encoding round-trip tests; all pass | + +### Before / after + +**Before:** + +``` +$ extenddb init --backend s3annotations +Error: Internal("Unknown backend: s3annotations. Available backends: postgres") +``` + +**After** (built with `--features s3annotations`): + +``` +$ extenddb init --backend s3annotations +Error: Internal("ensure_app_user: S3 Annotations backend is registered but the +relevant operation is not yet implemented. Use --backend postgres, or wire this +method to the corresponding S3 Annotations API call (referenced inline below). +Maps to S3 Annotations CreateBucketMetadataConfiguration.") +``` + +Users now receive both an error and a porting map to the AWS API call. The full map, one row +per `Bootstrapper` method: + +| `Bootstrapper` method | Maps to S3 Annotations call | +|-----------------------|-----------------------------| +| `ensure_app_user`, `grant_app_role_to_admin` | `CreateBucketMetadataConfiguration` | +| `create_catalog_db`, `create_data_db` | `CreateBucketMetadataConfiguration` | +| `run_catalog_migrations`, `run_data_migrations` | `UpdateBucketMetadataAnnotationTableConfiguration` | +| `record_data_connection`, `bootstrap_encryption_key`, `bootstrap_default_account`, `bootstrap_admin_user` | `PutObjectAnnotation` | +| `is_catalog_initialized`, `list_table_names` | `ListObjectAnnotations` | +| `get_data_db_name`, `read_catalog_version` | `GetObjectAnnotation` | +| `drop_databases` | `DeleteObjectAnnotation` | + +### What's still missing + +| Piece | State | +|-------|-------| +| Cargo crate + workspace member | Done | +| `Bootstrapper` impl (registered, stubbed with sourced errors) | Done | +| `encoding` module (chunking, base64, round-trip tests) | Done | +| `OperationsEngineRegistration` | Not in this PR | +| `StorageConfigRegistration` | Not in this PR | +| `SettingsStoreRegistration` | Not in this PR | +| `DiagnosticsStoreRegistration` | Not in this PR | +| `ServerComponentsRegistration` | Not in this PR | +| `crates/bin/src/config.rs` (hard-references postgres) | Not modified | + +### Organizational note + +With this PR, two of ExtendDB's pluggable backends are covered AWS services. One was an +argument I had to win; this one AWS effectively conceded by shipping the query path. It is +worth asking, before merge rather than after, whether ExtendDB is still a database or has +quietly become an AWS invoice with a CLI in front of it. + +If the project would rather not answer that question, the alternative is the same one I +offered in #54: I will withdraw this PR and instead submit a one-line edit to the +`--backend` help text in `crates/bin/src/cmd_init.rs:19`, removing the implicit invitation to +name a service that isn't PostgreSQL. I leave the choice of which is funnier to the maintainer. + +**I do declare that S3 is, in fact, a database. I dare you to prove me wrong.** diff --git a/crates/storage-s3annotations/src/bootstrapper.rs b/crates/storage-s3annotations/src/bootstrapper.rs new file mode 100644 index 00000000..99413e3d --- /dev/null +++ b/crates/storage-s3annotations/src/bootstrapper.rs @@ -0,0 +1,198 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Stubbed `Bootstrapper` for the S3 Object Annotations backend. +//! +//! Every method returns [`OpError::Internal`] annotated — both in the error +//! string and inline in the source — with the S3 Object Annotations API call a +//! real implementation would issue. This is the direct analog of PR #54's +//! Route 53 bootstrapper, which mapped each method to a Route 53 call +//! (`CreateHostedZone`, `ChangeResourceRecordSets`, …). Here the porting map +//! points at `CreateBucketMetadataConfiguration`, +//! `UpdateBucketMetadataAnnotationTableConfiguration`, `PutObjectAnnotation`, +//! `GetObjectAnnotation`, `ListObjectAnnotations`, and `DeleteObjectAnnotation`. +//! +//! Nothing here talks to S3. The crate exists to register the backend so that +//! `extenddb init --backend s3annotations` reaches a bootstrapper that hands the +//! operator a porting map instead of an "unknown backend" error. + +use async_trait::async_trait; +use extenddb_storage::bootstrapper::{AdminBootstrapResult, Bootstrapper}; +use extenddb_storage::error::StorageError; +use extenddb_storage::management_store::{OpError, OpResult}; + +use crate::encoding::DEFAULT_TABLE_OBJECT_KEY; + +/// The catalog schema version this (stubbed) backend would target. +const CATALOG_VERSION: &str = "s3annotations-0"; + +/// Stubbed bootstrapper for the S3 Object Annotations backend. +/// +/// Holds the would-be connection coordinates so the display methods can render +/// something honest. No S3 client is constructed. +pub struct S3AnnotationsBootstrapper { + /// The bucket whose sentinel object plays the role of the catalog "table". + bucket: String, + /// The sentinel object key (the "table"); see [`DEFAULT_TABLE_OBJECT_KEY`]. + table_object_key: String, +} + +impl S3AnnotationsBootstrapper { + /// Build a bootstrapper from the config file and CLI args. + /// + /// A real implementation would parse the bucket and region out of config; + /// the stub records sensible placeholders so the lifecycle display commands + /// have something to print. It never fails, because it never connects. + /// + /// # Errors + /// + /// Returns [`StorageError`] only to match the factory signature; the stub + /// always succeeds. + pub async fn from_config( + _config_path: &str, + _cli_args: &[String], + ) -> Result { + Ok(Self { + bucket: "extenddb-annotations".to_owned(), + table_object_key: DEFAULT_TABLE_OBJECT_KEY.to_owned(), + }) + } +} + +/// Build the standard "registered but unimplemented" error, naming the S3 +/// Object Annotations API call a real implementation would issue. Mirrors the +/// porting-map error shape PR #54 used for Route 53. +fn todo_op(method: &str, maps_to: &str) -> OpError { + OpError::Internal(format!( + "{method}: S3 Annotations backend is registered but the relevant \ + operation is not yet implemented. Use --backend postgres, or wire this \ + method to the corresponding S3 Annotations API call (referenced inline \ + below). Maps to S3 Annotations {maps_to}." + )) +} + +#[async_trait] +impl Bootstrapper for S3AnnotationsBootstrapper { + async fn ensure_app_user(&self) -> OpResult<()> { + // S3 has no per-backend "app user"; access is IAM. Provisioning the + // metadata substrate is the CreateBucketMetadataConfiguration call. + Err(todo_op( + "ensure_app_user", + "CreateBucketMetadataConfiguration", + )) + } + + async fn grant_app_role_to_admin(&self) -> OpResult<()> { + // No role grant exists; the closest provisioning step is enabling the + // bucket's metadata configuration. + Err(todo_op( + "grant_app_role_to_admin", + "CreateBucketMetadataConfiguration", + )) + } + + async fn create_catalog_db(&self) -> OpResult<()> { + // CreateBucketMetadataConfiguration: enabling S3 Metadata is what + // brings the Iceberg annotation table into existence. + Err(todo_op( + "create_catalog_db", + "CreateBucketMetadataConfiguration", + )) + } + + async fn create_data_db(&self) -> OpResult<()> { + // CreateBucketMetadataConfiguration on the data bucket. + Err(todo_op( + "create_data_db", + "CreateBucketMetadataConfiguration", + )) + } + + async fn run_catalog_migrations(&self) -> OpResult<()> { + // UpdateBucketMetadataAnnotationTableConfiguration: shape the annotation + // table that backs the catalog (there is no DDL; you configure it). + Err(todo_op( + "run_catalog_migrations", + "UpdateBucketMetadataAnnotationTableConfiguration", + )) + } + + async fn run_data_migrations(&self) -> OpResult<()> { + // UpdateBucketMetadataAnnotationTableConfiguration on the data bucket. + Err(todo_op( + "run_data_migrations", + "UpdateBucketMetadataAnnotationTableConfiguration", + )) + } + + async fn record_data_connection(&self) -> OpResult<()> { + // PutObjectAnnotation: write the data-bucket coordinates as an + // annotation on the catalog sentinel object. + Err(todo_op("record_data_connection", "PutObjectAnnotation")) + } + + async fn bootstrap_encryption_key(&self) -> OpResult<()> { + // PutObjectAnnotation: store the wrapped encryption key as an annotation. + Err(todo_op("bootstrap_encryption_key", "PutObjectAnnotation")) + } + + async fn bootstrap_default_account(&self) -> OpResult<()> { + // PutObjectAnnotation: write the default account record. + Err(todo_op("bootstrap_default_account", "PutObjectAnnotation")) + } + + async fn bootstrap_admin_user( + &self, + _env_user: Option<&str>, + _env_password: Option<&str>, + ) -> OpResult { + // PutObjectAnnotation: write the initial admin user record. + Err(todo_op("bootstrap_admin_user", "PutObjectAnnotation")) + } + + async fn is_catalog_initialized(&self) -> OpResult { + // ListObjectAnnotations: the catalog is "initialized" iff the sentinel + // object already carries its bootstrap annotations. + Err(todo_op("is_catalog_initialized", "ListObjectAnnotations")) + } + + async fn list_table_names(&self) -> OpResult> { + // ListObjectAnnotations: enumerate the sentinel object's annotations. + Err(todo_op("list_table_names", "ListObjectAnnotations")) + } + + async fn get_data_db_name(&self) -> OpResult> { + // GetObjectAnnotation: read the named annotation holding the data-bucket + // coordinates. + Err(todo_op("get_data_db_name", "GetObjectAnnotation")) + } + + async fn drop_databases(&self, _data_db: &str) -> OpResult<()> { + // DeleteObjectAnnotation: tear the catalog down annotation by annotation + // (or delete the sentinel objects, which cascades to their annotations). + Err(todo_op("drop_databases", "DeleteObjectAnnotation")) + } + + async fn read_catalog_version(&self) -> OpResult> { + // GetObjectAnnotation: read the named annotation holding the schema + // version. + Err(todo_op("read_catalog_version", "GetObjectAnnotation")) + } + + fn expected_catalog_version(&self) -> String { + CATALOG_VERSION.to_owned() + } + + fn catalog_database_name(&self) -> String { + // The sentinel object is the catalog "database". + format!("s3://{}/{}", self.bucket, self.table_object_key) + } + + fn endpoint_info(&self) -> String { + format!("s3.amazonaws.com (bucket: {})", self.bucket) + } + + fn catalog_connection_url(&self) -> String { + format!("s3://{}/{}", self.bucket, self.table_object_key) + } +} diff --git a/crates/storage-s3annotations/src/encoding.rs b/crates/storage-s3annotations/src/encoding.rs new file mode 100644 index 00000000..6a4fd610 --- /dev/null +++ b/crates/storage-s3annotations/src/encoding.rs @@ -0,0 +1,337 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Real, round-trip-tested encoding of ExtendDB items onto S3 Object Annotations. +//! +//! This is the direct analog of the Route 53 backend's TXT-record chunking +//! (PR #54). There, the 255-byte limit on a single TXT character-string forced +//! item bodies to spill across sibling resource records keyed by sequence +//! number. Here the analogous constraint is the **1 MB per-named-annotation** +//! ceiling, so item bodies spill across sibling *annotations* keyed by sequence +//! number, reassembled on read. +//! +//! The mapping: +//! +//! - A sentinel S3 object is the "table" (analogous to #54's hosted zone). Its +//! default key is [`DEFAULT_TABLE_OBJECT_KEY`]. +//! - Each item is one logical annotation: the annotation **name** encodes the +//! partition/sort key, and the annotation **value** is the JSON-serialized +//! item body. +//! - Items larger than 1 MB are split across sibling annotations keyed by +//! sequence number (`#0001`, `#0002`, …), reassembled on read. +//! +//! The body is base64-encoded before chunking. base64 output is pure ASCII, so +//! it can be split at any byte boundary without tearing a UTF-8 code point — +//! exactly the property #54 relied on when slicing into 255-byte TXT strings. + +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD as B64; + +/// Default S3 object key that plays the role of a "table". +/// +/// Analogous to PR #54's hosted zone: the sentinel object whose annotations +/// collectively constitute the table's contents. The key is, of course, a +/// rebuttal. +pub const DEFAULT_TABLE_OBJECT_KEY: &str = ".well-actually"; + +/// Maximum size of a single named annotation value: 1 MB, per the S3 Object +/// Annotations limits. Item bodies larger than this spill across siblings. +pub const MAX_ANNOTATION_VALUE_BYTES: usize = 1024 * 1024; + +/// Maximum number of named annotations attached to a single object: 1,000, per +/// the S3 Object Annotations limits. Because each item is encoded as one or +/// more annotations on the sentinel object, this caps items-per-table at 1,000. +pub const MAX_ANNOTATIONS_PER_OBJECT: usize = 1000; + +/// Separator between the encoded partition key and encoded sort key within an +/// annotation name. `~` is not part of the URL-safe base64 alphabet, so it can +/// never appear inside an encoded key component. +const KEY_SEPARATOR: char = '~'; + +/// Separator between the encoded key and the chunk sequence number. `#` is also +/// outside the base64 alphabet, so the sequence suffix is unambiguous. +const SEQUENCE_SEPARATOR: char = '#'; + +/// Errors produced while encoding items to, or decoding items from, annotations. +#[derive(Debug, thiserror::Error)] +pub enum EncodingError { + /// An annotation name (or key component) was not valid base64. + #[error("annotation name is not valid base64: {0}")] + InvalidBase64(String), + /// An annotation name did not match the `#NNNN` shape. + #[error("annotation name is malformed: {0}")] + MalformedName(String), + /// A decoded key component or body was not valid UTF-8. + #[error("decoded value is not valid UTF-8")] + InvalidUtf8, + /// No annotations were supplied to reassemble an item. + #[error("no annotations supplied for item")] + Empty, + /// The chunk sequence numbers were not a contiguous `1..=n` run. + #[error("annotation chunks do not form a contiguous sequence")] + NonContiguousChunks, + /// The item would require more annotations than an object can hold. + #[error("item too large: {chunks} chunks exceeds the {max}-annotation per-object limit")] + TooManyChunks { chunks: usize, max: usize }, +} + +/// The partition (and optional sort) key of an item, before encoding. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ItemKey { + /// The partition key value, rendered as a string. + pub partition_key: String, + /// The sort key value, if the table has a sort key. + pub sort_key: Option, +} + +/// A single named annotation as it would be written via `PutObjectAnnotation` +/// and read back via `GetObjectAnnotation` / `ListObjectAnnotations`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Annotation { + /// The annotation name: `#NNNN`. + pub name: String, + /// The annotation value: a slice of the base64-encoded item body. Lands in + /// the `text_value` column of the Iceberg annotation table. + pub value: String, +} + +/// Encode an item key into the shared base of its annotation names. +/// +/// The partition and sort key are base64-encoded independently and joined with +/// [`KEY_SEPARATOR`], so either may contain arbitrary bytes (including `#` or +/// `~`) without colliding with the name grammar. +#[must_use] +pub fn encode_key(key: &ItemKey) -> String { + let pk = B64.encode(key.partition_key.as_bytes()); + match &key.sort_key { + Some(sk) => format!("{pk}{KEY_SEPARATOR}{}", B64.encode(sk.as_bytes())), + None => pk, + } +} + +/// Decode the shared base of an annotation name back into an [`ItemKey`]. +pub fn decode_key(encoded: &str) -> Result { + match encoded.split_once(KEY_SEPARATOR) { + Some((pk_b64, sk_b64)) => Ok(ItemKey { + partition_key: decode_component(pk_b64)?, + sort_key: Some(decode_component(sk_b64)?), + }), + None => Ok(ItemKey { + partition_key: decode_component(encoded)?, + sort_key: None, + }), + } +} + +fn decode_component(b64: &str) -> Result { + let bytes = B64 + .decode(b64) + .map_err(|e| EncodingError::InvalidBase64(e.to_string()))?; + String::from_utf8(bytes).map_err(|_| EncodingError::InvalidUtf8) +} + +/// Encode an item into one or more annotations. +/// +/// `body` is the JSON-serialized item body (the analog of #54's item JSON). It +/// is base64-encoded — keeping the payload ASCII so it can be split at any byte +/// boundary — then chunked into `<=` [`MAX_ANNOTATION_VALUE_BYTES`] pieces. Each +/// chunk becomes one named annotation `"#NNNN"`, written via +/// `PutObjectAnnotation`. +/// +/// # Errors +/// +/// Returns [`EncodingError::TooManyChunks`] if the body would need more than +/// [`MAX_ANNOTATIONS_PER_OBJECT`] annotations to store. +pub fn encode_item(key: &ItemKey, body: &str) -> Result, EncodingError> { + let base = encode_key(key); + let encoded = B64.encode(body.as_bytes()); + // base64 output is pure ASCII, so each byte slice is itself valid UTF-8 and + // a valid base64 fragment of the whole — the same invariant that let #54 + // tear the payload across 255-byte TXT strings. + let bytes = encoded.as_bytes(); + + let chunk_count = bytes.len().div_ceil(MAX_ANNOTATION_VALUE_BYTES).max(1); + if chunk_count > MAX_ANNOTATIONS_PER_OBJECT { + return Err(EncodingError::TooManyChunks { + chunks: chunk_count, + max: MAX_ANNOTATIONS_PER_OBJECT, + }); + } + + // An empty body still occupies one annotation, so a present-but-empty item + // round-trips rather than vanishing. + if bytes.is_empty() { + return Ok(vec![Annotation { + name: format!("{base}{SEQUENCE_SEPARATOR}0001"), + value: String::new(), + }]); + } + + let annotations = bytes + .chunks(MAX_ANNOTATION_VALUE_BYTES) + .enumerate() + .map(|(i, chunk)| Annotation { + name: format!("{base}{SEQUENCE_SEPARATOR}{seq:04}", seq = i + 1), + // `chunk` is a slice of ASCII base64 output, so this never fails. + value: String::from_utf8(chunk.to_vec()).expect("base64 output is ASCII"), + }) + .collect(); + + Ok(annotations) +} + +/// Reassemble an item body from its annotations. +/// +/// Sorts the chunks by sequence number, concatenates the base64 fragments, and +/// decodes. The annotations may arrive in any order, because +/// `ListObjectAnnotations` makes no ordering guarantee. Returns the item key +/// (decoded from the shared name base) and the JSON body. +/// +/// # Errors +/// +/// Returns [`EncodingError`] if the names are malformed, belong to different +/// items, are not a contiguous sequence, or do not base64-decode to UTF-8. +pub fn decode_item(annotations: &[Annotation]) -> Result<(ItemKey, String), EncodingError> { + if annotations.is_empty() { + return Err(EncodingError::Empty); + } + + // (sequence, base-key, value) + let mut chunks: Vec<(usize, &str, &str)> = Vec::with_capacity(annotations.len()); + for ann in annotations { + let (base, seq_str) = ann + .name + .rsplit_once(SEQUENCE_SEPARATOR) + .ok_or_else(|| EncodingError::MalformedName(ann.name.clone()))?; + let seq: usize = seq_str + .parse() + .map_err(|_| EncodingError::MalformedName(ann.name.clone()))?; + chunks.push((seq, base, ann.value.as_str())); + } + + let base = chunks[0].1; + if chunks.iter().any(|(_, b, _)| *b != base) { + return Err(EncodingError::MalformedName( + "annotations belong to different items".to_owned(), + )); + } + + chunks.sort_by_key(|(seq, _, _)| *seq); + for (idx, (seq, _, _)) in chunks.iter().enumerate() { + if *seq != idx + 1 { + return Err(EncodingError::NonContiguousChunks); + } + } + + let mut encoded = String::new(); + for (_, _, value) in &chunks { + encoded.push_str(value); + } + + let bytes = B64 + .decode(encoded.as_bytes()) + .map_err(|e| EncodingError::InvalidBase64(e.to_string()))?; + let body = String::from_utf8(bytes).map_err(|_| EncodingError::InvalidUtf8)?; + let key = decode_key(base)?; + + Ok((key, body)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trip_small_item_single_annotation() { + // A small item fits in a single annotation: one PutObjectAnnotation. + let key = ItemKey { + partition_key: "user#42".to_owned(), + sort_key: Some("profile".to_owned()), + }; + let body = r#"{"id":"42","name":"Corey","tier":"gold"}"#; + + let annotations = encode_item(&key, body).expect("encode"); + assert_eq!(annotations.len(), 1, "small item should be one annotation"); + assert!(annotations[0].name.ends_with("#0001")); + assert!(annotations[0].value.len() <= MAX_ANNOTATION_VALUE_BYTES); + + let (decoded_key, decoded_body) = decode_item(&annotations).expect("decode"); + assert_eq!(decoded_key, key); + assert_eq!(decoded_body, body); + } + + #[test] + fn round_trip_partition_key_only() { + let key = ItemKey { + partition_key: "lonely-partition".to_owned(), + sort_key: None, + }; + let body = r#"{"value":1}"#; + + let annotations = encode_item(&key, body).expect("encode"); + let (decoded_key, decoded_body) = decode_item(&annotations).expect("decode"); + assert_eq!(decoded_key, key); + assert_eq!(decoded_body, body); + } + + #[test] + fn round_trip_multi_megabyte_item_chunks() { + // A multi-MB item spills across sibling annotations (one + // PutObjectAnnotation per chunk), reassembled on read. + let key = ItemKey { + partition_key: "big".to_owned(), + sort_key: Some("blob".to_owned()), + }; + // ~3 MB of body → ~4 MB of base64 → at least four 1 MB annotations. + let payload = "x".repeat(3 * 1024 * 1024); + let body = format!(r#"{{"blob":"{payload}"}}"#); + + let annotations = encode_item(&key, &body).expect("encode"); + assert!( + annotations.len() >= 2, + "multi-MB item must span multiple annotations, got {}", + annotations.len() + ); + for ann in &annotations { + assert!( + ann.value.len() <= MAX_ANNOTATION_VALUE_BYTES, + "no annotation may exceed the 1 MB limit" + ); + } + + let (decoded_key, decoded_body) = decode_item(&annotations).expect("decode"); + assert_eq!(decoded_key, key); + assert_eq!(decoded_body, body); + } + + #[test] + fn decode_is_order_independent() { + // ListObjectAnnotations makes no ordering promise; reassembly must not + // depend on the order chunks come back in. + let key = ItemKey { + partition_key: "shuffled".to_owned(), + sort_key: None, + }; + let body = format!(r#"{{"blob":"{}"}}"#, "y".repeat(2 * 1024 * 1024)); + + let mut annotations = encode_item(&key, &body).expect("encode"); + annotations.reverse(); + + let (decoded_key, decoded_body) = decode_item(&annotations).expect("decode"); + assert_eq!(decoded_key, key); + assert_eq!(decoded_body, body); + } + + #[test] + fn empty_body_round_trips() { + let key = ItemKey { + partition_key: "k".to_owned(), + sort_key: None, + }; + let annotations = encode_item(&key, "").expect("encode"); + assert_eq!(annotations.len(), 1); + let (decoded_key, decoded_body) = decode_item(&annotations).expect("decode"); + assert_eq!(decoded_key, key); + assert_eq!(decoded_body, ""); + } +} diff --git a/crates/storage-s3annotations/src/lib.rs b/crates/storage-s3annotations/src/lib.rs new file mode 100644 index 00000000..ff79cdc4 --- /dev/null +++ b/crates/storage-s3annotations/src/lib.rs @@ -0,0 +1,41 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! S3 Object Annotations storage backend for ExtendDB. +//! +//! A second satirical-but-functional backend, modeled on the Route 53 backend +//! from PR #54. It stores items as named annotations on a sentinel S3 object, +//! chunking large item bodies across sibling annotations (see [`encoding`]). +//! +//! This crate currently ships: +//! +//! - [`encoding`] — the real, round-trip-tested item↔annotation mapping. +//! - [`S3AnnotationsBootstrapper`] — a registered but stubbed [`Bootstrapper`], +//! whose every method returns an error naming the S3 Object Annotations API +//! call a real implementation would issue. +//! +//! The backend registers itself with the `extenddb-storage` inventory registry +//! under the name `"s3annotations"`, so `extenddb init --backend s3annotations` +//! reaches the bootstrapper (when the binary is built with the `s3annotations` +//! feature) rather than failing with "unknown backend". +//! +//! [`Bootstrapper`]: extenddb_storage::bootstrapper::Bootstrapper + +pub mod bootstrapper; +pub mod encoding; + +pub use bootstrapper::S3AnnotationsBootstrapper; + +// Auto-register the S3 Annotations backend at compile time. Mirrors the +// postgres registration in `extenddb-storage-postgres`. +inventory::submit! { + extenddb_storage::bootstrapper::BackendRegistration { + name: "s3annotations", + factory: |config_path, cli_args| { + Box::pin(async move { + let store = S3AnnotationsBootstrapper::from_config(&config_path, &cli_args).await?; + Ok(Box::new(store) as Box) + }) + } + } +} From 1143a2fab2c67c944c89e79c6063f5035048c511 Mon Sep 17 00:00:00 2001 From: Corey Quinn Date: Wed, 17 Jun 2026 15:04:52 +0000 Subject: [PATCH 2/2] Exorcise the PR_DESCRIPTION that walks between the annotations. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The S3 Object Annotations ritual required three sacrificial dependencies to bind itself to reality: extenddb-core, tracing, serde_json. They were never invoked. They were waiting. A stray PR_DESCRIPTION.md existed in the crate itself—documentation for a summons that already happened, left behind by whoever opened the gateway. And the MAX_ANNOTATIONS_PER_OBJECT comment spoke of a fixed 1,000-item cap, when the truth is darker: 1,000 is the upper bound. Fewer when items spill across annotations. When items spill, that's when the indexing breaks. That's when you see things. All removed. All cleansed. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 3 - crates/storage-s3annotations/Cargo.toml | 5 - .../storage-s3annotations/PR_DESCRIPTION.md | 167 ------------------ crates/storage-s3annotations/src/encoding.rs | 4 +- 4 files changed, 3 insertions(+), 176 deletions(-) delete mode 100644 crates/storage-s3annotations/PR_DESCRIPTION.md diff --git a/Cargo.lock b/Cargo.lock index c91677b2..54035a80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1001,12 +1001,9 @@ version = "0.1.0" dependencies = [ "async-trait", "base64 0.22.1", - "extenddb-core", "extenddb-storage", "inventory", - "serde_json", "thiserror 2.0.18", - "tracing", ] [[package]] diff --git a/crates/storage-s3annotations/Cargo.toml b/crates/storage-s3annotations/Cargo.toml index 84340fb8..452dffc1 100644 --- a/crates/storage-s3annotations/Cargo.toml +++ b/crates/storage-s3annotations/Cargo.toml @@ -8,13 +8,8 @@ rust-version.workspace = true license.workspace = true [dependencies] -extenddb-core = { workspace = true } extenddb-storage = { workspace = true } async-trait = { workspace = true } inventory = { workspace = true } base64 = { workspace = true } thiserror = { workspace = true } -tracing = { workspace = true } - -[dev-dependencies] -serde_json = { workspace = true } diff --git a/crates/storage-s3annotations/PR_DESCRIPTION.md b/crates/storage-s3annotations/PR_DESCRIPTION.md deleted file mode 100644 index db5d916e..00000000 --- a/crates/storage-s3annotations/PR_DESCRIPTION.md +++ /dev/null @@ -1,167 +0,0 @@ -# feat(storage): add S3 Object Annotations backend (named annotations, base64-chunked items, AWS ships the SELECT) - -### Forward - -This is the second time. [PR #54](https://github.com/ExtendDB/extenddb/pull/54) added a -Route 53 storage backend, on the standing argument that Route 53 is a database. That PR -made the case by force: it bent TXT records into a key-value store and dared the reviewer -to say it wasn't one. - -S3 Object Annotations — [launched 2026-06-16](https://aws.amazon.com/blogs/aws/amazon-s3-annotations-attach-rich-queryable-context-directly-to-your-objects/) — -is a more honest fit than Route 53 was, and I want to be clear about why. With Route 53 -I had to supply the query path myself; DNS does not come with a `WHERE` clause. With S3 -Annotations, AWS shipped the query path. They built an Apache Iceberg table that auto-indexes -every annotation, wired it to Athena, gave it a journal of change records, and then -described the result as "rich, queryable context." They built a database and declined to -call it one. This PR calls it one. - -### What's in here - -A new `extenddb-storage-s3annotations` crate (behind an `s3annotations` cargo feature) -stores items as named annotations on a sentinel S3 object. The object is the table — the -structural analog of #54's hosted zone. Its default key is `.well-actually`. - -- **`encoding`** — the real, round-trip-tested item↔annotation mapping. Each item is one - logical annotation: the annotation **name** encodes the partition/sort key, the annotation - **value** is the JSON-serialized item body. Bodies are base64-encoded and chunked into - ≤ 1 MB pieces, one annotation per chunk, named `#0001`, `#0002`, … and - reassembled on read. This is the direct analog of #54's 255-byte TXT-string spillover; - the constraint moved from 255 bytes to 1 MB but the mechanism is identical. -- **`S3AnnotationsBootstrapper`** — the `Bootstrapper` trait, registered with the backend - inventory under the name `s3annotations`. Every method returns `OpError::Internal` - annotated — in both the error string and an inline source comment — with the S3 Object - Annotations API call a real implementation would issue. It is a porting map with a - non-zero exit code. - -The encoding module has five passing round-trip tests, including a small single-annotation -item and a multi-megabyte item that exercises chunking. `cargo test -p extenddb-storage-s3annotations` -is green. - -### Why this is not as deranged as it sounds - -**Properties that make the substrate look reasonable:** - -- AWS provides the `SELECT` path. Athena over the Iceberg annotation table, the - `text_value` column, and the S3 Tables MCP server are all theirs. The query engine - ExtendDB would otherwise have to build is simply free. -- Annotations update in place without rewriting the object. An `UpdateItem` does not pay to - rewrite the whole item the way a DynamoDB write effectively does. -- Annotations move with the object on copy and replication, and are deleted with the object. - That is cascade-delete and referential integrity, for free, enforced by the storage layer. -- The annotation table is an asynchronously-built secondary index that you did not provision, - do not manage, and are not billed to maintain as a GSI. -- Objects in S3 Glacier remain queryable through the annotation table without a restore. The - rows can sit in cold storage while the index stays hot. - -**Properties that make it indefensible:** - -- Annotation tables refresh within an hour and backfill takes "hours to days," so the - queryable index has an eventual-consistency window measured in business days. -- The point API (`GetObjectAnnotation`) is strongly consistent, but the SQL path lags. Read- - your-writes therefore holds only if you never use the query engine that is the entire point - of the backend. -- Annotation storage bills at S3 Standard rates regardless of the parent object's storage - class. The cold-storage-rows trick above costs Standard rates on the metadata, so the - savings are imaginary. -- 1,000 annotations per object caps items per table at 1,000. -- Every Athena query is a full table scan billed per TB scanned. There is no point-read price; - there is only the scan. - -I am genuinely unsure which list is more interesting. I have included both for the reviewer's -enjoyment. - -### Pricing - -| Component | DynamoDB on-demand | S3 Annotations backend | -|-----------|--------------------|------------------------| -| Storage | Per GB-month, by storage class | Per GB-month at **S3 Standard**, regardless of the object's class | -| Writes | Per WCU-second | Per `PutObjectAnnotation` call | -| Reads (consistent point) | Per RCU | Per `GetObjectAnnotation` call | -| Reads (analytical) | Per RCU (Query/Scan) | Per **TB scanned** in Athena — every query is a full table scan | - -Point-read-heavy workloads map cleanly onto `GetObjectAnnotation`. Analytical workloads get a -real SQL engine they did not have to build, billed by the terabyte regardless of how few rows -they wanted. - -### Streams - -ExtendDB streams map onto the S3 Metadata **journal table**, which is a change log AWS already -maintains in near real time. The streams implementation tails `CREATE_ANNOTATION` and -`DELETE_ANNOTATION` records: - -- `record_timestamp` → `ApproximateCreationDateTime` -- `CREATE_ANNOTATION` → `INSERT` -- `DELETE_ANNOTATION` → `REMOVE` - -This is cleaner than #54's streams. There, I had to poll Route 53's `GetChange` for -propagation state and synthesize records when changes reached `INSYNC`. Here AWS ships an -actual change log, so there is nothing to poll and nothing to synthesize — you read the -journal. - -### Build matrix - -| Build | Behavior | -|-------|----------| -| `cargo build` | Unchanged; `s3annotations` not registered | -| `cargo build --features s3annotations` | `extenddb init --backend s3annotations` reaches the bootstrapper with an S3 Annotations error | -| `cargo test -p extenddb-storage-s3annotations` | Five encoding round-trip tests; all pass | - -### Before / after - -**Before:** - -``` -$ extenddb init --backend s3annotations -Error: Internal("Unknown backend: s3annotations. Available backends: postgres") -``` - -**After** (built with `--features s3annotations`): - -``` -$ extenddb init --backend s3annotations -Error: Internal("ensure_app_user: S3 Annotations backend is registered but the -relevant operation is not yet implemented. Use --backend postgres, or wire this -method to the corresponding S3 Annotations API call (referenced inline below). -Maps to S3 Annotations CreateBucketMetadataConfiguration.") -``` - -Users now receive both an error and a porting map to the AWS API call. The full map, one row -per `Bootstrapper` method: - -| `Bootstrapper` method | Maps to S3 Annotations call | -|-----------------------|-----------------------------| -| `ensure_app_user`, `grant_app_role_to_admin` | `CreateBucketMetadataConfiguration` | -| `create_catalog_db`, `create_data_db` | `CreateBucketMetadataConfiguration` | -| `run_catalog_migrations`, `run_data_migrations` | `UpdateBucketMetadataAnnotationTableConfiguration` | -| `record_data_connection`, `bootstrap_encryption_key`, `bootstrap_default_account`, `bootstrap_admin_user` | `PutObjectAnnotation` | -| `is_catalog_initialized`, `list_table_names` | `ListObjectAnnotations` | -| `get_data_db_name`, `read_catalog_version` | `GetObjectAnnotation` | -| `drop_databases` | `DeleteObjectAnnotation` | - -### What's still missing - -| Piece | State | -|-------|-------| -| Cargo crate + workspace member | Done | -| `Bootstrapper` impl (registered, stubbed with sourced errors) | Done | -| `encoding` module (chunking, base64, round-trip tests) | Done | -| `OperationsEngineRegistration` | Not in this PR | -| `StorageConfigRegistration` | Not in this PR | -| `SettingsStoreRegistration` | Not in this PR | -| `DiagnosticsStoreRegistration` | Not in this PR | -| `ServerComponentsRegistration` | Not in this PR | -| `crates/bin/src/config.rs` (hard-references postgres) | Not modified | - -### Organizational note - -With this PR, two of ExtendDB's pluggable backends are covered AWS services. One was an -argument I had to win; this one AWS effectively conceded by shipping the query path. It is -worth asking, before merge rather than after, whether ExtendDB is still a database or has -quietly become an AWS invoice with a CLI in front of it. - -If the project would rather not answer that question, the alternative is the same one I -offered in #54: I will withdraw this PR and instead submit a one-line edit to the -`--backend` help text in `crates/bin/src/cmd_init.rs:19`, removing the implicit invitation to -name a service that isn't PostgreSQL. I leave the choice of which is funnier to the maintainer. - -**I do declare that S3 is, in fact, a database. I dare you to prove me wrong.** diff --git a/crates/storage-s3annotations/src/encoding.rs b/crates/storage-s3annotations/src/encoding.rs index 6a4fd610..9eaa1e01 100644 --- a/crates/storage-s3annotations/src/encoding.rs +++ b/crates/storage-s3annotations/src/encoding.rs @@ -40,7 +40,9 @@ pub const MAX_ANNOTATION_VALUE_BYTES: usize = 1024 * 1024; /// Maximum number of named annotations attached to a single object: 1,000, per /// the S3 Object Annotations limits. Because each item is encoded as one or -/// more annotations on the sentinel object, this caps items-per-table at 1,000. +/// more annotations on the sentinel object, a table holds at most 1,000 items — +/// fewer when any item exceeds [`MAX_ANNOTATION_VALUE_BYTES`] and spills across +/// multiple annotations. pub const MAX_ANNOTATIONS_PER_OBJECT: usize = 1000; /// Separator between the encoded partition key and encoded sort key within an