diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 3d3d892a..6ed6b252 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -20,7 +20,7 @@ jobs: - uses: taiki-e/install-action@v2 with: - tool: mdbook,mdbook-linkcheck + tool: mdbook,mdbook-linkcheck,mdbook-admonish - name: Build Book run: mdbook build diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee19f913..5f2cca48 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: - uses: taiki-e/install-action@v2 with: - tool: mdbook,mdbook-linkcheck + tool: mdbook,mdbook-linkcheck,mdbook-admonish - name: Build Book run: mdbook build \ No newline at end of file diff --git a/book.toml b/book.toml index 56ee6498..40c34233 100644 --- a/book.toml +++ b/book.toml @@ -5,6 +5,11 @@ multilingual = false src = "doc" title = "Corrosion" +[preprocessor.admonish] +command = "mdbook-admonish" +assets_version = "2.0.2" # do not edit: managed by `mdbook-admonish install` + [output.html] +additional-css = ["./doc/mdbook-admonish.css"] -[output.linkcheck] \ No newline at end of file +[output.linkcheck] diff --git a/doc/SUMMARY.md b/doc/SUMMARY.md index 63803abc..010bfadd 100644 --- a/doc/SUMMARY.md +++ b/doc/SUMMARY.md @@ -3,9 +3,21 @@ [Introduction](intro.md) # User guide -- [Operations Guide]() +- [Authorization]() +- [Backup / Restore]() +- [CRDTs](crdts.md) +- [Encryption]() +- [Gossip]() +- [Schema](schema.md) +- [Subscriptions]() +- [Synchronization]() +- [Templates]() # Reference +- [API]() + - [POST /v1/exec]() + - [POST /v1/query]() + - [POST /v1/subscriptions]() - [Command-line Interface](cli/README.md) - [agent]() - [backup](cli/backup.md) @@ -18,7 +30,7 @@ - [template]() - [tls](cli/tls.md) - [Configuration](config/README.md) - - [db]() + - [db](config/db.md) - [gossip](config/gossip.md) - [api]() - [admin]() diff --git a/doc/config/db.md b/doc/config/db.md new file mode 100644 index 00000000..88b3a837 --- /dev/null +++ b/doc/config/db.md @@ -0,0 +1,27 @@ +# The `[db]` configuration + +The `[db]` block configures the Corrosion's SQLite database. + +### Required fields + +#### `db.path` + +Path of the sqlite3 database. + +```toml +[db] +path = "/var/lib/corrosion/db.sqlite" +``` + +### Optional fields + +#### `db.schema_paths` + +Array of paths where [schema]() .sql files are present. + +```toml +[db] +schema_paths = ["/etc/corrosion/schema", "/path/to/table_name.sql"] +``` + +If a directory is specified, all .sql files will be loaded. \ No newline at end of file diff --git a/doc/config/gossip.md b/doc/config/gossip.md index b0e18f37..16018a85 100644 --- a/doc/config/gossip.md +++ b/doc/config/gossip.md @@ -38,7 +38,9 @@ bootstrap = ["my-fly-app.internal:3333@[fdaa::3]:53"] Allows using QUIC without encryption. The only reason to set this to `true` is if you're running a toy cluster or if the underlying transport is already handling cryptography (such as WireGuard) AND authorization is bound by the network (such is the case for a [Fly.io](https://fly.io) app's private network). +```admonish warning It's highly recommended to use the `gossip.tls` configuration block to setup encryption and `gossip.tls.client` to setup authorization. +``` #### `gossip.max_mtu` diff --git a/doc/crdts.md b/doc/crdts.md new file mode 100644 index 00000000..8b6c715f --- /dev/null +++ b/doc/crdts.md @@ -0,0 +1,25 @@ +# Conflict-free Replicated Data Types + +## What's a CRDT? + +[About CRDTs](https://crdt.tech/#:~:text=Conflict%2Dfree%20Replicated%20Data%20Types%20(CRDTs)%20are%20used%20in,merged%20into%20a%20consistent%20state.) + +> Conflict-free Replicated Data Types (CRDTs) are used in systems with optimistic replication, where they take care of conflict resolution. CRDTs ensure that, no matter what data modifications are made on different replicas, the data can always be merged into a consistent state. This merge is performed automatically by the CRDT, without requiring any special conflict resolution code or user intervention. + +## cr-sqlite + +Corrosion uses the [`cr-sqlite` SQLite extension](https://github.com/vlcn-io/cr-sqlite) to accomplish its multi-writer and eventual consistency promise. Here's a [short intro](https://vlcn.io/docs/cr-sqlite/intro) about how it generally works. The rest of this section assumes some knowledge of cr-sqlite. + +`cr-sqlite` provides functions to mark tables, in a SQLite database, as backed by CRDTs. These include Causal-Length ([pdf paper](https://dl.acm.org/doi/pdf/10.1145/3380787.3393678)) and Last-Write-Wins (LWW). + +### Basics + +With the extension loaded, writes to CRDT-backed tables will trigger insertions in internal tables for each column in a row. + +An aggregate view table is available as `crsql_changes`. By `SELECT`-ing from this table and `INSERT`-ing into it, it's possible to distribute changes to a cluster. Each set of changes (a transaction produces a single set of changes) gets a new `db_version`, unique to the database. + +## Corrosion and cr-sqlite + +Corrosion executes transactions by processing requests made to its client HTTP API. Each transaction triggers 1+ broadcast (big changesets are chunked). Each change is serialized in an efficient format and sent to ~random members of the cluster. + +The main caveat of this approach is: **writes to the database all have to go through Corrosion**. If a sqlite client were to issue writes w/ or w/o the proper extension loaded, then data would become inconsistent for CRDT-backed tables. \ No newline at end of file diff --git a/doc/mdbook-admonish.css b/doc/mdbook-admonish.css new file mode 100644 index 00000000..c3e9869e --- /dev/null +++ b/doc/mdbook-admonish.css @@ -0,0 +1,353 @@ +@charset "UTF-8"; +:root { + --md-admonition-icon--note: + url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--abstract: + url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--info: + url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--tip: + url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--success: + url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--question: + url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--warning: + url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--failure: + url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--danger: + url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--bug: + url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--example: + url("data:image/svg+xml;charset=utf-8,"); + --md-admonition-icon--quote: + url("data:image/svg+xml;charset=utf-8,"); + --md-details-icon: + url("data:image/svg+xml;charset=utf-8,"); +} + +:is(.admonition) { + display: flow-root; + margin: 1.5625em 0; + padding: 0 1.2rem; + color: var(--fg); + page-break-inside: avoid; + background-color: var(--bg); + border: 0 solid black; + border-inline-start-width: 0.4rem; + border-radius: 0.2rem; + box-shadow: 0 0.2rem 1rem rgba(0, 0, 0, 0.05), 0 0 0.1rem rgba(0, 0, 0, 0.1); +} +@media print { + :is(.admonition) { + box-shadow: none; + } +} +:is(.admonition) > * { + box-sizing: border-box; +} +:is(.admonition) :is(.admonition) { + margin-top: 1em; + margin-bottom: 1em; +} +:is(.admonition) > .tabbed-set:only-child { + margin-top: 0; +} +html :is(.admonition) > :last-child { + margin-bottom: 1.2rem; +} + +a.admonition-anchor-link { + display: none; + position: absolute; + left: -1.2rem; + padding-right: 1rem; +} +a.admonition-anchor-link:link, a.admonition-anchor-link:visited { + color: var(--fg); +} +a.admonition-anchor-link:link:hover, a.admonition-anchor-link:visited:hover { + text-decoration: none; +} +a.admonition-anchor-link::before { + content: "ยง"; +} + +:is(.admonition-title, summary.admonition-title) { + position: relative; + min-height: 4rem; + margin-block: 0; + margin-inline: -1.6rem -1.2rem; + padding-block: 0.8rem; + padding-inline: 4.4rem 1.2rem; + font-weight: 700; + background-color: rgba(68, 138, 255, 0.1); + display: flex; +} +:is(.admonition-title, summary.admonition-title) p { + margin: 0; +} +html :is(.admonition-title, summary.admonition-title):last-child { + margin-bottom: 0; +} +:is(.admonition-title, summary.admonition-title)::before { + position: absolute; + top: 0.625em; + inset-inline-start: 1.6rem; + width: 2rem; + height: 2rem; + background-color: #448aff; + mask-image: url('data:image/svg+xml;charset=utf-8,'); + -webkit-mask-image: url('data:image/svg+xml;charset=utf-8,'); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-size: contain; + content: ""; +} +:is(.admonition-title, summary.admonition-title):hover a.admonition-anchor-link { + display: initial; +} + +details.admonition > summary.admonition-title::after { + position: absolute; + top: 0.625em; + inset-inline-end: 1.6rem; + height: 2rem; + width: 2rem; + background-color: currentcolor; + mask-image: var(--md-details-icon); + -webkit-mask-image: var(--md-details-icon); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-size: contain; + content: ""; + transform: rotate(0deg); + transition: transform 0.25s; +} +details[open].admonition > summary.admonition-title::after { + transform: rotate(90deg); +} + +:is(.admonition):is(.note) { + border-color: #448aff; +} + +:is(.note) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(68, 138, 255, 0.1); +} +:is(.note) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #448aff; + mask-image: var(--md-admonition-icon--note); + -webkit-mask-image: var(--md-admonition-icon--note); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.abstract, .summary, .tldr) { + border-color: #00b0ff; +} + +:is(.abstract, .summary, .tldr) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(0, 176, 255, 0.1); +} +:is(.abstract, .summary, .tldr) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #00b0ff; + mask-image: var(--md-admonition-icon--abstract); + -webkit-mask-image: var(--md-admonition-icon--abstract); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.info, .todo) { + border-color: #00b8d4; +} + +:is(.info, .todo) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(0, 184, 212, 0.1); +} +:is(.info, .todo) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #00b8d4; + mask-image: var(--md-admonition-icon--info); + -webkit-mask-image: var(--md-admonition-icon--info); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.tip, .hint, .important) { + border-color: #00bfa5; +} + +:is(.tip, .hint, .important) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(0, 191, 165, 0.1); +} +:is(.tip, .hint, .important) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #00bfa5; + mask-image: var(--md-admonition-icon--tip); + -webkit-mask-image: var(--md-admonition-icon--tip); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.success, .check, .done) { + border-color: #00c853; +} + +:is(.success, .check, .done) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(0, 200, 83, 0.1); +} +:is(.success, .check, .done) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #00c853; + mask-image: var(--md-admonition-icon--success); + -webkit-mask-image: var(--md-admonition-icon--success); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.question, .help, .faq) { + border-color: #64dd17; +} + +:is(.question, .help, .faq) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(100, 221, 23, 0.1); +} +:is(.question, .help, .faq) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #64dd17; + mask-image: var(--md-admonition-icon--question); + -webkit-mask-image: var(--md-admonition-icon--question); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.warning, .caution, .attention) { + border-color: #ff9100; +} + +:is(.warning, .caution, .attention) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(255, 145, 0, 0.1); +} +:is(.warning, .caution, .attention) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #ff9100; + mask-image: var(--md-admonition-icon--warning); + -webkit-mask-image: var(--md-admonition-icon--warning); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.failure, .fail, .missing) { + border-color: #ff5252; +} + +:is(.failure, .fail, .missing) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(255, 82, 82, 0.1); +} +:is(.failure, .fail, .missing) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #ff5252; + mask-image: var(--md-admonition-icon--failure); + -webkit-mask-image: var(--md-admonition-icon--failure); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.danger, .error) { + border-color: #ff1744; +} + +:is(.danger, .error) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(255, 23, 68, 0.1); +} +:is(.danger, .error) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #ff1744; + mask-image: var(--md-admonition-icon--danger); + -webkit-mask-image: var(--md-admonition-icon--danger); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.bug) { + border-color: #f50057; +} + +:is(.bug) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(245, 0, 87, 0.1); +} +:is(.bug) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #f50057; + mask-image: var(--md-admonition-icon--bug); + -webkit-mask-image: var(--md-admonition-icon--bug); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.example) { + border-color: #7c4dff; +} + +:is(.example) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(124, 77, 255, 0.1); +} +:is(.example) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #7c4dff; + mask-image: var(--md-admonition-icon--example); + -webkit-mask-image: var(--md-admonition-icon--example); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +:is(.admonition):is(.quote, .cite) { + border-color: #9e9e9e; +} + +:is(.quote, .cite) > :is(.admonition-title, summary.admonition-title) { + background-color: rgba(158, 158, 158, 0.1); +} +:is(.quote, .cite) > :is(.admonition-title, summary.admonition-title)::before { + background-color: #9e9e9e; + mask-image: var(--md-admonition-icon--quote); + -webkit-mask-image: var(--md-admonition-icon--quote); + mask-repeat: no-repeat; + -webkit-mask-repeat: no-repeat; + mask-size: contain; + -webkit-mask-repeat: no-repeat; +} + +.navy :is(.admonition) { + background-color: var(--sidebar-bg); +} + +.ayu :is(.admonition), .coal :is(.admonition) { + background-color: var(--theme-hover); +} + +.rust :is(.admonition) { + background-color: var(--sidebar-bg); + color: var(--sidebar-fg); +} +.rust .admonition-anchor-link:link, .rust .admonition-anchor-link:visited { + color: var(--sidebar-fg); +} diff --git a/doc/schema.md b/doc/schema.md new file mode 100644 index 00000000..219490e3 --- /dev/null +++ b/doc/schema.md @@ -0,0 +1,29 @@ +# Schema + +Corrosion's schema definition happens via files each representing one or more tables, written in SQL (SQLite-flavored). This is done through `CREATE TABLE` and `CREATE INDEX` exclusively! + +Manual migrations are not supported (yet). When schema files change, Corrosion can be reloaded (or restarted) and it will compute a diff between the old and new schema and make the changes. + +Any destructive actions on the table schemas are ignored / prohibited. This includes removing a table definition entirely or removing a column from a table. Indexes can be removed or added. + +## Constraints + +- Only `CREATE TABLE` and `CREATE INDEX` are allowed +- No unique indexes allowed (except for the default primary key unique index that does not need to be created) +- Non-nullable columns require a default value + - This is a cr-sqlite constraint, but in practice w/ Corrosion: it does not matter. Entire changes will be applied all at once and no fields will be missing. + - If table schemas are modified, then a default value is definitely required. + +## Example + +```sql +-- /etc/corrosion/schema/apps.sql + +CREATE TABLE apps ( + id INT PRIMARY KEY, + name TEXT NOT NULL DEFAULT "", + user_id INT NOT NULL DEFAULT 0, +); + +CREATE INDEX apps_user_id ON apps (user_id); +``` \ No newline at end of file