3.0.0

.github/workflows/release.yml

@@ -8,7 +8,7 @@ jobs:
   publish:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - uses: katyo/publish-crates@v2
         with:
           registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}

.github/workflows/test.yml

@@ -18,15 +18,15 @@ jobs:
       matrix:
         rust_version:
           - stable
-          - "1.76.0" # MSRV
+          - "1.87.0" # MSRV
         os:
           - ubuntu-latest
           - windows-latest
           - macos-latest
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ matrix.rust_version }}
@@ -38,9 +38,6 @@ jobs:
         uses: Swatinem/rust-cache@v2
         with:
           prefix-key: ${{ runner.os }}-cargo
-          workspaces: >
-            . -> target
-            examples/kv -> target
       - name: Install cargo-all-features
         run: cargo install cargo-all-features
       - uses: taiki-e/install-action@nextest
@@ -53,9 +50,7 @@ jobs:
       - name: Run tests
         run: cargo nextest run --all-features
       - name: Run doc tests
-        run: cargo test --doc
-      - name: Build & test LSM examples
-        run: node compile_examples.mjs
+        run: cargo test --doc --features lz4
   cross:
     timeout-minutes: 15
     name: cross
@@ -79,4 +74,4 @@ jobs:
       - name: cross test
         run: |
           cargo install cross
-          cross test -r --features lz4,miniz --target ${{ matrix.target }}
+          cross test -r --features lz4 --target ${{ matrix.target }}

.gitignore

@@ -18,3 +18,9 @@ Cargo.lock
 .bench
 
 mutants*
+profile.json
+fuzz*/**/out*
+
+microbench/**/data.jsonl
+microbench/**/*.svg
+old

Cargo.toml

@@ -2,14 +2,14 @@
 name = "lsm-tree"
 description = "A K.I.S.S. implementation of log-structured merge trees (LSM-trees/LSMTs)"
 license = "MIT OR Apache-2.0"
-version = "2.10.4"
+version = "3.0.0-pre.1"
 edition = "2021"
-rust-version = "1.76.0"
+rust-version = "1.87.0"
 readme = "README.md"
 include = ["src/**/*", "LICENSE-APACHE", "LICENSE-MIT", "README.md"]
 repository = "https://github.com/fjall-rs/lsm-tree"
 homepage = "https://github.com/fjall-rs/lsm-tree"
-keywords = ["database", "lsmt", "lsm", "rocksdb", "leveldb"]
+keywords = ["lsmt", "lsm", "rocksdb", "leveldb", "key-value"]
 categories = ["data-structures", "database-implementations", "algorithms"]
 
 [lib]
@@ -19,40 +19,34 @@ path = "src/lib.rs"
 [features]
 default = []
 lz4 = ["dep:lz4_flex"]
-miniz = ["dep:miniz_oxide"]
-bytes = ["value-log/bytes"]
+bytes_1 = ["dep:bytes"]
+metrics = []
+use_unsafe = []         # TODO: 3.0.0 remove
 
 [dependencies]
-byteorder = "1.5.0"
+bytes = { version = "1", optional = true }
+byteorder = { package = "byteorder-lite", version = "0.1.0" }
+byteview = "~0.8.0"
 crossbeam-skiplist = "0.1.3"
-double-ended-peekable = "0.1.0"
 enum_dispatch = "0.3.13"
-guardian = "1.1.0"
 interval-heap = "0.0.5"
-log = "0.4.22"
-lz4_flex = { version = "0.11.3", optional = true, default-features = false }
-miniz_oxide = { version = "0.8.0", optional = true }
-path-absolutize = "3.1.1"
-quick_cache = { version = "0.6.5", default-features = false, features = [] }
-rustc-hash = "2.0.0"
-self_cell = "1.0.4"
-tempfile = "3.12.0"
-value-log = { version = "~1.9", default-features = false, features = [] }
+log = "0.4.27"
+lz4_flex = { version = "0.11.5", optional = true, default-features = false }
+quick_cache = { version = "0.6.16", default-features = false, features = [] }
+rustc-hash = "2.1.1"
+self_cell = "1.2.0"
+tempfile = "3.20.0"
 varint-rs = "2.2.0"
-xxhash-rust = { version = "0.8.12", features = ["xxh3"] }
+xxhash-rust = { version = "0.8.15", features = ["xxh3"] }
+sfa = "~0.0.2"
 
 [dev-dependencies]
 criterion = { version = "0.5.1", features = ["html_reports"] }
 fs_extra = "1.3.0"
 nanoid = "0.4.0"
-rand = "0.9.0"
-test-log = "0.2.16"
-
-# half 2.5.0 has MSRV 1.81
-half = "=2.4.0"
-
-# rayon has MSRV 1.80
-rayon-core = "=1.12.1"
+rand = "0.9.2"
+strum = { version = "0.27.2", features = ["derive"] }
+test-log = "0.2.18"
 
 [package.metadata.cargo-all-features]
 denylist = []
@@ -85,13 +79,13 @@ required-features = []
 name = "block"
 harness = false
 path = "benches/block.rs"
-required-features = ["lz4", "miniz"]
+required-features = ["lz4"]
 
 [[bench]]
 name = "tree"
 harness = false
 path = "benches/tree.rs"
-required-features = ["lz4", "miniz"]
+required-features = ["lz4"]
 
 [[bench]]
 name = "level_manifest"

README.md

@@ -5,7 +5,7 @@
 [![CI](https://github.com/fjall-rs/lsm-tree/actions/workflows/test.yml/badge.svg)](https://github.com/fjall-rs/lsm-tree/actions/workflows/test.yml)
 [![docs.rs](https://img.shields.io/docsrs/lsm-tree?color=green)](https://docs.rs/lsm-tree)
 [![Crates.io](https://img.shields.io/crates/v/lsm-tree?color=blue)](https://crates.io/crates/lsm-tree)
-![MSRV](https://img.shields.io/badge/MSRV-1.76.0-blue)
+![MSRV](https://img.shields.io/badge/MSRV-1.87.0-blue)
 [![dependency status](https://deps.rs/repo/github/fjall-rs/lsm-tree/status.svg)](https://deps.rs/repo/github/fjall-rs/lsm-tree)
 
 A K.I.S.S. implementation of log-structured merge trees (LSM-trees/LSMTs) in Rust.
@@ -19,21 +19,24 @@ A K.I.S.S. implementation of log-structured merge trees (LSM-trees/LSMTs) in Rus
 
 This is the most feature-rich LSM-tree implementation in Rust! It features:
 
-- Thread-safe BTreeMap-like API
-- [99.9% safe](./UNSAFE.md) & stable Rust
-- Block-based tables with compression support
+- Thread-safe `BTreeMap`-like API
+- Mostly [safe](./UNSAFE.md) & 100% stable Rust
+- Block-based tables with compression support & prefix truncation
+  - Optional block hash indexes in data blocks for faster point lookups [[3]](#footnotes)
+  - Per-level filter/index block pinning configuration
 - Range & prefix searching with forward and reverse iteration
-- Size-tiered, (concurrent) Leveled and FIFO compaction
-- Multi-threaded flushing (immutable/sealed memtables)
-- Partitioned block index to reduce memory footprint and keep startup time short [[1]](#footnotes)
 - Block caching to keep hot data in memory
-- Bloom filters to increase point lookup performance
-- Snapshots (MVCC)
+- File descriptor caching with upper bound to reduce fopen calls
+- *AMQ* filters (currently Bloom filters) to improve point lookup performance
+- Multi-versioning of KVs, enabling snapshot reads
+- Optionally partitioned block index & filters for better cache efficiency [[1]](#footnotes)
+- Size-tiered, (concurrent) Leveled and FIFO compaction 
+- Multi-threaded flushing (immutable/sealed memtables)
 - Key-value separation (optional) [[2]](#footnotes)
 - Single deletion tombstones ("weak" deletion)
 
-Keys are limited to 65536 bytes, values are limited to 2^32 bytes. As is normal with any kind of storage
-engine, larger keys and values have a bigger performance impact.
+Keys are limited to 65536 bytes, values are limited to 2^32 bytes.
+As is normal with any kind of storage engine, larger keys and values have a bigger performance impact.
 
 ## Feature flags
 
@@ -43,12 +46,6 @@ Allows using `LZ4` compression, powered by [`lz4_flex`](https://github.com/PSeit
 
 *Disabled by default.*
 
-### miniz
-
-Allows using `DEFLATE/zlib` compression, powered by [`miniz_oxide`](https://github.com/Frommi/miniz_oxide).
-
-*Disabled by default.*
-
 ### bytes
 
 Uses [`bytes`](https://github.com/tokio-rs/bytes) as the underlying `Slice` type.
@@ -66,7 +63,7 @@ Future breaking changes will result in a major version bump and a migration path
 ## Run unit benchmarks
 
 ```bash
-cargo bench --features lz4,miniz
+cargo bench --features lz4
 ```
 
 ## License
@@ -80,3 +77,5 @@ All contributions are to be licensed as MIT OR Apache-2.0.
 [1] https://rocksdb.org/blog/2017/05/12/partitioned-index-filter.html
 
 [2] https://github.com/facebook/rocksdb/wiki/BlobDB
+
+[3] https://rocksdb.org/blog/2018/08/23/data-block-hash-index.html

UNSAFE.md

@@ -1,5 +1,28 @@
 # Unsafe usage
 
-Currently, the project itself only uses one **1** unsafe block (ignoring dependencies which are tested themselves separately):
+...
 
-- https://github.com/fjall-rs/lsm-tree/blob/2d8686e873369bd9c4ff2b562ed988c1cea38331/src/binary_search.rs#L23-L25
+## Run fuzz testing
+
+```bash
+cd fuzz/data_block
+mkdir in
+cat /dev/random | head -n 100 > in/input
+cargo afl build && cargo afl fuzz -i in -o out target/debug/data_block
+
+cd fuzz/index_block
+mkdir in
+cat /dev/random | head -n 100 > in/input
+cargo afl build && cargo afl fuzz -i in -o out target/debug/index_block
+
+cd fuzz/table_read
+mkdir in
+cat /dev/random | head -n 100 > in/input
+cargo afl build && cargo afl fuzz -i in -o out target/debug/table_read
+```
+
+## Run mutation testing
+
+```bash
+cargo-mutants mutants --test-tool=nextest
+```

benches/block.rs

@@ -2,12 +2,13 @@ use criterion::{criterion_group, criterion_main, Criterion};
 use lsm_tree::{
     coding::Encode,
     segment::{
-        block::{header::Header as BlockHeader, ItemSize},
+        block::{header::Header as BlockHeader, offset::BlockOffset, ItemSize},
         meta::CompressionType,
-        value_block::{BlockOffset, ValueBlock},
+        value_block::ValueBlock,
     },
     Checksum, InternalValue,
 };
+use rand::Rng;
 use std::io::Write;
 
 /* fn value_block_size(c: &mut Criterion) {
@@ -50,19 +51,11 @@ fn value_block_find(c: &mut Criterion) {
     for item_count in [10, 100, 1_000, 10_000] {
         let mut items = vec![];
 
-        for seqno in (0..(item_count - 2)).rev() {
+        for item in 0u64..item_count {
             items.push(InternalValue::from_components(
-                *b"a",
-                *b"a",
-                seqno,
-                lsm_tree::ValueType::Value,
-            ));
-        }
-        for seqno in (0..2).rev() {
-            items.push(InternalValue::from_components(
-                *b"b",
-                *b"b",
-                seqno,
+                item.to_be_bytes(),
+                b"",
+                0,
                 lsm_tree::ValueType::Value,
             ));
         }
@@ -78,22 +71,29 @@ fn value_block_find(c: &mut Criterion) {
             },
         };
 
+        let mut rng = rand::rng();
+
         group.bench_function(format!("{item_count} items (linear)"), |b| {
             b.iter(|| {
+                let needle = rng.random_range(0..item_count).to_be_bytes();
+
                 let item = block
                     .items
                     .iter()
-                    .find(|item| &*item.key.user_key == b"b")
+                    .find(|item| &*item.key.user_key == needle)
                     .cloned()
                     .unwrap();
-                assert_eq!(item.key.seqno, 1);
+
+                assert_eq!(item.key.user_key, needle);
             })
         });
 
         group.bench_function(format!("{item_count} items (binary search)"), |b| {
             b.iter(|| {
-                let item = block.get_latest(b"b").unwrap();
-                assert_eq!(item.key.seqno, 1);
+                let needle = rng.random_range(0..item_count).to_be_bytes();
+
+                let item = block.get_latest(&needle).unwrap();
+                assert_eq!(item.key.user_key, needle);
             })
         });
     }
@@ -102,11 +102,7 @@ fn value_block_find(c: &mut Criterion) {
 fn encode_block(c: &mut Criterion) {
     let mut group = c.benchmark_group("Encode block");
 
-    for comp_type in [
-        CompressionType::None,
-        CompressionType::Lz4,
-        CompressionType::Miniz(3),
-    ] {
+    for comp_type in [CompressionType::None, CompressionType::Lz4] {
         for block_size in [4, 8, 16, 32, 64, 128] {
             let block_size = block_size * 1_024;
 
@@ -145,11 +141,7 @@ fn encode_block(c: &mut Criterion) {
 fn load_value_block_from_disk(c: &mut Criterion) {
     let mut group = c.benchmark_group("Load block from disk");
 
-    for comp_type in [
-        CompressionType::None,
-        CompressionType::Lz4,
-        CompressionType::Miniz(3),
-    ] {
+    for comp_type in [CompressionType::None, CompressionType::Lz4] {
         for block_size in [4, 8, 16, 32, 64, 128] {
             let block_size = block_size * 1_024;