Performance improvements

Author: Hexagon

CHANGELOG.md

@@ -1,3 +1,16 @@
+## 0.16.0
+
+**Breaking change, databases created using earlier versions is not compatible with 0.16.0 and up.**
+
+- Use 32 bit MurmurHash3 (from new dependency `npm:ohash` instead of SHA-1 for transaction integrity check,
+  saving time and storage
+- Precalculate the Uint32Array of the transaction signature, improving
+  performance
+- Use CBOR-encoding of key instead of custom implementation, improving
+  performance
+- Avoid copying arraybuffers in certain situations
+- Prefetch data on transaction read, severely reducing the number of disk reads
+
 ## 0.15.11
 
 - Remove option doLock from `.sync()`

deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@cross/kv",
-  "version": "0.15.11",
+  "version": "0.16.0",
   "exports": {
     ".": "./mod.ts",
     "./cli": "./src/cli/mod.ts"
@@ -12,7 +12,8 @@
     "@cross/utils": "jsr:@cross/utils@^0.13.0",
     "@std/assert": "jsr:@std/assert@^0.226.0",
     "@std/path": "jsr:@std/path@^0.225.1",
-    "cbor-x": "npm:cbor-x@^1.5.9"
+    "cbor-x": "npm:cbor-x@^1.5.9",
+    "ohash": "npm:ohash@^1.1.3"
   },
   "publish": {
     "exclude": [".github", "test/*"]

src/lib/cache.ts

@@ -2,7 +2,7 @@ import { LEDGER_CACHE_MEMORY_FACTOR } from "./constants.ts";
 import type { KVLedgerResult } from "./ledger.ts";
 
 /**
- * An in-memory cache for `KVLedgerResult` objects, optimized for append-only ledgers.
+ * An in-memory cache for `KVLedgerResult` objects.
  *
  * This cache stores transaction results (`KVLedgerResult`) associated with their offsets within the ledger.
  * It maintains a fixed maximum size and evicts the oldest entries (Least Recently Used - LRU)

src/lib/constants.ts

@@ -2,15 +2,12 @@
 export const LOCK_DEFAULT_MAX_RETRIES = 32;
 export const LOCK_DEFAULT_INITIAL_RETRY_INTERVAL_MS = 30; // Increased with itself on each retry, so the actual retry interval is 20, 40, 60 etc. 32 and 20 become about 10 seconds total.
 export const LOCK_STALE_TIMEOUT_MS = 6 * 60 * 60 * 1000; // Automatically unlock a ledger that has been locked for more than 2*60*60*1000 = 2 hours.
-export const LEDGER_CURRENT_VERSION: string = "B015";
+export const LEDGER_CURRENT_VERSION: string = "B016";
 export const SUPPORTED_LEDGER_VERSIONS: string[] = [
   LEDGER_CURRENT_VERSION,
-  "B011",
-  "B012",
-  "B013",
 ];
+export const LEDGER_PREFETCH_BYTES = 50 * 1024; // Prefetch chunks of 50KB of data while reading the ledger
 export const LEDGER_MAX_READ_FAILURES = 10;
-export const LEDGER_PREFETCH_BYTES = 256;
 export const SYNC_INTERVAL_MS = 2_500; // Overridable with instance configuration
 export const LEDGER_CACHE_MB = 100; // Allow 100 MBytes of the ledger to exist in RAM. Not an exact science due to LEDGER_CACHE_MEMORY_FACTOR.
 export const LEDGER_CACHE_MEMORY_FACTOR = 3; // Assume that ledger entries take about n times as much space when unwrapped in RAM. Used for ledger cache memory limit, does not need to be exakt.
@@ -21,5 +18,7 @@ export const LOCKED_BYTES_LENGTH = 8; // Length of timestamp
 export const LOCK_BYTE_OFFSET = LEDGER_BASE_OFFSET - LOCKED_BYTES_LENGTH; // Last 8 bytes of the header
 export const KV_KEY_ALLOWED_CHARS = /^[@\p{L}\p{N}_-]+$/u; // Unicode letters and numbers, undescore, hyphen and at
 export const LEDGER_FILE_ID: string = "CKVD"; // Cross/KV Database
-export const TRANSACTION_SIGNATURE: string = "T;"; // Cross/Kv Transaction
+export const ENCODED_TRANSACTION_SIGNATURE: Uint8Array = new TextEncoder()
+  .encode("T;"); // Cross/Kv Transaction
 export const UNLOCKED_BYTES = new Uint8Array(LOCKED_BYTES_LENGTH);
+export const LOCKED_BYTES = new Uint8Array(LOCKED_BYTES_LENGTH);

src/lib/key.ts

@@ -1,3 +1,4 @@
+import { decode, encode } from "cbor-x";
 import { KV_KEY_ALLOWED_CHARS } from "./constants.ts";
 
 // Helper function to stringify range values correctly
@@ -68,11 +69,11 @@ export class KVKeyInstance {
   private isQuery: boolean;
   public byteLength?: number;
   constructor(
-    key: KVQuery | KVKey | DataView,
+    key: KVQuery | KVKey | Uint8Array,
     isQuery: boolean = false,
     validate: boolean = true,
   ) {
-    if (key instanceof DataView) {
+    if (key instanceof Uint8Array) {
       this.key = this.fromUint8Array(key);
     } else {
       this.key = key;
@@ -87,48 +88,8 @@ export class KVKeyInstance {
    * Encodes the key elements into a byte array suitable for storage in a transaction header.
    */
   public toUint8Array(): Uint8Array {
-    const keyBytesArray = [];
-    for (const element of this.key) {
-      if (typeof element === "string") {
-        keyBytesArray.push(new Uint8Array([0])); // Type: String
-        const strBytes = new TextEncoder().encode(element);
-        const strLengthBytes = new Uint8Array(4);
-        new DataView(strLengthBytes.buffer).setUint32(
-          0,
-          strBytes.length,
-          false,
-        );
-        keyBytesArray.push(strLengthBytes);
-        keyBytesArray.push(strBytes);
-      } else if (typeof element === "number") {
-        keyBytesArray.push(new Uint8Array([1])); // Type: Number
-        const numBytes = new Uint8Array(8);
-        new DataView(numBytes.buffer).setFloat64(0, element, false);
-        keyBytesArray.push(numBytes);
-      } else {
-        // This should never happen if validate() is working correctly
-        throw new TypeError("Invalid key element type");
-      }
-    }
-
-    // Encode the number of key elements
-    const numKeyElementsBytes = new Uint8Array(1);
-    new DataView(numKeyElementsBytes.buffer).setUint8(
-      0,
-      this.key.length,
-    );
-    keyBytesArray.unshift(numKeyElementsBytes); // Add to the beginning
-
-    const keyArray = new Uint8Array(
-      keyBytesArray.reduce((a, b) => a + b.length, 0),
-    );
-    let keyOffset = 0;
-    for (const bytes of keyBytesArray) {
-      keyArray.set(bytes, keyOffset);
-      keyOffset += bytes.length;
-    }
-
-    return keyArray;
+    const data = encode(this.key);
+    return new Uint8Array(data, 0, data.byteLength);
   }
 
   /**
@@ -137,44 +98,9 @@ export class KVKeyInstance {
    * @param data - The byte array containing the encoded key.
    * @throws {Error} If the key cannot be decoded.
    */
-  private fromUint8Array(dataView: DataView): KVKey {
-    let offset = 0;
-
-    // 1. Decode Number of Key Elements (uint32)
-    const numKeyElements = dataView.getUint8(offset);
-    offset += 1;
-
-    const keyToBe: KVKey = [];
-
-    for (let i = 0; i < numKeyElements; i++) {
-      // 2. Decode Element Type (uint8): 0 for string, 1 for number
-      const elementType = dataView.getUint8(offset);
-      offset += 1;
-
-      if (elementType === 0) { // String
-        // 3a. Decode String Length (uint32)
-        const strLength = dataView.getUint32(offset, false);
-        offset += 4;
-
-        // 3b. Decode String Bytes
-        const strBytes = new DataView(
-          dataView.buffer,
-          dataView.byteOffset + offset,
-          strLength,
-        );
-        keyToBe.push(new TextDecoder().decode(strBytes));
-        offset += strLength;
-      } else if (elementType === 1) { // Number
-        // 3c. Decode Number (float64)
-        const numValue = dataView.getFloat64(offset, false);
-        keyToBe.push(numValue);
-        offset += 8;
-      } else {
-        throw new Error(`Invalid key element type ${elementType}`);
-      }
-    }
-    this.byteLength = offset;
-    return keyToBe;
+  private fromUint8Array(data: Uint8Array): KVKey {
+    this.key = decode(data);
+    return this.key as KVKey;
   }
 
   get(): KVQuery | KVKey {
@@ -301,7 +227,7 @@ export class KVKeyInstance {
       }
     }
 
-    const instance = new KVKeyInstance(result, isQuery);
+    const instance = new KVKeyInstance(result, isQuery, false);
     instance.validate();
 
     return isQuery ? result : result as KVKey;
@@ -380,7 +306,10 @@ export class KVKeyInstance {
         const subquery = query.slice(i + 1);
         const subkey = thisKey.slice(i + 1);
         if (
-          !new KVKeyInstance(subkey, true).matchesQuery(subquery, recursive)
+          !new KVKeyInstance(subkey, true, false).matchesQuery(
+            subquery,
+            recursive,
+          )
         ) {
           return false;
         }

src/lib/kv.ts

@@ -159,7 +159,7 @@ export class KV extends EventEmitter {
         "Invalid option: syncIntervalMs must be a positive integer",
       );
     }
-    this.syncIntervalMs = options.syncIntervalMs ?? SYNC_INTERVAL_MS;
+    this.syncIntervalMs = options.syncIntervalMs ?? this.syncIntervalMs;
     // - ledgerCacheSize
     if (
       options.ledgerCacheSize !== undefined &&
@@ -209,9 +209,9 @@ export class KV extends EventEmitter {
     this.ledger = new KVLedger(filePath, this.ledgerCacheSize);
     this.ledgerPath = filePath;
     await this.ledger.open(createIfMissing);
-
     // Do the initial synchronization
     // - If `this.autoSync` is enabled, additional synchronizations will be carried out every `this.syncIntervalMs`
+
     const syncResult = await this.sync();
     if (syncResult.error) {
       throw syncResult.error;
@@ -747,8 +747,12 @@ export class KV extends EventEmitter {
     this.ensureOpen();
     this.ensureIndex();
 
+    const validatedQuery: KVKeyInstance | null = key === null
+      ? null
+      : new KVKeyInstance(key, true);
+
     return this.index.getChildKeys(
-      key === null ? null : new KVKeyInstance(key, true),
+      key === null ? null : validatedQuery,
     );
   }