Add transactions and transaction log

Author: Hexagon

README.md

@@ -1,21 +1,19 @@
 **cross/kv**
 
-## Efficient cross-runtime Key/Value database for JavaScript and TypeScript
+A cross-platform, hierarchical Key/Value database for JavaScript and TypeScript,
+that work in all major runtimes (Node.js, Deno and Bun).
 
 ### **Features**
 
 - **Simple Key/Value Storage:** Store and retrieve data easily using
   hierarchical keys.
-- **Cross-Runtime Compatibility:** Works in Node.js, Deno, browser environments,
-  and potentially other JavaScript runtimes.
-- **Flexible Data Types:** Support for strings, numbers, objects, dates, and
-  more.
+- **Cross-Runtime Compatibility:** Works in Node.js, Deno and Bun.
+- **Flexible Data Types:** Support for strings, numbers, objects, dates, maps,
+  sets and more.
 - **Hierarchical Structure:** Organize data with multi-level keys for a logical
   structure.
-- **Key Ranges:** Retrieve ranges of data efficiently using `KVKeyRange`
-  objects.
-- **Indexed:** Data is indexed to provide faster lookups (detail how your
-  indexing works).
+- **Key Ranges:** Retrieve ranges of data efficiently using key ranges.
+- **Indexed:** In-memory index to provide faster lookups of large datasets.
 
 ### **Installation**
 
@@ -62,7 +60,7 @@ import { CrossKV } from "@cross/kv";
 const kvStore = new CrossKV();
 
 // Open the database
-await kvStore.open("./lab/db19");
+await kvStore.open("./mydatabase/");
 
 // Store some values/documents indexed by users.by_id.<id>
 await kvStore.set(["users", "by_id", 1], {

deno.json

@@ -1,21 +1,23 @@
 {
   "name": "@cross/kv",
-  "version": "0.0.3",
+  "version": "0.0.4",
   "exports": {
     ".": "./mod.ts"
   },
   "imports": {
     "@cross/fs": "jsr:@cross/fs@^0.1.11",
+    "@cross/runtime": "jsr:@cross/runtime@^1.0.0",
     "@cross/test": "jsr:@cross/test@^0.0.9",
     "@std/assert": "jsr:@std/assert@^0.224.0",
     "@std/path": "jsr:@std/path@^0.224.0",
     "cbor-x": "npm:cbor-x@^1.5.9"
   },
   "publish": {
-    "exclude": [".github", "*.test.ts"]
+    "exclude": [".github", "*.test.ts", "*.bench.ts"]
   },
   "tasks": {
     "check": "deno fmt --check && deno lint && deno check mod.ts && deno doc --lint mod.ts && deno run -A mod.ts --slim --ignore-unused",
+    "bench": "deno bench -A --unstable-kv",
     "check-deps": "deno run -rA jsr:@check/deps"
   }
 }

mydatabase/.data

@@ -1,30 +1,46 @@
-import { exists, readFile, writeFile } from "@cross/fs";
 import type { KVKey, KVKeyRange } from "./key.ts";
-import { decode, encode } from "cbor-x";
+import { type KVFinishedTransaction, KVOperation } from "./transaction.ts";
 
-// Nested class to represent a node in the index tree
+/**
+ * Represents content of a node within the KVIndex tree.
+ */
 interface KVIndexContent {
-  children: KVIndexNode;
+  /**
+   * Holds references to child nodes in the index.
+   */
+  children: KVIndexNodes;
+  /**
+   * Optional reference to a data offset. Present for leaf nodes.
+   */
   reference?: number;
 }
 
-type KVIndexNode = Map<string | number, KVIndexContent>;
+/**
+ * A Map containing child keys and their corresponding Node within the tree.
+ */
+type KVIndexNodes = Map<string | number, KVIndexContent>;
 
+/**
+ * In-memory representation of a Key-Value index enabling efficient key-based lookups.
+ * It uses a tree-like structure for fast prefix and range-based searches.
+ */
 export class KVIndex {
-  private indexPath: string;
   private index: KVIndexContent;
-  private isDirty: boolean = false;
-  constructor(indexPath: string) {
-    this.indexPath = indexPath;
+  constructor() {
     this.index = {
       children: new Map(),
     };
   }
 
-  add(key: KVKey, entry: number, overwrite: boolean = false) {
+  /**
+   * Adds an entry to the index.
+   * @param transaction - The transaction to add
+   * @throws {Error} If 'overwrite' is false and a duplicate key is found.
+   */
+  add(transaction: KVFinishedTransaction) {
     let current = this.index;
     let lastPart;
-    for (const part of key.get()) {
+    for (const part of transaction.key.get()) {
       lastPart = part;
       const currentPart = current.children?.get(part as string | number);
       if (currentPart) {
@@ -38,19 +54,22 @@ export class KVIndex {
       }
     }
     if (current!.reference === undefined) {
-      current!.reference = entry;
-    } else if (overwrite) {
-      /* ToDo: Some sort of callback if overwritten? */
-      current!.reference = entry;
+      current!.reference = transaction.offset;
+    } else if (transaction.oper === KVOperation.UPSERT) {
+      current!.reference = transaction.offset;
     } else {
       throw new Error(`Duplicate key: ${lastPart}`);
     }
-    this.isDirty = true;
   }
 
-  delete(key: KVKey): number | undefined {
+  /**
+   * Removes an entry from the index based on a provided key.
+   * @param transaction - The transaction to remove.
+   * @returns The removed data row reference, or undefined if the key was not found.
+   */
+  delete(transaction: KVFinishedTransaction): number | undefined {
     let current = this.index;
-    for (const part of key.get()) {
+    for (const part of transaction.key.get()) {
       const currentPart = current.children.get(part as (string | number));
       if (!currentPart || !currentPart.children) { // Key path not found
         return undefined;
@@ -62,15 +81,17 @@ export class KVIndex {
     const oldReference = current.reference;
     current.reference = undefined;
     delete current.reference;
-    this.isDirty = true;
-
-    /* ToDo recursive cleanup if (!current.children.size) {
-      delete current.children;
-    }*/
 
+    // No need to cleanup as leftover nodes will be lost at next rebuild
     return oldReference;
   }
 
+  /**
+   * Retrieves a list of data row references associated with a given key.
+   * Supports prefix and range-based searches.
+   * @param key - The key to search for (can include ranges)
+   * @returns An array of data row references.
+   */
   get(key: KVKey): number[] {
     const resultSet: number[] = [];
 
@@ -119,23 +140,4 @@ export class KVIndex {
 
     return resultSet;
   }
-
-  async loadIndex(): Promise<KVIndexContent> {
-    if (await exists(this.indexPath)) {
-      const fileContents = await readFile(this.indexPath);
-      try {
-        const index = decode(fileContents);
-        this.index = index as KVIndexContent;
-        this.isDirty = false;
-      } catch (_e) { /* Ignore for now */ }
-    }
-    return this.index;
-  }
-
-  async saveIndex(): Promise<void> {
-    if (!this.isDirty) return;
-    const serializedIndex = encode(this.index);
-    await writeFile(this.indexPath, serializedIndex);
-    this.isDirty = false;
-  }
 }

mydatabase/.idx

@@ -0,0 +1,45 @@
+import { tempfile } from "@cross/fs";
+import { CrossKV } from "./kv.ts";
+
+const DATABASE_FILE_CROSS = await tempfile();
+const DATABASE_FILE_DENO = await tempfile();
+
+// Your setup and cleanup functions
+async function setupCrossKV() {
+  const kvStore = new CrossKV();
+  await kvStore.open(DATABASE_FILE_CROSS);
+  return kvStore;
+}
+
+async function setupDenoKV() {
+  const kvStore = await Deno.openKv(DATABASE_FILE_DENO);
+  return kvStore;
+}
+
+/*async function cleanup() {
+  denoStore.close();
+  await crossStore.close();
+}*/
+
+const crossStore = await setupCrossKV();
+const denoStore = await setupDenoKV();
+let crossIter = 0;
+let denoIter = 0;
+
+await Deno.bench("cross_kv_set", async () => {
+  await crossStore.set(["testKey", crossIter++], { data: "testData" });
+});
+
+await Deno.bench("deno_kv_set", async () => {
+  await denoStore.set(["testKey", denoIter++], { data: "testData" });
+});
+
+await Deno.bench("cross_kv_get", async () => {
+  await crossStore.get(["testKey", crossIter--]);
+});
+
+await Deno.bench("deno_kv_get", async () => {
+  await denoStore.get(["testKey", crossIter--]);
+});
+
+//await cleanup();

src/index.test.ts

@@ -1,9 +1,10 @@
 import { assertEquals, assertNotEquals, assertRejects } from "@std/assert";
 import { test } from "@cross/test";
 import { CrossKV } from "./kv.ts";
+import { tempfile } from "@cross/fs";
 
 test("CrossKV: set, get and delete (numbers and strings)", async () => {
-  const tempFilePrefix = await Deno.makeTempFile();
+  const tempFilePrefix = await tempfile();
   const kvStore = new CrossKV();
   await kvStore.open(tempFilePrefix);
   await kvStore.set(["name"], "Alice");
@@ -19,7 +20,7 @@ test("CrossKV: set, get and delete (numbers and strings)", async () => {
 });
 
 test("CrossKV: set, get and delete (objects)", async () => {
-  const tempFilePrefix = await Deno.makeTempFile();
+  const tempFilePrefix = await tempfile();
   const kvStore = new CrossKV();
   await kvStore.open(tempFilePrefix);
   await kvStore.set(["name"], { data: "Alice" });
@@ -35,7 +36,7 @@ test("CrossKV: set, get and delete (objects)", async () => {
 });
 
 test("CrossKV: set, get and delete (dates)", async () => {
-  const tempFilePrefix = await Deno.makeTempFile();
+  const tempFilePrefix = await tempfile();
   const kvStore = new CrossKV();
   await kvStore.open(tempFilePrefix);
   const date = new Date();
@@ -50,7 +51,7 @@ test("CrossKV: set, get and delete (dates)", async () => {
 });
 
 test("CrossKV: throws on duplicate key insertion", async () => {
-  const tempFilePrefix = await Deno.makeTempFile();
+  const tempFilePrefix = await tempfile();
   const kvStore = new CrossKV();
   await kvStore.open(tempFilePrefix);
 
@@ -59,23 +60,23 @@ test("CrossKV: throws on duplicate key insertion", async () => {
   assertRejects(
     async () => await kvStore.set(["name"], "Bob"),
     Error,
-    "Duplicate key: name",
+    "Duplicate key: Key already exists",
   );
 });
 
 test("CrossKV: throws when trying to delete a non-existing key", async () => {
-  const tempFilePrefix = await Deno.makeTempFile();
+  const tempFilePrefix = await tempfile();
   const kvStore = new CrossKV();
   await kvStore.open(tempFilePrefix);
 
-  assertRejects(
-    () => kvStore.delete(["unknownKey"]),
+  await assertRejects(
+    async () => await kvStore.delete(["unknownKey"]),
     Error,
   ); // We don't have a specific error type for this yet
 });
 
 test("CrossKV: supports multi-level nested keys", async () => {
-  const tempFilePrefix = await Deno.makeTempFile();
+  const tempFilePrefix = await tempfile();
   const kvStore = new CrossKV();
   await kvStore.open(tempFilePrefix);
 
@@ -87,7 +88,7 @@ test("CrossKV: supports multi-level nested keys", async () => {
 });
 
 test("CrossKV: supports multi-level nested keys with numbers", async () => {
-  const tempFilePrefix = await Deno.makeTempFile();
+  const tempFilePrefix = await tempfile();
   const kvStore = new CrossKV();
   await kvStore.open(tempFilePrefix);
 
@@ -100,7 +101,7 @@ test("CrossKV: supports multi-level nested keys with numbers", async () => {
 });
 
 test("CrossKV: supports numeric key ranges", async () => {
-  const tempFilePrefix = await Deno.makeTempFile();
+  const tempFilePrefix = await tempfile();
   const kvStore = new CrossKV();
   await kvStore.open(tempFilePrefix);
 
@@ -119,7 +120,7 @@ test("CrossKV: supports numeric key ranges", async () => {
 });
 
 test("CrossKV: supports string key ranges", async () => {
-  const tempFilePrefix = await Deno.makeTempFile();
+  const tempFilePrefix = await tempfile();
   const kvStore = new CrossKV();
   await kvStore.open(tempFilePrefix);