parameterized database and updated storage consumers

Author: kautukkundan

test/client/integration.test.ts

@@ -29,11 +29,13 @@ import { CustomToken__factory } from "../../types/ethers-contracts";
  */
 describe("Client Integration", function() {
     before(async function() {
-        await del("./leveldb/*");
+        await del("./leveldb/test/syncer/*");
+        await del("./leveldb/test/packer/*");
     });
 
     after(async function() {
-        await del("./leveldb/*");
+        await del("./leveldb/test/syncer/*");
+        await del("./leveldb/test/packer/*");
     });
 
     it("run", async function() {
@@ -47,8 +49,12 @@ describe("Client Integration", function() {
 
         await deployKeyless(signer, false);
 
-        const storageSyncer = await storageManagerFactory();
-        const storagePacker = await storageManagerFactory();
+        const storageSyncer = await storageManagerFactory({
+            storageDirectory: "./levelDB/test/syncer"
+        });
+        const storagePacker = await storageManagerFactory({
+            storageDirectory: "./levelDB/test/packer"
+        });
 
         // Ensure initial states match
         assert.equal(storageSyncer.state.root, storagePacker.state.root);

test/client/pubkey2states.test.ts

@@ -1,13 +1,14 @@
 import chai, { assert } from "chai";
 import chaiAsPromised from "chai-as-promised";
 import { StateDatabaseEngine } from "../../ts/client/database";
-import { PRODUCTION_PARAMS } from "../../ts/constants";
+import { PRODUCTION_PARAMS, TESTING_PARAMS } from "../../ts/constants";
 import { State } from "../../ts/state";
 import del from "del";
 import { PubkeyLeaf } from "../../ts/tree/leaves/PubkeyLeaf";
 import { init } from "../../ts/mcl";
 import { BlsSigner } from "../../ts/blsSigner";
 import { Pubkey2StatesDB } from "../../ts/client/database/pubkey2states";
+import { Connection } from "../../ts/client/database/connection";
 
 chai.use(chaiAsPromised);
 
@@ -22,26 +23,37 @@ describe("StateDBEngine", () => {
     let p1: PubkeyLeaf;
     let statesP0: State[];
     let statesP1: State[];
+    let connection: Connection;
 
     before(async function() {
-        await del("./leveldb/*");
+        await del("./leveldb/test/*");
+        connection = await Connection.create(TESTING_PARAMS.STORAGE_DIRECTORY);
     });
 
     after(async function() {
-        await del("./leveldb/*");
+        await connection.close();
+        await del("./leveldb/test/*");
     });
 
     beforeEach(async function() {
-        engine = new StateDatabaseEngine(maxDepth);
+        engine = new StateDatabaseEngine(maxDepth, connection);
         statesP0 = [];
         statesP1 = [];
 
         await init();
 
-        p0 = PubkeyLeaf.fromSolG2(BlsSigner.new().pubkey, 0);
+        p0 = PubkeyLeaf.fromSolG2(
+            BlsSigner.new().pubkey,
+            0,
+            connection.pubkeyDB
+        );
         await p0.toDB();
 
-        p1 = PubkeyLeaf.fromSolG2(BlsSigner.new().pubkey, 1);
+        p1 = PubkeyLeaf.fromSolG2(
+            BlsSigner.new().pubkey,
+            1,
+            connection.pubkeyDB
+        );
         await p1.toDB();
 
         for (let i = 0; i < 4; i++) {
@@ -56,15 +68,19 @@ describe("StateDBEngine", () => {
         await engine.updateBatch(0, maxSubtreeDepth, statesP0);
         await engine.updateBatch(1, maxSubtreeDepth, statesP1);
 
-        assert.deepEqual(await Pubkey2StatesDB.getStates(p0.item.hash()), [
-            0,
-            1,
-            2,
-            3
-        ]);
-        assert.deepEqual(await Pubkey2StatesDB.getStates(p1.item.hash()), [
-            4,
-            5
-        ]);
+        assert.deepEqual(
+            await Pubkey2StatesDB.getStates(
+                p0.item.hash(),
+                connection.pubkey2statesDB
+            ),
+            [0, 1, 2, 3]
+        );
+        assert.deepEqual(
+            await Pubkey2StatesDB.getStates(
+                p1.item.hash(),
+                connection.pubkey2statesDB
+            ),
+            [4, 5]
+        );
     });
 });

test/client/transaction.test.ts

@@ -4,10 +4,12 @@ import chai, { assert } from "chai";
 import chaiAsPromised from "chai-as-promised";
 import del from "del";
 import { BlsSigner } from "../../ts/blsSigner";
+import { Connection } from "../../ts/client/database/connection";
 import { OffchainTx } from "../../ts/client/features/interface";
 import { TransferOffchainTx } from "../../ts/client/features/transfer";
 import { Status } from "../../ts/client/storageEngine/transactions/constants";
 import { TransactionDBStorage } from "../../ts/client/storageEngine/transactions/db";
+import { TESTING_PARAMS } from "../../ts/constants";
 import {
     StatusTransitionInvalid,
     TransactionAlreadyExists,
@@ -38,11 +40,18 @@ const txFactory = (
 };
 
 describe("TransactionDBStorage", () => {
-    let storage = new TransactionDBStorage();
+    let connection: Connection;
+    let storage: TransactionDBStorage;
 
     before(async function() {
         await del("./leveldb/*");
         await mcl.init();
+        connection = await Connection.create(TESTING_PARAMS.STORAGE_DIRECTORY);
+        storage = new TransactionDBStorage(connection);
+    });
+
+    after(async function() {
+        await connection.close();
     });
 
     describe("get", () => {

ts/client/database/connection.ts

@@ -1,15 +1,41 @@
 import level from "level";
 import sub from "subleveldown";
+import { LevelUp } from "levelup";
 
-const db = level("./leveldb", { valueEncoding: "json" });
+import { mkdir, stat } from "fs";
+import { promisify } from "util";
 
-export const close = async (): Promise<void> => {
-    await db.close();
-};
+const mkdirAsync = promisify(mkdir);
+const existsAsync = promisify(stat);
 
-export const pubkeyDB = sub(db, "pubkey");
-export const stateDB = sub(db, "state");
-export const nodeDB = sub(db, "node");
-// export const batchDB = sub(db, "batch");
-export const txDB = sub(db, "tx");
-export const pubkey2statesDB = sub(db, "pubkey2states");
+export class Connection {
+    public readonly pubkeyDB: LevelUp;
+    public readonly stateDB: LevelUp;
+    public readonly txDB: LevelUp;
+    public readonly pubkey2statesDB: LevelUp;
+
+    private db: LevelUp;
+
+    constructor(path: string) {
+        this.db = level(`${path}`, { valueEncoding: "json" });
+
+        this.pubkeyDB = sub(this.db, "pubkey");
+        this.stateDB = sub(this.db, "state");
+        this.txDB = sub(this.db, "tx");
+        this.pubkey2statesDB = sub(this.db, "pubkey2states");
+    }
+
+    public static async create(path: string) {
+        try {
+            await existsAsync(path);
+        } catch (error) {
+            await mkdirAsync(path, { recursive: true });
+        }
+
+        return new Connection(path);
+    }
+
+    public async close(): Promise<void> {
+        await this.db.close();
+    }
+}

ts/client/database/databaseEngine.ts

@@ -5,16 +5,18 @@ import { Hasher } from "../../tree";
 import { DBTree } from "../../tree/dbTree";
 import { LeafFactoryFunc } from "../../tree/leaves/Leaf";
 import { StorageEngine, WithWitness } from "../storageEngine/interfaces";
+import { LevelUp } from "levelup";
 
 export class DatabaseEngine<Item extends Hashable>
     implements StorageEngine<Item> {
     private tree: DBTree;
     public readonly leafFactory: LeafFactoryFunc<Item>;
 
-    constructor(depth: number, factory: LeafFactoryFunc<Item>) {
+    constructor(depth: number, db: LevelUp, factory: LeafFactoryFunc<Item>) {
         this.tree = DBTree.new(
             depth,
             factory.name,
+            db,
             Hasher.new("bytes", ZERO_BYTES32)
         );
         this.leafFactory = factory;

ts/client/database/pubkey2states.ts

@@ -1,14 +1,22 @@
 import _ from "lodash";
 import { PubkeyLeafFactory } from "../../tree/leaves/PubkeyLeaf";
-import { pubkey2statesDB } from "./connection";
+import { LevelUp } from "levelup";
 
 export class Pubkey2StatesDB {
-    static async getStates(pubkeyHash: string): Promise<number[]> {
+    static async getStates(
+        pubkeyHash: string,
+        pubkey2statesDB: LevelUp
+    ): Promise<number[]> {
         return JSON.parse(await pubkey2statesDB.get(pubkeyHash));
     }
 
-    static async update(pubkeyID: number, stateID: number): Promise<void> {
-        const pubkeyLeaf = await PubkeyLeafFactory().fromDB(pubkeyID);
+    static async update(
+        pubkeyID: number,
+        stateID: number,
+        pubkey2statesDB: LevelUp,
+        pubkeyDB: LevelUp
+    ): Promise<void> {
+        const pubkeyLeaf = await PubkeyLeafFactory(pubkeyDB).fromDB(pubkeyID);
         const pubkeyHash = pubkeyLeaf.item.hash();
 
         try {

ts/client/database/pubkeyEngine.ts

@@ -1,13 +1,18 @@
 import { Pubkey } from "../../pubkey";
 import { PubkeyLeafFactory } from "../../tree/leaves/PubkeyLeaf";
 import { StorageEngine } from "../storageEngine/interfaces";
+import { Connection } from "./connection";
 import { DatabaseEngine } from "./databaseEngine";
 
 export interface PubkeyStorageEngine extends StorageEngine<Pubkey> {}
 
 export class PubkeyDatabaseEngine extends DatabaseEngine<Pubkey>
     implements PubkeyStorageEngine {
-    constructor(depth: number) {
-        super(depth, PubkeyLeafFactory());
+    constructor(depth: number, connections: Connection) {
+        super(
+            depth,
+            connections.pubkeyDB,
+            PubkeyLeafFactory(connections.pubkeyDB)
+        );
     }
 }

ts/client/database/stateEngine.ts

@@ -1,15 +1,20 @@
 import { State } from "../../state";
 import { StateLeafFactory } from "../../tree/leaves/StateLeaf";
 import { StorageEngine } from "../storageEngine/interfaces";
+import { Connection } from "./connection";
 import { DatabaseEngine } from "./databaseEngine";
 import { Pubkey2StatesDB } from "./pubkey2states";
 
 export interface StateStorageEngine extends StorageEngine<State> {}
 
 export class StateDatabaseEngine extends DatabaseEngine<State>
     implements StateStorageEngine {
-    constructor(depth: number) {
-        super(depth, StateLeafFactory());
+    constructor(depth: number, public readonly connections: Connection) {
+        super(
+            depth,
+            connections.stateDB,
+            StateLeafFactory(connections.stateDB)
+        );
     }
 
     public async updateBatch(
@@ -20,7 +25,12 @@ export class StateDatabaseEngine extends DatabaseEngine<State>
         for (const [i, item] of items.entries()) {
             const itemID = path * 2 ** depth + i;
             await this.update(itemID, item);
-            await Pubkey2StatesDB.update(item.pubkeyID.toNumber(), itemID);
+            await Pubkey2StatesDB.update(
+                item.pubkeyID.toNumber(),
+                itemID,
+                this.connections.pubkey2statesDB,
+                this.connections.pubkeyDB
+            );
         }
     }
 }

ts/client/node.ts

@@ -15,7 +15,6 @@ import { SyncCompleteEvent } from "./constants";
 import { ClientConfig } from "./config";
 import { Genesis } from "../genesis";
 import { EmptyConfigPropError, MissingConfigPropError } from "../exceptions";
-import { close as closeDB } from "./database/connection";
 
 export type NodeModes = {
     isProposer: boolean;
@@ -45,10 +44,11 @@ export class HubbleNode {
             console.error(err);
         });
 
-        const { MAX_DEPTH } = genesis.parameters;
+        const { MAX_DEPTH, STORAGE_DIRECTORY } = genesis.parameters;
         const storageManager = await storageManagerFactory({
             stateTreeDepth: MAX_DEPTH,
-            pubkeyTreeDepth: MAX_DEPTH
+            pubkeyTreeDepth: MAX_DEPTH,
+            storageDirectory: STORAGE_DIRECTORY
         });
 
         const signer = provider.getSigner();
@@ -133,7 +133,6 @@ export class HubbleNode {
         this.packer?.stop();
         this.bidder?.stop();
         console.log("closing leveldb connection");
-        await closeDB();
     }
 
     onSyncComplete = async () => {

ts/client/services/rpc.ts

@@ -50,7 +50,8 @@ export class RPC {
                 try {
                     const { pubkeyHash } = request.params;
                     const stateIndices = await Pubkey2StatesDB.getStates(
-                        pubkeyHash
+                        pubkeyHash,
+                        l2Storage.connection.pubkey2statesDB
                     );
                     let data = stateIndices.map(async id => {
                         let state = await l2Storage.state.get(Number(id));
@@ -94,7 +95,8 @@ export class RPC {
                 try {
                     const { pubkeyHash } = request.params;
                     const stateIndices = await Pubkey2StatesDB.getStates(
-                        pubkeyHash
+                        pubkeyHash,
+                        l2Storage.connection.pubkey2statesDB
                     );
                     let data = await l2Storage.state.get(
                         Number(stateIndices[0])