diff --git a/packages/api-development/source/controllers/blocks.ts b/packages/api-development/source/controllers/blocks.ts index ebd515002..eab89e7c9 100644 --- a/packages/api-development/source/controllers/blocks.ts +++ b/packages/api-development/source/controllers/blocks.ts @@ -99,6 +99,6 @@ export class BlocksController extends Controller { // TODO: Support height only private async getBlock(idOrHeight: string): Promise { - return (await this.database.getCommit(Number.parseInt(idOrHeight)))?.block; + return await this.database.getBlock(Number.parseInt(idOrHeight)); } } diff --git a/packages/api-evm/source/actions/eth-get-block-by-hash.ts b/packages/api-evm/source/actions/eth-get-block-by-hash.ts index fc1be3c19..69c3d5e7a 100644 --- a/packages/api-evm/source/actions/eth-get-block-by-hash.ts +++ b/packages/api-evm/source/actions/eth-get-block-by-hash.ts @@ -26,13 +26,13 @@ export class EthGetBlockByHashAction implements Contracts.Api.RPC.Action { public async handle(parameters: [string, boolean]): Promise { const transactionObject = parameters[1]; - const commit = await this.databaseService.getCommitById(parameters[0].slice(2)); + const block = await this.databaseService.getBlockById(parameters[0].slice(2)); - if (!commit) { + if (!block) { // eslint-disable-next-line unicorn/no-null return null; } - return this.app.resolve(BlockResource).transform(commit.block, transactionObject); + return this.app.resolve(BlockResource).transform(block, transactionObject); } } diff --git a/packages/api-evm/source/actions/eth-get-block-by-number.ts b/packages/api-evm/source/actions/eth-get-block-by-number.ts index d212cc7a7..54e9f7f86 100644 --- a/packages/api-evm/source/actions/eth-get-block-by-number.ts +++ b/packages/api-evm/source/actions/eth-get-block-by-number.ts @@ -33,13 +33,13 @@ export class EthGetBlockByNumberAction implements Contracts.Api.RPC.Action { const height = parameters[0].startsWith("0x") ? Number.parseInt(parameters[0]) : this.stateStore.getHeight(); const transactionObject = parameters[1]; - const commit = await this.databaseService.getCommit(height); + const block = await this.databaseService.getBlock(height); - if (!commit) { + if (!block) { // eslint-disable-next-line unicorn/no-null return null; } - return this.app.resolve(BlockResource).transform(commit.block, transactionObject); + return this.app.resolve(BlockResource).transform(block, transactionObject); } } diff --git a/packages/api-sync/source/restore.ts b/packages/api-sync/source/restore.ts index 737f59232..e9f61d897 100644 --- a/packages/api-sync/source/restore.ts +++ b/packages/api-sync/source/restore.ts @@ -194,7 +194,7 @@ export class Restore { let currentHeight = 0; do { - const commits = await this.databaseService.findCommits( + const commits = this.databaseService.readCommits( Math.min(currentHeight, mostRecentCommit.block.header.height), Math.min(currentHeight + BATCH_SIZE, mostRecentCommit.block.header.height), ); @@ -202,7 +202,7 @@ export class Restore { const blocks: Models.Block[] = []; const transactions: Models.Transaction[] = []; - for (const { proof, block } of commits) { + for await (const { proof, block } of commits) { blocks.push({ commitRound: proof.round, generatorAddress: block.header.generatorAddress, diff --git a/packages/bootstrap/source/bootstrapper.ts b/packages/bootstrap/source/bootstrapper.ts index 46238cb1f..a879c9db0 100644 --- a/packages/bootstrap/source/bootstrapper.ts +++ b/packages/bootstrap/source/bootstrapper.ts @@ -96,13 +96,13 @@ export class Bootstrapper { } async #checkStoredGenesisCommit(): Promise { - const genesisCommit = await this.databaseService.getCommit(0); + const genesisBlock = await this.databaseService.getBlock(0); - if (!genesisCommit) { + if (!genesisBlock) { return; } - if (this.stateStore.getGenesisCommit().block.data.id !== genesisCommit.block.data.id) { + if (this.stateStore.getGenesisCommit().block.data.id !== genesisBlock.data.id) { throw new Error("Block from crypto.json doesn't match stored genesis block"); } } diff --git a/packages/consensus-storage/package.json b/packages/consensus-storage/package.json index fbf27b5e9..40c47e205 100644 --- a/packages/consensus-storage/package.json +++ b/packages/consensus-storage/package.json @@ -25,7 +25,7 @@ "@mainsail/container": "workspace:*", "@mainsail/contracts": "workspace:*", "@mainsail/kernel": "workspace:*", - "lmdb": "3.1.4" + "lmdb": "3.2.0" }, "devDependencies": { "uvu": "^0.5.6" diff --git a/packages/contracts/source/contracts/crypto/block.ts b/packages/contracts/source/contracts/crypto/block.ts index 1c5417e1c..8a68f257c 100644 --- a/packages/contracts/source/contracts/crypto/block.ts +++ b/packages/contracts/source/contracts/crypto/block.ts @@ -75,8 +75,6 @@ export interface BlockFactory { } export interface BlockSerializer { - headerSize(): number; - totalSize(block: BlockDataSerializable): number; serializeHeader(block: BlockDataSerializable): Promise; diff --git a/packages/contracts/source/contracts/crypto/commit.ts b/packages/contracts/source/contracts/crypto/commit.ts index 206ce48cc..49c1f3683 100644 --- a/packages/contracts/source/contracts/crypto/commit.ts +++ b/packages/contracts/source/contracts/crypto/commit.ts @@ -33,8 +33,6 @@ export interface CommitProof { } export interface CommitSerializer { - proofSize(): number; - serializeCommitProof(proof: CommitProof): Promise; serializeCommit(commit: CommitSerializable): Promise; diff --git a/packages/contracts/source/contracts/database.ts b/packages/contracts/source/contracts/database.ts index 90abe6f44..fe308701f 100644 --- a/packages/contracts/source/contracts/database.ts +++ b/packages/contracts/source/contracts/database.ts @@ -1,5 +1,4 @@ -import { Block } from "./crypto/block.js"; -import { Commit } from "./crypto/commit.js"; +import { Block, BlockHeader, Commit, Transaction } from "./crypto/index.js"; export interface State { height: number; @@ -11,15 +10,23 @@ export interface DatabaseService { isEmpty(): boolean; getState(): State; + getCommit(height: number): Promise; getCommitById(id: string): Promise; + getLastCommit(): Promise; hasCommitById(id: string): boolean; findCommitBuffers(start: number, end: number): Promise; readCommits(start: number, end: number): AsyncGenerator; + + getBlock(height: number): Promise; + getBlockById(id: string): Promise; findBlocks(start: number, end: number): Promise; - findCommits(start: number, end: number): Promise; - getLastCommit(): Promise; + getBlockHeader(height: number): Promise; + getBlockHeaderById(id: string): Promise; + + getTransactionById(id: string): Promise; + addCommit(block: Commit): void; persist(): Promise; } diff --git a/packages/contracts/source/identifiers.ts b/packages/contracts/source/identifiers.ts index cc880e376..ec8936cf0 100644 --- a/packages/contracts/source/identifiers.ts +++ b/packages/contracts/source/identifiers.ts @@ -57,6 +57,7 @@ export const Identifiers = { Block: { Deserializer: Symbol("Crypto"), Factory: Symbol("Crypto"), + HeaderSize: Symbol("Crypto"), IDFactory: Symbol("Crypto"), Serializer: Symbol("Crypto"), Verifier: Symbol("Crypto"), @@ -64,6 +65,7 @@ export const Identifiers = { Commit: { Deserializer: Symbol("Crypto"), Factory: Symbol("Crypto"), + ProofSize: Symbol("Crypto"), Serializer: Symbol("Crypto"), }, Configuration: Symbol("Crypto"), @@ -130,7 +132,10 @@ export const Identifiers = { Storage: { Block: Symbol("Database"), BlockId: Symbol("Database"), + Commit: Symbol("Database"), State: Symbol("Database"), + Transaction: Symbol("Database"), + TransactionIds: Symbol("Database"), }, }, Evm: { diff --git a/packages/crypto-block/source/deserializer.ts b/packages/crypto-block/source/deserializer.ts index 9a26012c3..92dd979ca 100644 --- a/packages/crypto-block/source/deserializer.ts +++ b/packages/crypto-block/source/deserializer.ts @@ -18,8 +18,8 @@ export class Deserializer implements Contracts.Crypto.BlockDeserializer { @inject(Identifiers.Cryptography.Serializer) private readonly serializer!: Contracts.Serializer.Serializer; - @inject(Identifiers.Cryptography.Block.Serializer) - private readonly blockSerializer!: Contracts.Crypto.BlockSerializer; + @inject(Identifiers.Cryptography.Block.HeaderSize) + private readonly headerSize!: () => number; public async deserializeHeader(serialized: Buffer): Promise { const buffer: ByteBuffer = ByteBuffer.fromBuffer(serialized); @@ -51,7 +51,7 @@ export class Deserializer implements Contracts.Crypto.BlockDeserializer { const block = {} as Contracts.Crypto.BlockHeader; await this.serializer.deserialize(buffer, block, { - length: this.blockSerializer.headerSize(), + length: this.headerSize(), schema: { version: { type: "uint8", diff --git a/packages/crypto-block/source/index.ts b/packages/crypto-block/source/index.ts index 297374387..6dbc9dc2c 100644 --- a/packages/crypto-block/source/index.ts +++ b/packages/crypto-block/source/index.ts @@ -17,6 +17,28 @@ export * from "./verifier.js"; export class ServiceProvider extends Providers.ServiceProvider { public async register(): Promise { + this.app.bind(Identifiers.Cryptography.Block.HeaderSize).toFunction(() => { + const hashByteLength = this.app.get(Identifiers.Cryptography.Hash.Size.SHA256); + const generatorAddressByteLength = this.app.get(Identifiers.Cryptography.Identity.Address.Size); + + return ( + 1 + // version + 6 + // timestamp + 4 + // height + 4 + // round + hashByteLength + // previousBlock + hashByteLength + // stateHash + 2 + // numberOfTransactions + 4 + // totalGasUsed + 32 + // totalAmount + 32 + // totalFee + 32 + // reward + 4 + // payloadLength + hashByteLength + // payloadHash + generatorAddressByteLength + ); + }); + this.app.bind(Identifiers.Cryptography.Block.Deserializer).to(Deserializer).inSingletonScope(); this.app.bind(Identifiers.Cryptography.Block.Factory).to(BlockFactory).inSingletonScope(); this.app.bind(Identifiers.Cryptography.Block.IDFactory).to(IDFactory).inSingletonScope(); diff --git a/packages/crypto-block/source/serializer.ts b/packages/crypto-block/source/serializer.ts index 8497dc9ea..b21c0eff3 100644 --- a/packages/crypto-block/source/serializer.ts +++ b/packages/crypto-block/source/serializer.ts @@ -8,30 +8,8 @@ export class Serializer implements Contracts.Crypto.BlockSerializer { @tagged("type", "wallet") private readonly serializer!: Contracts.Serializer.Serializer; - @inject(Identifiers.Cryptography.Hash.Size.SHA256) - private readonly hashByteLength!: number; - - @inject(Identifiers.Cryptography.Identity.Address.Size) - private readonly generatorAddressByteLength!: number; - - public headerSize(): number { - return ( - 1 + // version - 6 + // timestamp - 4 + // height - 4 + // round - this.hashByteLength + // previousBlock - this.hashByteLength + // stateHash - 2 + // numberOfTransactions - 4 + // totalGasUsed - 32 + // totalAmount - 32 + // totalFee - 32 + // reward - 4 + // payloadLength - this.hashByteLength + // payloadHash - this.generatorAddressByteLength - ); - } + @inject(Identifiers.Cryptography.Block.HeaderSize) + private readonly headerSize!: () => number; public totalSize(block: Contracts.Crypto.BlockDataSerializable): number { return this.headerSize() + block.payloadLength; diff --git a/packages/crypto-block/source/verifier.ts b/packages/crypto-block/source/verifier.ts index 74b71b41f..79bd57397 100644 --- a/packages/crypto-block/source/verifier.ts +++ b/packages/crypto-block/source/verifier.ts @@ -7,12 +7,12 @@ export class Verifier implements Contracts.Crypto.BlockVerifier { @inject(Identifiers.Cryptography.Configuration) private readonly configuration!: Contracts.Crypto.Configuration; - @inject(Identifiers.Cryptography.Block.Serializer) - private readonly serializer!: Contracts.Crypto.BlockSerializer; - @inject(Identifiers.Cryptography.Hash.Factory) private readonly hashFactory!: Contracts.Crypto.HashFactory; + @inject(Identifiers.Cryptography.Block.HeaderSize) + private readonly headerSize!: () => number; + public async verify(block: Contracts.Crypto.Block): Promise { const blockData: Contracts.Crypto.BlockData = block.data; const result: Utils.Mutable = { @@ -45,8 +45,7 @@ export class Verifier implements Contracts.Crypto.BlockVerifier { result.errors.push("Invalid block version"); } - const headerSize = this.serializer.headerSize(); - const totalSize = headerSize + block.header.payloadLength; + const totalSize = this.headerSize() + block.header.payloadLength; if (totalSize > constants.block.maxPayload) { result.errors.push(`Payload is too large: ${totalSize} > ${constants.block.maxPayload}`); } diff --git a/packages/crypto-block/test/helpers/prepare-sandbox.ts b/packages/crypto-block/test/helpers/prepare-sandbox.ts index 7241ae73f..a2134500b 100644 --- a/packages/crypto-block/test/helpers/prepare-sandbox.ts +++ b/packages/crypto-block/test/helpers/prepare-sandbox.ts @@ -22,6 +22,30 @@ import { Serializer } from "../../source/serializer"; export const prepareSandbox = async (context) => { context.sandbox = new Sandbox(); + context.sandbox.app.bind(Identifiers.Cryptography.Block.HeaderSize).toFunction(() => { + const hashByteLength = context.sandbox.app.get(Identifiers.Cryptography.Hash.Size.SHA256); + const generatorAddressByteLength = context.sandbox.app.get( + Identifiers.Cryptography.Identity.Address.Size, + ); + + return ( + 1 + // version + 6 + // timestamp + 4 + // height + 4 + // round + hashByteLength + // previousBlock + hashByteLength + // stateHash + 2 + // numberOfTransactions + 4 + // totalGasUsed + 32 + // totalAmount + 32 + // totalFee + 32 + // reward + 4 + // payloadLength + hashByteLength + // payloadHash + generatorAddressByteLength + ); + }); + context.sandbox.app.get(Identifiers.Config.Repository).set("crypto", crypto); context.sandbox.app.bind(Identifiers.Services.EventDispatcher.Service).toConstantValue({ dispatchSync: () => {} }); context.sandbox.app.bind(Identifiers.Services.Log.Service).toConstantValue({}); diff --git a/packages/crypto-commit/source/deserializer.ts b/packages/crypto-commit/source/deserializer.ts index 5995a2c33..4bb24a1e2 100644 --- a/packages/crypto-commit/source/deserializer.ts +++ b/packages/crypto-commit/source/deserializer.ts @@ -7,8 +7,8 @@ export class Deserializer implements Contracts.Crypto.CommitDeserializer { @inject(Identifiers.Cryptography.Serializer) private readonly serializer!: Contracts.Serializer.Serializer; - @inject(Identifiers.Cryptography.Commit.Serializer) - private readonly commitSerializer!: Contracts.Crypto.CommitSerializer; + @inject(Identifiers.Cryptography.Commit.ProofSize) + private readonly proofSize!: () => number; public async deserializeCommitProof(serialized: Buffer): Promise { const buffer: ByteBuffer = ByteBuffer.fromBuffer(serialized); @@ -16,7 +16,7 @@ export class Deserializer implements Contracts.Crypto.CommitDeserializer { const proof = {} as Contracts.Crypto.CommitProof; await this.serializer.deserialize(buffer, proof, { - length: this.commitSerializer.proofSize(), + length: this.proofSize(), schema: { round: { type: "uint32", diff --git a/packages/crypto-commit/source/factory.ts b/packages/crypto-commit/source/factory.ts index b2b25a148..0cb5b8331 100644 --- a/packages/crypto-commit/source/factory.ts +++ b/packages/crypto-commit/source/factory.ts @@ -7,16 +7,16 @@ export class CommitFactory implements Contracts.Crypto.CommitFactory { @inject(Identifiers.Cryptography.Block.Factory) private readonly blockFactory!: Contracts.Crypto.BlockFactory; - @inject(Identifiers.Cryptography.Commit.Serializer) - private readonly commitSerializer!: Contracts.Crypto.CommitSerializer; - @inject(Identifiers.Cryptography.Commit.Deserializer) private readonly commitDeserializer!: Contracts.Crypto.CommitDeserializer; + @inject(Identifiers.Cryptography.Commit.ProofSize) + private readonly proofSize!: () => number; + public async fromBytes(buff: Buffer): Promise { const buffer = ByteBuffer.fromBuffer(buff); - const proofBuffer = buffer.readBytes(this.commitSerializer.proofSize()); + const proofBuffer = buffer.readBytes(this.proofSize()); const proof = await this.commitDeserializer.deserializeCommitProof(proofBuffer); const block = await this.blockFactory.fromBytes(buffer.getRemainder()); diff --git a/packages/crypto-commit/source/index.ts b/packages/crypto-commit/source/index.ts index a8f726bfb..952eade6e 100644 --- a/packages/crypto-commit/source/index.ts +++ b/packages/crypto-commit/source/index.ts @@ -7,6 +7,14 @@ import { Serializer } from "./serializer.js"; export class ServiceProvider extends Providers.ServiceProvider { public async register(): Promise { + this.app.bind(Identifiers.Cryptography.Commit.ProofSize).toFunction( + () => + 4 + // round + this.app.getTagged(Identifiers.Cryptography.Signature.Size, "type", "consensus") + // signature + 1 + + 8, // validator set bitmap); + ); + this.app.bind(Identifiers.Cryptography.Commit.Serializer).to(Serializer).inSingletonScope(); this.app.bind(Identifiers.Cryptography.Commit.Deserializer).to(Deserializer).inSingletonScope(); diff --git a/packages/crypto-commit/source/serializer.ts b/packages/crypto-commit/source/serializer.ts index 6f35b4fe9..4942d97fe 100644 --- a/packages/crypto-commit/source/serializer.ts +++ b/packages/crypto-commit/source/serializer.ts @@ -6,15 +6,8 @@ export class Serializer implements Contracts.Crypto.CommitSerializer { @inject(Identifiers.Cryptography.Serializer) private readonly serializer!: Contracts.Serializer.Serializer; - @inject(Identifiers.Cryptography.Message.Serializer) - private readonly messageSerializer!: Contracts.Crypto.MessageSerializer; - - public proofSize(): number { - return ( - 4 + // round - +this.messageSerializer.lockProofSize() - ); - } + @inject(Identifiers.Cryptography.Commit.ProofSize) + private readonly proofSize!: () => number; public async serializeCommitProof(commit: Contracts.Crypto.CommitProof): Promise { return this.serializer.serialize(commit, { diff --git a/packages/crypto-commit/test/helpers/prepare-sandbox.ts b/packages/crypto-commit/test/helpers/prepare-sandbox.ts index a1728375b..701b75f0e 100644 --- a/packages/crypto-commit/test/helpers/prepare-sandbox.ts +++ b/packages/crypto-commit/test/helpers/prepare-sandbox.ts @@ -21,6 +21,14 @@ import { Serializer } from "../../source/serializer"; export const prepareSandbox = async (context) => { context.sandbox = new Sandbox(); + context.sandbox.app.bind(Identifiers.Cryptography.Commit.ProofSize).toFunction( + () => + 4 + // round + context.sandbox.app.getTagged(Identifiers.Cryptography.Signature.Size, "type", "consensus") + // signature + 1 + + 8, // validator set bitmap); + ); + context.sandbox.app.get(Identifiers.Config.Repository).set("crypto", crypto); context.sandbox.app.bind(Identifiers.Services.EventDispatcher.Service).toConstantValue({ dispatchSync: () => {} }); context.sandbox.app.bind(Identifiers.Services.Log.Service).toConstantValue({}); diff --git a/packages/database/package.json b/packages/database/package.json index 7ff03ec28..d00ec023e 100644 --- a/packages/database/package.json +++ b/packages/database/package.json @@ -25,7 +25,7 @@ "@mainsail/container": "workspace:*", "@mainsail/contracts": "workspace:*", "@mainsail/kernel": "workspace:*", - "lmdb": "3.1.4" + "lmdb": "3.2.0" }, "devDependencies": { "@types/tmp": "0.2.6", diff --git a/packages/database/source/database-service.ts b/packages/database/source/database-service.ts index b67b6a1f5..5918a7273 100644 --- a/packages/database/source/database-service.ts +++ b/packages/database/source/database-service.ts @@ -1,5 +1,6 @@ import { inject, injectable } from "@mainsail/container"; import { Contracts, Identifiers } from "@mainsail/contracts"; +import { Utils } from "@mainsail/kernel"; import * as lmdb from "lmdb"; @injectable() @@ -7,6 +8,9 @@ export class DatabaseService implements Contracts.Database.DatabaseService { @inject(Identifiers.Database.Root) private readonly rootDb!: lmdb.RootDatabase; + @inject(Identifiers.Database.Storage.Commit) + private readonly commitStorage!: lmdb.Database; + @inject(Identifiers.Database.Storage.Block) private readonly blockStorage!: lmdb.Database; @@ -16,11 +20,34 @@ export class DatabaseService implements Contracts.Database.DatabaseService { @inject(Identifiers.Database.Storage.State) private readonly stateStorage!: lmdb.Database; + @inject(Identifiers.Database.Storage.Transaction) + private readonly transactionStorage!: lmdb.Database; + + @inject(Identifiers.Database.Storage.TransactionIds) + private readonly transactionIdsStorage!: lmdb.Database; + @inject(Identifiers.Cryptography.Commit.Factory) private readonly commitFactory!: Contracts.Crypto.CommitFactory; - #blockCache = new Map(); + @inject(Identifiers.Cryptography.Block.Factory) + private readonly blockFactory!: Contracts.Crypto.BlockFactory; + + @inject(Identifiers.Cryptography.Block.Deserializer) + private readonly blockDeserializer!: Contracts.Crypto.BlockDeserializer; + + @inject(Identifiers.Cryptography.Transaction.Factory) + private readonly transactionFactory!: Contracts.Crypto.TransactionFactory; + + @inject(Identifiers.Cryptography.Commit.ProofSize) + private readonly proofSize!: () => number; + + @inject(Identifiers.Cryptography.Block.HeaderSize) + private readonly headerSize!: () => number; + + #commitCache = new Map(); #blockIdCache = new Map(); + #transactionCache = new Map(); + #state = { height: 0, totalRound: 0 }; public async initialize(): Promise { @@ -39,11 +66,11 @@ export class DatabaseService implements Contracts.Database.DatabaseService { } public isEmpty(): boolean { - return this.#blockCache.size === 0 && this.blockStorage.getKeysCount() === 0; + return this.#commitCache.size === 0 && this.blockStorage.getKeysCount() === 0; } public async getCommit(height: number): Promise { - const bytes = this.#get(height); + const bytes = this.#readCommitBytes(height); if (bytes) { return await this.commitFactory.fromBytes(bytes); @@ -59,7 +86,7 @@ export class DatabaseService implements Contracts.Database.DatabaseService { return undefined; } - const bytes = this.#get(height); + const bytes = this.#readCommitBytes(height); if (bytes) { return await this.commitFactory.fromBytes(bytes); } @@ -81,7 +108,7 @@ export class DatabaseService implements Contracts.Database.DatabaseService { return heights .map((height: number) => { try { - return this.#get(height); + return this.#readCommitBytes(height); } catch { return; } @@ -89,6 +116,56 @@ export class DatabaseService implements Contracts.Database.DatabaseService { .filter((block): block is Buffer => !!block); } + public async getBlock(height: number): Promise { + const bytes = this.#readBlockBytes(height); + + if (bytes) { + return await this.blockFactory.fromBytes(bytes); + } + + return undefined; + } + + public async getBlockById(id: string): Promise { + const height = this.#getHeightById(id); + + if (height === undefined) { + return undefined; + } + + const bytes = this.#readBlockBytes(height); + if (bytes) { + return await this.blockFactory.fromBytes(bytes); + } + + return undefined; + } + + public async getBlockHeader(height: number): Promise { + const bytes = this.#readBlockHeaderBytes(height); + + if (bytes) { + return await this.blockDeserializer.deserializeHeader(bytes); + } + + return undefined; + } + + public async getBlockHeaderById(id: string): Promise { + const height = this.#getHeightById(id); + + if (height === undefined) { + return undefined; + } + + const bytes = this.#readBlockHeaderBytes(height); + if (bytes) { + return await this.blockDeserializer.deserializeHeader(bytes); + } + + return undefined; + } + public async findBlocks(start: number, end: number): Promise { return await this.#map( await this.findCommitBuffers(start, end), @@ -96,16 +173,23 @@ export class DatabaseService implements Contracts.Database.DatabaseService { ); } - public async findCommits(start: number, end: number): Promise { - return await this.#map( - await this.findCommitBuffers(start, end), - async (block: Buffer) => await this.commitFactory.fromBytes(block), - ); + public async getTransactionById(id: string): Promise { + if (this.#transactionCache.has(id)) { + return this.#transactionCache.get(id); + } + + const transactionBytes: Buffer | undefined = this.transactionStorage.get(id); + + if (!transactionBytes) { + return undefined; + } + + return await this.transactionFactory.fromBytes(transactionBytes); } public async *readCommits(start: number, end: number): AsyncGenerator { for (let height = start; height <= end; height++) { - const data = this.#get(height); + const data = this.#readCommitBytes(height); if (!data) { throw new Error(`Failed to read commit at height ${height}`); @@ -121,27 +205,41 @@ export class DatabaseService implements Contracts.Database.DatabaseService { throw new Error("Database is empty"); } - if (this.#blockCache.size > 0) { - return [...this.#blockCache.values()].pop()!; + if (this.#commitCache.size > 0) { + return [...this.#commitCache.values()].pop()!; } - return await this.commitFactory.fromBytes( - this.blockStorage.getRange({ limit: 1, reverse: true }).asArray[0].value, - ); + const height = this.blockIdStorage.getRange({ limit: 1, reverse: true }).asArray[0].value; + return await this.commitFactory.fromBytes(this.#readCommitBytes(height)!); } public addCommit(commit: Contracts.Crypto.Commit): void { - this.#blockCache.set(commit.block.data.height, commit); + this.#commitCache.set(commit.block.data.height, commit); this.#blockIdCache.set(commit.block.data.id, commit.block.data.height); + for (const tx of commit.block.transactions) { + this.#transactionCache.set(tx.id, tx); + } + this.#state.height = commit.block.data.height; this.#state.totalRound += commit.proof.round + 1; } async persist(): Promise { await this.rootDb.transaction(() => { - for (const [height, commit] of this.#blockCache.entries()) { - void this.blockStorage.put(height, Buffer.from(commit.serialized, "hex")); + for (const [height, commit] of this.#commitCache.entries()) { + const proofSize = this.proofSize(); + const buff = Buffer.from(commit.serialized, "hex"); // TODO: Slice to reduce buffer size + + void this.commitStorage.put(height, buff.subarray(0, proofSize)); + void this.blockStorage.put(height, buff.subarray(proofSize, proofSize + this.headerSize())); + void this.transactionIdsStorage.put( + height, + commit.block.transactions.map((tx) => tx.id), + ); + for (const tx of commit.block.transactions) { + void this.transactionStorage.put(tx.id, tx.serialized); + } void this.blockIdStorage.put(commit.block.data.id, height); } @@ -150,15 +248,7 @@ export class DatabaseService implements Contracts.Database.DatabaseService { await this.rootDb.flushed; - this.#blockCache.clear(); - } - - #get(height: number): Buffer | undefined { - if (this.#blockCache.has(height)) { - return Buffer.from(this.#blockCache.get(height)!.serialized, "hex"); - } - - return this.blockStorage.get(height); + this.#commitCache.clear(); } #getHeightById(id: string): number | undefined { @@ -169,6 +259,59 @@ export class DatabaseService implements Contracts.Database.DatabaseService { return this.blockIdStorage.get(id); } + #readCommitBytes(height: number): Buffer | undefined { + if (this.#commitCache.has(height)) { + return Buffer.from(this.#commitCache.get(height)!.serialized, "hex"); + } + + const commitBuffer: Buffer | undefined = this.commitStorage.get(height); + if (!commitBuffer) { + return; + } + + const blockBuffer: Buffer | undefined = this.#readBlockBytes(height); + Utils.assert.defined(blockBuffer); + + return Buffer.concat([commitBuffer, blockBuffer]); + } + + #readBlockBytes(height: number): Buffer | undefined { + if (this.#commitCache.has(height)) { + return Buffer.from(this.#commitCache.get(height)!.serialized, "hex").subarray(this.proofSize()); + } + + const blockBuffer: Buffer | undefined = this.blockStorage.get(height); + if (!blockBuffer) { + return; + } + + const transactionIds: string[] | undefined = this.transactionIdsStorage.get(height); + Utils.assert.defined(transactionIds); + + const transactions: Buffer[] = []; + for (const id of transactionIds) { + const transaction: Buffer | undefined = this.transactionStorage.get(id); + Utils.assert.defined(transaction); + + const sizeBuff = Buffer.alloc(2); + sizeBuff.writeUInt16LE(transaction.length, 0); + transactions.push(sizeBuff, transaction); + } + + return Buffer.concat([blockBuffer, ...transactions]); + } + + #readBlockHeaderBytes(height: number): Buffer | undefined { + if (this.#commitCache.has(height)) { + return Buffer.from(this.#commitCache.get(height)!.serialized, "hex").subarray( + this.proofSize(), + this.proofSize() + this.headerSize(), + ); + } + + return this.blockStorage.get(height); + } + async #map(data: unknown[], callback: (...arguments_: any[]) => Promise): Promise { const result: T[] = []; diff --git a/packages/database/source/index.ts b/packages/database/source/index.ts index 26cddbc2e..27f8d150d 100644 --- a/packages/database/source/index.ts +++ b/packages/database/source/index.ts @@ -35,5 +35,12 @@ export class ServiceProvider extends Providers.ServiceProvider { this.app.bind(Identifiers.Database.Storage.Block).toConstantValue(rootStorage.openDB({ name: "blocks" })); this.app.bind(Identifiers.Database.Storage.BlockId).toConstantValue(rootStorage.openDB({ name: "block-id" })); this.app.bind(Identifiers.Database.Storage.State).toConstantValue(rootStorage.openDB({ name: "state" })); + this.app.bind(Identifiers.Database.Storage.Commit).toConstantValue(rootStorage.openDB({ name: "commit" })); + this.app + .bind(Identifiers.Database.Storage.Transaction) + .toConstantValue(rootStorage.openDB({ name: "transaction" })); + this.app + .bind(Identifiers.Database.Storage.TransactionIds) + .toConstantValue(rootStorage.openDB({ name: "transactionIds" })); } } diff --git a/packages/p2p/source/peer-verifier.ts b/packages/p2p/source/peer-verifier.ts index d737b8e4a..5aa767f60 100644 --- a/packages/p2p/source/peer-verifier.ts +++ b/packages/p2p/source/peer-verifier.ts @@ -88,7 +88,7 @@ export class PeerVerifier implements Contracts.P2P.PeerVerifier { const receivedCommit = await this.commitFactory.fromBytes(blocks[0]); const blockToCompare = - block.data.height === heightToRequest ? block : (await this.database.getCommit(heightToRequest))?.block; + block.data.height === heightToRequest ? block : await this.database.getBlock(heightToRequest); Utils.assert.defined(blockToCompare); diff --git a/packages/transaction-pool-worker/source/handlers/get-transactions.ts b/packages/transaction-pool-worker/source/handlers/get-transactions.ts index 9b051a194..2e472c05b 100644 --- a/packages/transaction-pool-worker/source/handlers/get-transactions.ts +++ b/packages/transaction-pool-worker/source/handlers/get-transactions.ts @@ -9,12 +9,12 @@ export class GetTransactionsHandler { @inject(Identifiers.Cryptography.Configuration) private readonly configuration!: Contracts.Crypto.Configuration; - @inject(Identifiers.Cryptography.Block.Serializer) - private readonly blockSerializer!: Contracts.Crypto.BlockSerializer; + @inject(Identifiers.Cryptography.Block.HeaderSize) + private readonly headerSize!: () => number; public async handle(): Promise { const milestone = this.configuration.getMilestone(); - let bytesLeft: number = milestone.block.maxPayload - this.blockSerializer.headerSize(); + let bytesLeft: number = milestone.block.maxPayload - this.headerSize(); const candidateTransactions: Contracts.Crypto.Transaction[] = []; for (const transaction of await this.poolQuery.getFromHighestPriority().all()) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 497209db9..21970026b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -736,8 +736,8 @@ importers: specifier: workspace:* version: link:../kernel lmdb: - specifier: 3.1.4 - version: 3.1.4 + specifier: 3.2.0 + version: 3.2.0 devDependencies: uvu: specifier: ^0.5.6 @@ -1771,8 +1771,8 @@ importers: specifier: workspace:* version: link:../kernel lmdb: - specifier: 3.1.4 - version: 3.1.4 + specifier: 3.2.0 + version: 3.2.0 devDependencies: '@types/tmp': specifier: 0.2.6 @@ -3540,33 +3540,33 @@ packages: resolution: {integrity: sha512-0hZxUPKnHwehUO2xC4ldtdX9bW0W1UosxebDIQlZL2STnZnA2IFmIk2lJVUyFW+cmTPQzV93jfS0i69T9Z+teg==} engines: {node: ^14.17.0 || >=16.0.0} - '@lmdb/lmdb-darwin-arm64@3.1.4': - resolution: {integrity: sha512-bXwBegGNDGAlshGqUmV8MxVFPsqEpU2yWWxoJ4AA4UkEd7gA1Rzh7KtvM5Tww0dQfp5t+P/SjNV3vjpAgof+uA==} + '@lmdb/lmdb-darwin-arm64@3.2.0': + resolution: {integrity: sha512-Ca5N6DGDlH/lIycMj2U3FtokNPdUmGyL+htto3G+gexoXYaDE9cbojVgwXd3/Zih9Friqh7l5qZk+LZEVDwJvQ==} cpu: [arm64] os: [darwin] - '@lmdb/lmdb-darwin-x64@3.1.4': - resolution: {integrity: sha512-cb1/yeoUfWhVWqo8VImUWo6bXVn57AHPX98VqIkpfRw5Yh0z2DHPBZmsELb1OLJKeikAAOyxM+vPWRJYtAg9rw==} + '@lmdb/lmdb-darwin-x64@3.2.0': + resolution: {integrity: sha512-s/MXLuRXxJjQpg0aM/yN3FJh34tqEPo6Zg+FJvc9+gUNpzXzZwBB9MOTYA05WVrvxwtIKxMg7ocLjAH1LQUT3A==} cpu: [x64] os: [darwin] - '@lmdb/lmdb-linux-arm64@3.1.4': - resolution: {integrity: sha512-Hs1cmv8SKEkczsiQbRYVeqI7vzpJ0LI29RyeaVNDDFJxzoua7IcuyG0wSXu12kpXlGTTLVOh1Wp4rK79Ixpxmg==} + '@lmdb/lmdb-linux-arm64@3.2.0': + resolution: {integrity: sha512-XRkaZok4AkzMXKLfsdJYVBXYJ/6idDpuLIPGiVjelxKLbZIKB7F+Xp2BDfeelAPdjRbW/qhzF7FNA0u1blz/Og==} cpu: [arm64] os: [linux] - '@lmdb/lmdb-linux-arm@3.1.4': - resolution: {integrity: sha512-9O3kU6i7crV0vi+ImbZG6SkD+T8sxjbugq4pY424tjV8X/EjSfs1E0n25We5Z7qpJFxZSJZKsv40tJlz1w4pLg==} + '@lmdb/lmdb-linux-arm@3.2.0': + resolution: {integrity: sha512-e9pljI8rZk1UAaDdi7sGiY0zkqsNAS3a4llOuk2UslAH4UP9vGZfjfCR5D+HKPUPbSEk28adOiNmIUT4N2lTBw==} cpu: [arm] os: [linux] - '@lmdb/lmdb-linux-x64@3.1.4': - resolution: {integrity: sha512-e/xPxFjSBzuN7/nb5WBYO1t9X1NBiNYy+gvWB3rb95K2W5qJU9fnjx+CNFp7ucvQZWF08EsVzMBa7eXKGGmHjg==} + '@lmdb/lmdb-linux-x64@3.2.0': + resolution: {integrity: sha512-c8HMb044qzMT/wvk4HzBesRv3wQNeFkUFz6laH3FKVs0+ztM7snuT3izPWdeYhgCLkAiIqshqlcbvzQfPDeg2Q==} cpu: [x64] os: [linux] - '@lmdb/lmdb-win32-x64@3.1.4': - resolution: {integrity: sha512-mUcWgKmpbquKaDEcJ+FBtJpcqHvJW2Ce+GKMP/B/Hm9IxGjUfGs0aGlax2Nh/mjzXx/7qfwyCGD8y+KXfDuMsA==} + '@lmdb/lmdb-win32-x64@3.2.0': + resolution: {integrity: sha512-xcrdSOPtpZ4ScWJM2x4g+eWCOctINOcaEWGSvZbmXPFD69SAFywyhqNsB3snAY3assYV0B52PWmiAwXWfijd+g==} cpu: [x64] os: [win32] @@ -7233,8 +7233,8 @@ packages: resolution: {integrity: sha512-RmAl7su35BFd/xoMamRjpIE4j3v+L28o8CT5YhAXQJm1fD+1l9ngXY8JAQRJ+tFK2i5njvi0iRUKV09vPwA0iA==} engines: {node: '>=6'} - lmdb@3.1.4: - resolution: {integrity: sha512-M3P0HBm0e6SUCrWqMf+wjs5LTRQekh1TtC10e3NjCfLECCXLRXHFROG2kErdyyv2MQ5335Qvb3KQKCSLUuwUDg==} + lmdb@3.2.0: + resolution: {integrity: sha512-cDeZAM4mXOwN1IdH93a91qXppn4jXV4NHphg53bqQDRFjJnpYZTgGcjrqpsmm209DtXTvmKMcYJd+XrHybwFZg==} hasBin: true load-json-file@4.0.0: @@ -7644,8 +7644,8 @@ packages: resolution: {integrity: sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==} hasBin: true - msgpackr@1.10.2: - resolution: {integrity: sha512-L60rsPynBvNE+8BWipKKZ9jHcSGbtyJYIwjRq0VrIvQ08cRjntGXJYW/tmciZ2IHWIY8WEW32Qa2xbh5+SKBZA==} + msgpackr@1.11.2: + resolution: {integrity: sha512-F9UngXRlPyWCDEASDpTf6c9uNhGPTqnTeLVt7bN+bU1eajoR/8V9ys2BRaV5C/e5ihE6sJ9uPIKaYt6bFuO32g==} multimatch@5.0.0: resolution: {integrity: sha512-ypMKuglUrZUD99Tk2bUQ+xNQj43lPEfAeX2o9cTteAmShXy2VHDJpuwu1o0xqoKCt9jLVAvwyFKdLTPXKAfJyA==} @@ -10593,22 +10593,22 @@ snapshots: - nx - supports-color - '@lmdb/lmdb-darwin-arm64@3.1.4': + '@lmdb/lmdb-darwin-arm64@3.2.0': optional: true - '@lmdb/lmdb-darwin-x64@3.1.4': + '@lmdb/lmdb-darwin-x64@3.2.0': optional: true - '@lmdb/lmdb-linux-arm64@3.1.4': + '@lmdb/lmdb-linux-arm64@3.2.0': optional: true - '@lmdb/lmdb-linux-arm@3.1.4': + '@lmdb/lmdb-linux-arm@3.2.0': optional: true - '@lmdb/lmdb-linux-x64@3.1.4': + '@lmdb/lmdb-linux-x64@3.2.0': optional: true - '@lmdb/lmdb-win32-x64@3.1.4': + '@lmdb/lmdb-win32-x64@3.2.0': optional: true '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': @@ -14905,20 +14905,20 @@ snapshots: - zen-observable - zenObservable - lmdb@3.1.4: + lmdb@3.2.0: dependencies: - msgpackr: 1.10.2 + msgpackr: 1.11.2 node-addon-api: 6.1.0 node-gyp-build-optional-packages: 5.2.2 ordered-binary: 1.5.3 weak-lru-cache: 1.2.2 optionalDependencies: - '@lmdb/lmdb-darwin-arm64': 3.1.4 - '@lmdb/lmdb-darwin-x64': 3.1.4 - '@lmdb/lmdb-linux-arm': 3.1.4 - '@lmdb/lmdb-linux-arm64': 3.1.4 - '@lmdb/lmdb-linux-x64': 3.1.4 - '@lmdb/lmdb-win32-x64': 3.1.4 + '@lmdb/lmdb-darwin-arm64': 3.2.0 + '@lmdb/lmdb-darwin-x64': 3.2.0 + '@lmdb/lmdb-linux-arm': 3.2.0 + '@lmdb/lmdb-linux-arm64': 3.2.0 + '@lmdb/lmdb-linux-x64': 3.2.0 + '@lmdb/lmdb-win32-x64': 3.2.0 load-json-file@4.0.0: dependencies: @@ -15437,7 +15437,7 @@ snapshots: '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.3 optional: true - msgpackr@1.10.2: + msgpackr@1.11.2: optionalDependencies: msgpackr-extract: 3.0.3