feat(database): split storage to commits, blocks and transactions (#794)

* Add net_version

* Register handler

* Return nethash

* Add test

* Add identifiers

* Open databases

* Extract ProofSize function

* Split commit and block storage

* Rename variable

* Extract headerSize

* Store transactions

* Remove findCommits

* Reorder

* Add getBlock & getBlockBy id

* Add getTransactionById

* Update lmdb version

* Update getBlockById

* Replace getCommit with get block

* Add getBlockHeader

* Fix typo

* Fix tests

packages/api-development/source/controllers/blocks.ts

@@ -99,6 +99,6 @@ export class BlocksController extends Controller {
 
 	// TODO: Support height only
 	private async getBlock(idOrHeight: string): Promise<Contracts.Crypto.Block | undefined> {
-		return (await this.database.getCommit(Number.parseInt(idOrHeight)))?.block;
+		return await this.database.getBlock(Number.parseInt(idOrHeight));
 	}
 }

packages/api-evm/source/actions/eth-get-block-by-hash.ts

@@ -26,13 +26,13 @@ export class EthGetBlockByHashAction implements Contracts.Api.RPC.Action {
 	public async handle(parameters: [string, boolean]): Promise<object | null> {
 		const transactionObject = parameters[1];
 
-		const commit = await this.databaseService.getCommitById(parameters[0].slice(2));
+		const block = await this.databaseService.getBlockById(parameters[0].slice(2));
 
-		if (!commit) {
+		if (!block) {
 			// eslint-disable-next-line unicorn/no-null
 			return null;
 		}
 
-		return this.app.resolve(BlockResource).transform(commit.block, transactionObject);
+		return this.app.resolve(BlockResource).transform(block, transactionObject);
 	}
 }

packages/api-evm/source/actions/eth-get-block-by-number.ts

@@ -33,13 +33,13 @@ export class EthGetBlockByNumberAction implements Contracts.Api.RPC.Action {
 		const height = parameters[0].startsWith("0x") ? Number.parseInt(parameters[0]) : this.stateStore.getHeight();
 		const transactionObject = parameters[1];
 
-		const commit = await this.databaseService.getCommit(height);
+		const block = await this.databaseService.getBlock(height);
 
-		if (!commit) {
+		if (!block) {
 			// eslint-disable-next-line unicorn/no-null
 			return null;
 		}
 
-		return this.app.resolve(BlockResource).transform(commit.block, transactionObject);
+		return this.app.resolve(BlockResource).transform(block, transactionObject);
 	}
 }

packages/api-sync/source/restore.ts

@@ -194,15 +194,15 @@ export class Restore {
 		let currentHeight = 0;
 
 		do {
-			const commits = await this.databaseService.findCommits(
+			const commits = this.databaseService.readCommits(
 				Math.min(currentHeight, mostRecentCommit.block.header.height),
 				Math.min(currentHeight + BATCH_SIZE, mostRecentCommit.block.header.height),
 			);
 
 			const blocks: Models.Block[] = [];
 			const transactions: Models.Transaction[] = [];
 
-			for (const { proof, block } of commits) {
+			for await (const { proof, block } of commits) {
 				blocks.push({
 					commitRound: proof.round,
 					generatorAddress: block.header.generatorAddress,

packages/bootstrap/source/bootstrapper.ts

@@ -96,13 +96,13 @@ export class Bootstrapper {
 	}
 
 	async #checkStoredGenesisCommit(): Promise<void> {
-		const genesisCommit = await this.databaseService.getCommit(0);
+		const genesisBlock = await this.databaseService.getBlock(0);
 
-		if (!genesisCommit) {
+		if (!genesisBlock) {
 			return;
 		}
 
-		if (this.stateStore.getGenesisCommit().block.data.id !== genesisCommit.block.data.id) {
+		if (this.stateStore.getGenesisCommit().block.data.id !== genesisBlock.data.id) {
 			throw new Error("Block from crypto.json doesn't match stored genesis block");
 		}
 	}

packages/consensus-storage/package.json

@@ -25,7 +25,7 @@
 		"@mainsail/container": "workspace:*",
 		"@mainsail/contracts": "workspace:*",
 		"@mainsail/kernel": "workspace:*",
-		"lmdb": "3.1.4"
+		"lmdb": "3.2.0"
 	},
 	"devDependencies": {
 		"uvu": "^0.5.6"

packages/contracts/source/contracts/crypto/block.ts

@@ -75,8 +75,6 @@ export interface BlockFactory {
 }
 
 export interface BlockSerializer {
-	headerSize(): number;
-
 	totalSize(block: BlockDataSerializable): number;
 
 	serializeHeader(block: BlockDataSerializable): Promise<Buffer>;

packages/contracts/source/contracts/crypto/commit.ts

@@ -33,8 +33,6 @@ export interface CommitProof {
 }
 
 export interface CommitSerializer {
-	proofSize(): number;
-
 	serializeCommitProof(proof: CommitProof): Promise<Buffer>;
 
 	serializeCommit(commit: CommitSerializable): Promise<Buffer>;

packages/contracts/source/contracts/database.ts

@@ -1,5 +1,4 @@
-import { Block } from "./crypto/block.js";
-import { Commit } from "./crypto/commit.js";
+import { Block, BlockHeader, Commit, Transaction } from "./crypto/index.js";
 
 export interface State {
 	height: number;
@@ -11,15 +10,23 @@ export interface DatabaseService {
 	isEmpty(): boolean;
 
 	getState(): State;
+
 	getCommit(height: number): Promise<Commit | undefined>;
 	getCommitById(id: string): Promise<Commit | undefined>;
+	getLastCommit(): Promise<Commit>;
 	hasCommitById(id: string): boolean;
 	findCommitBuffers(start: number, end: number): Promise<Buffer[]>;
 	readCommits(start: number, end: number): AsyncGenerator<Commit>;
+
+	getBlock(height: number): Promise<Block | undefined>;
+	getBlockById(id: string): Promise<Block | undefined>;
 	findBlocks(start: number, end: number): Promise<Block[]>;
-	findCommits(start: number, end: number): Promise<Commit[]>;
 
-	getLastCommit(): Promise<Commit>;
+	getBlockHeader(height: number): Promise<BlockHeader | undefined>;
+	getBlockHeaderById(id: string): Promise<BlockHeader | undefined>;
+
+	getTransactionById(id: string): Promise<Transaction | undefined>;
+
 	addCommit(block: Commit): void;
 	persist(): Promise<void>;
 }

packages/contracts/source/identifiers.ts

@@ -57,13 +57,15 @@ export const Identifiers = {
 		Block: {
 			Deserializer: Symbol("Crypto<Block.Deserializer>"),
 			Factory: Symbol("Crypto<Block.Factory>"),
+			HeaderSize: Symbol("Crypto<Block.HeaderSize>"),
 			IDFactory: Symbol("Crypto<Block.IDFactory>"),
 			Serializer: Symbol("Crypto<Block.Serializer>"),
 			Verifier: Symbol("Crypto<Block.Verifier>"),
 		},
 		Commit: {
 			Deserializer: Symbol("Crypto<Commit.Deserializer>"),
 			Factory: Symbol("Crypto<Commit.Factory>"),
+			ProofSize: Symbol("Crypto<Commit.ProofSize>"),
 			Serializer: Symbol("Crypto<Commit.Serializer>"),
 		},
 		Configuration: Symbol("Crypto<Configuration>"),
@@ -130,7 +132,10 @@ export const Identifiers = {
 		Storage: {
 			Block: Symbol("Database<Storage.Block>"),
 			BlockId: Symbol("Database<Storage.BlockId>"),
+			Commit: Symbol("Database<Storage.Commit>"),
 			State: Symbol("Database<Storage.State>"),
+			Transaction: Symbol("Database<Storage.Transaction>"),
+			TransactionIds: Symbol("Database<Storage.TransactionIds>"),
 		},
 	},
 	Evm: {

packages/crypto-block/source/deserializer.ts

@@ -18,8 +18,8 @@ export class Deserializer implements Contracts.Crypto.BlockDeserializer {
 	@inject(Identifiers.Cryptography.Serializer)
 	private readonly serializer!: Contracts.Serializer.Serializer;
 
-	@inject(Identifiers.Cryptography.Block.Serializer)
-	private readonly blockSerializer!: Contracts.Crypto.BlockSerializer;
+	@inject(Identifiers.Cryptography.Block.HeaderSize)
+	private readonly headerSize!: () => number;
 
 	public async deserializeHeader(serialized: Buffer): Promise<Contracts.Crypto.BlockHeader> {
 		const buffer: ByteBuffer = ByteBuffer.fromBuffer(serialized);
@@ -51,7 +51,7 @@ export class Deserializer implements Contracts.Crypto.BlockDeserializer {
 		const block = {} as Contracts.Crypto.BlockHeader;
 
 		await this.serializer.deserialize<Contracts.Crypto.BlockData>(buffer, block, {
-			length: this.blockSerializer.headerSize(),
+			length: this.headerSize(),
 			schema: {
 				version: {
 					type: "uint8",

packages/crypto-block/source/index.ts

@@ -17,6 +17,28 @@ export * from "./verifier.js";
 
 export class ServiceProvider extends Providers.ServiceProvider {
 	public async register(): Promise<void> {
+		this.app.bind(Identifiers.Cryptography.Block.HeaderSize).toFunction(() => {
+			const hashByteLength = this.app.get<number>(Identifiers.Cryptography.Hash.Size.SHA256);
+			const generatorAddressByteLength = this.app.get<number>(Identifiers.Cryptography.Identity.Address.Size);
+
+			return (
+				1 + // version
+				6 + // timestamp
+				4 + // height
+				4 + // round
+				hashByteLength + // previousBlock
+				hashByteLength + // stateHash
+				2 + // numberOfTransactions
+				4 + // totalGasUsed
+				32 + // totalAmount
+				32 + // totalFee
+				32 + // reward
+				4 + // payloadLength
+				hashByteLength + // payloadHash
+				generatorAddressByteLength
+			);
+		});
+
 		this.app.bind(Identifiers.Cryptography.Block.Deserializer).to(Deserializer).inSingletonScope();
 		this.app.bind(Identifiers.Cryptography.Block.Factory).to(BlockFactory).inSingletonScope();
 		this.app.bind(Identifiers.Cryptography.Block.IDFactory).to(IDFactory).inSingletonScope();

packages/crypto-block/source/serializer.ts

@@ -8,30 +8,8 @@ export class Serializer implements Contracts.Crypto.BlockSerializer {
 	@tagged("type", "wallet")
 	private readonly serializer!: Contracts.Serializer.Serializer;
 
-	@inject(Identifiers.Cryptography.Hash.Size.SHA256)
-	private readonly hashByteLength!: number;
-
-	@inject(Identifiers.Cryptography.Identity.Address.Size)
-	private readonly generatorAddressByteLength!: number;
-
-	public headerSize(): number {
-		return (
-			1 + // version
-			6 + // timestamp
-			4 + // height
-			4 + // round
-			this.hashByteLength + // previousBlock
-			this.hashByteLength + // stateHash
-			2 + // numberOfTransactions
-			4 + // totalGasUsed
-			32 + // totalAmount
-			32 + // totalFee
-			32 + // reward
-			4 + // payloadLength
-			this.hashByteLength + // payloadHash
-			this.generatorAddressByteLength
-		);
-	}
+	@inject(Identifiers.Cryptography.Block.HeaderSize)
+	private readonly headerSize!: () => number;
 
 	public totalSize(block: Contracts.Crypto.BlockDataSerializable): number {
 		return this.headerSize() + block.payloadLength;

packages/crypto-block/source/verifier.ts

@@ -7,12 +7,12 @@ export class Verifier implements Contracts.Crypto.BlockVerifier {
 	@inject(Identifiers.Cryptography.Configuration)
 	private readonly configuration!: Contracts.Crypto.Configuration;
 
-	@inject(Identifiers.Cryptography.Block.Serializer)
-	private readonly serializer!: Contracts.Crypto.BlockSerializer;
-
 	@inject(Identifiers.Cryptography.Hash.Factory)
 	private readonly hashFactory!: Contracts.Crypto.HashFactory;
 
+	@inject(Identifiers.Cryptography.Block.HeaderSize)
+	private readonly headerSize!: () => number;
+
 	public async verify(block: Contracts.Crypto.Block): Promise<Contracts.Crypto.BlockVerification> {
 		const blockData: Contracts.Crypto.BlockData = block.data;
 		const result: Utils.Mutable<Contracts.Crypto.BlockVerification> = {
@@ -45,8 +45,7 @@ export class Verifier implements Contracts.Crypto.BlockVerifier {
 				result.errors.push("Invalid block version");
 			}
 
-			const headerSize = this.serializer.headerSize();
-			const totalSize = headerSize + block.header.payloadLength;
+			const totalSize = this.headerSize() + block.header.payloadLength;
 			if (totalSize > constants.block.maxPayload) {
 				result.errors.push(`Payload is too large: ${totalSize} > ${constants.block.maxPayload}`);
 			}

packages/crypto-block/test/helpers/prepare-sandbox.ts

@@ -22,6 +22,30 @@ import { Serializer } from "../../source/serializer";
 export const prepareSandbox = async (context) => {
 	context.sandbox = new Sandbox();
 
+	context.sandbox.app.bind(Identifiers.Cryptography.Block.HeaderSize).toFunction(() => {
+		const hashByteLength = context.sandbox.app.get<number>(Identifiers.Cryptography.Hash.Size.SHA256);
+		const generatorAddressByteLength = context.sandbox.app.get<number>(
+			Identifiers.Cryptography.Identity.Address.Size,
+		);
+
+		return (
+			1 + // version
+			6 + // timestamp
+			4 + // height
+			4 + // round
+			hashByteLength + // previousBlock
+			hashByteLength + // stateHash
+			2 + // numberOfTransactions
+			4 + // totalGasUsed
+			32 + // totalAmount
+			32 + // totalFee
+			32 + // reward
+			4 + // payloadLength
+			hashByteLength + // payloadHash
+			generatorAddressByteLength
+		);
+	});
+
 	context.sandbox.app.get<Contracts.Kernel.Repository>(Identifiers.Config.Repository).set("crypto", crypto);
 	context.sandbox.app.bind(Identifiers.Services.EventDispatcher.Service).toConstantValue({ dispatchSync: () => {} });
 	context.sandbox.app.bind(Identifiers.Services.Log.Service).toConstantValue({});

packages/crypto-commit/source/deserializer.ts

@@ -7,16 +7,16 @@ export class Deserializer implements Contracts.Crypto.CommitDeserializer {
 	@inject(Identifiers.Cryptography.Serializer)
 	private readonly serializer!: Contracts.Serializer.Serializer;
 
-	@inject(Identifiers.Cryptography.Commit.Serializer)
-	private readonly commitSerializer!: Contracts.Crypto.CommitSerializer;
+	@inject(Identifiers.Cryptography.Commit.ProofSize)
+	private readonly proofSize!: () => number;
 
 	public async deserializeCommitProof(serialized: Buffer): Promise<Contracts.Crypto.CommitProof> {
 		const buffer: ByteBuffer = ByteBuffer.fromBuffer(serialized);
 
 		const proof = {} as Contracts.Crypto.CommitProof;
 
 		await this.serializer.deserialize<Contracts.Crypto.CommitProof>(buffer, proof, {
-			length: this.commitSerializer.proofSize(),
+			length: this.proofSize(),
 			schema: {
 				round: {
 					type: "uint32",

packages/crypto-commit/source/factory.ts

@@ -7,16 +7,16 @@ export class CommitFactory implements Contracts.Crypto.CommitFactory {
 	@inject(Identifiers.Cryptography.Block.Factory)
 	private readonly blockFactory!: Contracts.Crypto.BlockFactory;
 
-	@inject(Identifiers.Cryptography.Commit.Serializer)
-	private readonly commitSerializer!: Contracts.Crypto.CommitSerializer;
-
 	@inject(Identifiers.Cryptography.Commit.Deserializer)
 	private readonly commitDeserializer!: Contracts.Crypto.CommitDeserializer;
 
+	@inject(Identifiers.Cryptography.Commit.ProofSize)
+	private readonly proofSize!: () => number;
+
 	public async fromBytes(buff: Buffer): Promise<Contracts.Crypto.Commit> {
 		const buffer = ByteBuffer.fromBuffer(buff);
 
-		const proofBuffer = buffer.readBytes(this.commitSerializer.proofSize());
+		const proofBuffer = buffer.readBytes(this.proofSize());
 		const proof = await this.commitDeserializer.deserializeCommitProof(proofBuffer);
 
 		const block = await this.blockFactory.fromBytes(buffer.getRemainder());

packages/crypto-commit/source/index.ts

@@ -7,6 +7,14 @@ import { Serializer } from "./serializer.js";
 
 export class ServiceProvider extends Providers.ServiceProvider {
 	public async register(): Promise<void> {
+		this.app.bind(Identifiers.Cryptography.Commit.ProofSize).toFunction(
+			() =>
+				4 + // round
+				this.app.getTagged<number>(Identifiers.Cryptography.Signature.Size, "type", "consensus") + // signature
+				1 +
+				8, // validator set bitmap);
+		);
+
 		this.app.bind(Identifiers.Cryptography.Commit.Serializer).to(Serializer).inSingletonScope();
 
 		this.app.bind(Identifiers.Cryptography.Commit.Deserializer).to(Deserializer).inSingletonScope();

packages/crypto-commit/source/serializer.ts

@@ -6,15 +6,8 @@ export class Serializer implements Contracts.Crypto.CommitSerializer {
 	@inject(Identifiers.Cryptography.Serializer)
 	private readonly serializer!: Contracts.Serializer.Serializer;
 
-	@inject(Identifiers.Cryptography.Message.Serializer)
-	private readonly messageSerializer!: Contracts.Crypto.MessageSerializer;
-
-	public proofSize(): number {
-		return (
-			4 + // round
-			+this.messageSerializer.lockProofSize()
-		);
-	}
+	@inject(Identifiers.Cryptography.Commit.ProofSize)
+	private readonly proofSize!: () => number;
 
 	public async serializeCommitProof(commit: Contracts.Crypto.CommitProof): Promise<Buffer> {
 		return this.serializer.serialize<Contracts.Crypto.CommitProof>(commit, {

packages/crypto-commit/test/helpers/prepare-sandbox.ts

@@ -21,6 +21,14 @@ import { Serializer } from "../../source/serializer";
 export const prepareSandbox = async (context) => {
 	context.sandbox = new Sandbox();
 
+	context.sandbox.app.bind(Identifiers.Cryptography.Commit.ProofSize).toFunction(
+		() =>
+			4 + // round
+			context.sandbox.app.getTagged<number>(Identifiers.Cryptography.Signature.Size, "type", "consensus") + // signature
+			1 +
+			8, // validator set bitmap);
+	);
+
 	context.sandbox.app.get<Contracts.Kernel.Repository>(Identifiers.Config.Repository).set("crypto", crypto);
 	context.sandbox.app.bind(Identifiers.Services.EventDispatcher.Service).toConstantValue({ dispatchSync: () => {} });
 	context.sandbox.app.bind(Identifiers.Services.Log.Service).toConstantValue({});